1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/ib/mgt/ibmf/ibmf_saa_impl.h> 27 #include <sys/ib/mgt/ibmf/ibmf_saa_utils.h> 28 29 /* Global sa_access State Pointer */ 30 saa_state_t *saa_statep; 31 _NOTE(READ_ONLY_DATA(saa_statep)) 32 33 extern int ibmf_trace_level; 34 35 extern int ibmf_taskq_max_tasks; 36 37 static int 38 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 39 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags); 40 static int 41 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp, 42 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags); 43 static int 44 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 45 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, 46 hrtime_t trans_send_time, int transport_flags); 47 static int ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, 48 boolean_t sleep_flag, ibmf_msg_t **msgp, uint32_t *transport_flagsp, 49 ibmf_retrans_t *ibmf_retransp); 50 static int ibmf_saa_must_purge(saa_port_t *saa_portp); 51 static void ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp); 52 static void ibmf_saa_impl_destroy_port(saa_port_t *saa_portp); 53 static void ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp); 54 static void ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, 55 int status); 56 static void ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle, 57 void *clnt_private, ibmf_async_event_t event_type); 58 static void ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num); 59 static void ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num); 60 static void ibmf_saa_impl_hca_detach(saa_port_t *saa_removed); 61 static void ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle, 62 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result, 63 size_t *length, boolean_t sleep_flag); 64 static int ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id); 65 static uint_t ibmf_saa_impl_get_attr_id_length(uint16_t attr_id); 66 static void ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp); 67 static int ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop, 68 ib_guid_t *guid_ret); 69 static void ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp, 70 ibt_hca_portinfo_t *portinfop); 71 static void ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, 72 ibmf_msg_t *msgp); 73 static int ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp); 74 75 int ibmf_saa_max_wait_time = IBMF_SAA_MAX_WAIT_TIME_IN_SECS; 76 int ibmf_saa_trans_wait_time = IBMF_SAA_TRANS_WAIT_TIME_IN_SECS; 77 78 /* 79 * ibmf_saa_impl_init: 80 * Allocates memory for the ibmf_saa state structure and initializes the taskq. 81 * Called from the modules init() routine. 82 * 83 * Input Arguments 84 * none 85 * 86 * Output Arguments 87 * none 88 * 89 * Returns 90 * IBMF_NO_RESOURCES if taskq could not be created. 91 * IBMF_SUCCESS on success 92 * 93 */ 94 int 95 ibmf_saa_impl_init() 96 { 97 int res; 98 99 /* CONSTCOND */ 100 ASSERT(NO_COMPETING_THREADS); 101 102 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_init_start, 103 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() enter\n"); 104 105 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_statep)) 106 107 saa_statep = kmem_zalloc(sizeof (saa_state_t), KM_SLEEP); 108 109 /* create taskq for notifying event subscribers */ 110 saa_statep->saa_event_taskq = taskq_create( 111 "ibmf_saa_event_taskq", IBMF_TASKQ_NTHREADS, 112 MINCLSYSPRI, 1, ibmf_taskq_max_tasks, TASKQ_DYNAMIC | 113 TASKQ_PREPOPULATE); 114 if (saa_statep->saa_event_taskq == NULL) { 115 116 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L4, 117 ibmf_saa_impl_init_end_err, 118 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init(): %s\n", 119 tnf_string, msg, "event taskq create failed"); 120 121 kmem_free(saa_statep, sizeof (saa_state_t)); 122 123 res = IBMF_NO_RESOURCES; 124 125 goto bail; 126 } 127 128 mutex_init(&saa_statep->saa_port_list_mutex, NULL, MUTEX_DRIVER, 129 NULL); 130 131 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_statep)) 132 133 res = IBMF_SUCCESS; 134 bail: 135 136 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_init_end, 137 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() exit: status = %d\n", 138 tnf_int, res, res); 139 140 return (res); 141 } 142 143 /* 144 * ibmf_saa_impl_fini: 145 * If there are no registered clients, cleans up all memory associated with the 146 * state, including each of the port list entries. 147 * Called from the modules fini() routine. 148 * 149 * Input Arguments 150 * none 151 * 152 * Output Arguments 153 * none 154 * 155 * Returns 156 * EBUSY if there are outstanding transactions or registered clients 157 * 0 if cleanup was sucessfull 158 * 159 */ 160 int 161 ibmf_saa_impl_fini() 162 { 163 int ret = 0; 164 saa_port_t *saa_portp; 165 saa_port_t *next; 166 167 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_start, 168 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() enter\n"); 169 170 /* make sure there are no registered clients */ 171 mutex_enter(&saa_statep->saa_port_list_mutex); 172 173 saa_portp = saa_statep->saa_port_list; 174 while (saa_portp != NULL) { 175 176 mutex_enter(&saa_portp->saa_pt_mutex); 177 178 if (saa_portp->saa_pt_reference_count > 0) { 179 180 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 181 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "", 182 "ibmf_saa_impl_fini: %s, port %016" PRIx64 "\n", 183 tnf_string, msg, 184 "cannot unload ibmf_saa. Client on port still" 185 " registered", tnf_opaque, port, 186 saa_portp->saa_pt_port_guid); 187 188 mutex_exit(&saa_portp->saa_pt_mutex); 189 190 mutex_exit(&saa_statep->saa_port_list_mutex); 191 192 ret = EBUSY; 193 goto bail; 194 } 195 196 /* make sure there are no outstanding transactions */ 197 198 if (saa_portp->saa_pt_num_outstanding_trans > 0) { 199 200 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1, 201 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "", 202 "ibmf_saa_impl_fini: %s, port = %016" PRIx64 203 ", num transactions = %d\n", 204 tnf_string, msg, "Cannot unload ibmf_saa." 205 " Outstanding transactions on port.", 206 tnf_opaque, port, 207 saa_portp->saa_pt_port_guid, 208 tnf_uint, outstanding_transactions, 209 saa_portp->saa_pt_num_outstanding_trans); 210 211 mutex_exit(&saa_portp->saa_pt_mutex); 212 213 mutex_exit(&saa_statep->saa_port_list_mutex); 214 215 ret = EBUSY; 216 goto bail; 217 } 218 219 mutex_exit(&saa_portp->saa_pt_mutex); 220 221 saa_portp = saa_portp->next; 222 } 223 224 mutex_exit(&saa_statep->saa_port_list_mutex); 225 226 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(saa_statep->saa_port_list, 227 *saa_portp)) 228 229 /* 230 * no more clients nor pending transaction: 231 * unregister ibmf and destroy port entries 232 */ 233 while (saa_statep->saa_port_list != NULL) { 234 235 saa_portp = saa_statep->saa_port_list; 236 next = saa_portp->next; 237 238 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 239 ibmf_saa_impl_fini, IBMF_TNF_TRACE, "", 240 "ibmf_saa_impl_fini: %s, prefix = %016" PRIx64 "\n", 241 tnf_string, msg, "deinitializing port", 242 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 243 244 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp)) 245 246 mutex_enter(&saa_portp->saa_pt_mutex); 247 248 /* unregister from ibmf */ 249 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) { 250 251 mutex_exit(&saa_portp->saa_pt_mutex); 252 253 if (ibmf_saa_impl_ibmf_unreg(saa_portp) 254 != IBMF_SUCCESS) { 255 ret = EBUSY; 256 goto bail; 257 } 258 } else 259 mutex_exit(&saa_portp->saa_pt_mutex); 260 261 ibmf_saa_impl_destroy_port(saa_portp); 262 263 saa_statep->saa_port_list = next; 264 } 265 266 taskq_destroy(saa_statep->saa_event_taskq); 267 268 mutex_destroy(&saa_statep->saa_port_list_mutex); 269 270 kmem_free(saa_statep, sizeof (saa_state_t)); 271 272 bail: 273 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_end, 274 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() exit\n"); 275 276 return (ret); 277 } 278 279 /* 280 * ibmf_saa_is_valid 281 * Returns true the entry is valid. 282 * 283 * Input Arguments 284 * saa_portp pointer to state structure 285 * add_ref if B_TRUE ref count is incremented on a valid portp 286 * 287 * Output Arguments 288 * none 289 * 290 * Returns 291 * B_TRUE if entry was in a valid state, B_FALSE otherwise 292 */ 293 boolean_t 294 ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref) 295 { 296 boolean_t is_valid = B_TRUE; 297 298 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_start, 299 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() enter\n"); 300 301 mutex_enter(&saa_portp->saa_pt_mutex); 302 303 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID || 304 saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_PURGING) { 305 306 is_valid = B_FALSE; 307 308 } else if (add_ref == B_TRUE) { 309 /* 310 * increment reference count here to ensure that 311 * entry does not get purged behind our backs 312 */ 313 saa_portp->saa_pt_reference_count++; 314 } 315 mutex_exit(&saa_portp->saa_pt_mutex); 316 317 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_end, 318 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() exit\n"); 319 320 return (is_valid); 321 } 322 323 /* 324 * ibmf_saa_must_purge 325 * Determines if we can purge a portp (remove it from the list) based on the 326 * state and number of clients 327 * 328 * Input Arguments 329 * saa_portp pointer to state structure 330 * 331 * Output Arguments 332 * none 333 * 334 * Returns 335 * B_TRUE if the entry can be removed, B_FALSE otherwise 336 */ 337 static int 338 ibmf_saa_must_purge(saa_port_t *saa_portp) 339 { 340 int must_purge = B_FALSE; 341 342 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_start, 343 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() enter\n"); 344 345 mutex_enter(&saa_portp->saa_pt_mutex); 346 347 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID && 348 saa_portp->saa_pt_reference_count == 0) { 349 350 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_PURGING; 351 must_purge = B_TRUE; 352 } 353 354 mutex_exit(&saa_portp->saa_pt_mutex); 355 356 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_end, 357 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() exit\n"); 358 359 return (must_purge); 360 } 361 362 363 /* 364 * ibmf_saa_impl_purge: 365 * Removes invalid port state entries from the list 366 * 367 * Input Arguments 368 * none 369 * 370 * Output Arguments 371 * none 372 * 373 * Returns 374 * void 375 */ 376 void 377 ibmf_saa_impl_purge() 378 { 379 saa_port_t *cur_portp = NULL; 380 saa_port_t *prev_portp = NULL; 381 saa_port_t *rem_portp = NULL; 382 383 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_start, 384 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() enter\n"); 385 386 mutex_enter(&saa_statep->saa_port_list_mutex); 387 388 cur_portp = saa_statep->saa_port_list; 389 prev_portp = cur_portp; 390 391 while (cur_portp != NULL) { 392 393 if (ibmf_saa_must_purge(cur_portp) == B_TRUE) { 394 395 rem_portp = cur_portp; 396 397 /* unlink entry */ 398 if (cur_portp == saa_statep->saa_port_list) { 399 400 saa_statep->saa_port_list = cur_portp->next; 401 cur_portp = saa_statep->saa_port_list; 402 prev_portp = cur_portp; 403 404 } else { 405 406 prev_portp->next = cur_portp->next; 407 cur_portp = cur_portp->next; 408 } 409 410 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rem_portp)) 411 412 /* destroy entry */ 413 ASSERT(rem_portp != NULL); 414 ibmf_saa_impl_destroy_port(rem_portp); 415 416 } else { 417 418 prev_portp = cur_portp; 419 cur_portp = cur_portp->next; 420 } 421 } 422 423 mutex_exit(&saa_statep->saa_port_list_mutex); 424 425 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_end, 426 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() exit\n"); 427 } 428 429 /* 430 * saa_impl_add_client: 431 * Adds a client for a particular portp. Reference count has been incremented 432 * before this call. It is decremented by saa_impl_add_client() if the call 433 * fails. 434 * 435 * Input Arguments 436 * none 437 * 438 * Output Arguments 439 * none 440 * 441 * Returns 442 * IBMF_BUSY if there are already too many clients registered, 443 * IBMF_BAD_PORT_STATE if the port is invalid (generally because a previous 444 * client failed during registration for this port) 445 * IBMF_SUCCESS otherwise 446 */ 447 int 448 ibmf_saa_impl_add_client(saa_port_t *saa_portp) 449 { 450 int status = IBMF_SUCCESS; 451 452 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_add_client_start, 453 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client() enter\n"); 454 455 mutex_enter(&saa_portp->saa_pt_mutex); 456 457 /* 458 * check that we don't exceed max clients 459 */ 460 if (saa_portp->saa_pt_reference_count > 461 SAA_MAX_CLIENTS_PER_PORT) { 462 463 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 464 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, "", 465 "ibmf_saa_impl_add_client: %s, num_reg_clients %d\n", 466 tnf_string, msg, "too many clients registered for" 467 " port", tnf_uint, num_reg_clients, 468 saa_portp->saa_pt_reference_count); 469 470 status = IBMF_BUSY; 471 goto bail; 472 } 473 474 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 475 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "", 476 "ibmf_saa_impl_add_client: num_registered_clients %d\n", 477 tnf_uint, num_registered_clients, 478 saa_portp->saa_pt_reference_count); 479 480 /* 481 * wait until anyone who is currently registering 482 * this port with ibmf is done 483 */ 484 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) { 485 486 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 487 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "", 488 "ibmf_saa_impl_add_client: %s\n", 489 tnf_string, msg, "someone is registering. waiting" 490 " for them to finish"); 491 492 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv, 493 &saa_portp->saa_pt_mutex); 494 495 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 496 ibmf_saa_impl_add_client, 497 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client: %s\n", 498 tnf_string, msg, "done waiting"); 499 } 500 501 /* 502 * if port isn't ready here, fail. 503 */ 504 if (saa_portp->saa_pt_state != IBMF_SAA_PORT_STATE_READY) { 505 506 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 507 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, 508 "", "ibmf_saa_impl_add_client: %s\n", 509 tnf_string, msg, "port state not ready," 510 " removing client."); 511 512 status = IBMF_BAD_PORT_STATE; 513 goto bail; 514 } 515 516 bail: 517 mutex_exit(&saa_portp->saa_pt_mutex); 518 519 if (status != IBMF_SUCCESS) { 520 521 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 522 523 IBMF_SAA_ADD32_KSTATS(saa_portp, 524 clients_reg_failed, 1); 525 526 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 527 528 /* decrementing refcount is last thing we do on entry */ 529 530 mutex_enter(&saa_portp->saa_pt_mutex); 531 532 ASSERT(saa_portp->saa_pt_reference_count > 0); 533 saa_portp->saa_pt_reference_count--; 534 535 mutex_exit(&saa_portp->saa_pt_mutex); 536 } 537 538 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 539 ibmf_saa_impl_add_client_end, IBMF_TNF_TRACE, "", 540 "ibmf_saa_impl_add_client() exit\n"); 541 542 return (status); 543 } 544 545 /* 546 * ibmf_saa_impl_create_port() 547 * Create port entry with mimimal inits because 548 * we're holding the list mutex: NO BLOCKING CALLS HERE, please. 549 * 550 * Initialize port state to "registering", so that clients accessing 551 * same port concurrently will wait for the end of the ibmf registration. 552 * Note: this thread will access port members without locking mutex. 553 * 554 * Input Arguments 555 * pt_guid guid of port 556 * 557 * Output Arguments 558 * saa_portpp pointer to new saa_portp structure 559 * 560 * Returns 561 * IBMF_NO_MEMORY if memory could not be allocated 562 * IBMF_SUCCESS otherwise 563 */ 564 int 565 ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp) 566 { 567 int status = IBMF_SUCCESS; 568 saa_port_t *saa_portp = NULL; 569 570 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_create_port_start, 571 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port:" 572 " guid %016" PRIx64 "\n", 573 tnf_opaque, port_guid, pt_guid); 574 575 ASSERT(MUTEX_HELD(&saa_statep->saa_port_list_mutex)); 576 577 /* create & initialize new port */ 578 saa_portp = kmem_zalloc(sizeof (saa_port_t), KM_NOSLEEP); 579 580 if (saa_portp == NULL) { 581 582 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 583 ibmf_sa_session_open_err, IBMF_TNF_ERROR, "", 584 "ibmf_saa_impl_create_port: %s\n", 585 tnf_string, msg, "could not allocate memory for " 586 "new port"); 587 588 status = IBMF_NO_MEMORY; 589 goto bail; 590 } 591 592 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_sa_session_open, 593 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port: %s\n", 594 tnf_string, msg, "first client registering, initializing"); 595 596 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp)) 597 598 /* tell everyone that kstats are not initialized */ 599 saa_portp->saa_pt_kstatp = NULL; 600 601 /* 602 * set up mutexe and state variable to indicate to 603 * other clients that were currently in the process of 604 * setting up the port data. This will prevent a subsequent 605 * client from trying to to register with ibmf before the 606 * port data has been initialized. 607 */ 608 mutex_init(&saa_portp->saa_pt_mutex, NULL, MUTEX_DRIVER, NULL); 609 cv_init(&saa_portp->saa_pt_ibmf_reg_cv, NULL, CV_DRIVER, NULL); 610 611 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_REGISTERING; 612 613 /* create other mutexes */ 614 mutex_init(&saa_portp->saa_pt_kstat_mutex, NULL, MUTEX_DRIVER, NULL); 615 616 mutex_init(&saa_portp->saa_pt_event_sub_mutex, NULL, MUTEX_DRIVER, 617 NULL); 618 619 /* 620 * clients assume all arrive; set mask to this so we only notify 621 * if something failed 622 */ 623 saa_portp->saa_pt_event_sub_last_success_mask = 624 IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE; 625 626 /* 627 * set port_guid now so any immediately subsequent clients 628 * registering on this port, guid will know we're already here 629 */ 630 saa_portp->saa_pt_port_guid = pt_guid; 631 saa_portp->saa_pt_reference_count = 1; 632 saa_portp->saa_pt_current_tid = pt_guid << 32; 633 634 saa_portp->saa_pt_redirect_active = B_FALSE; 635 636 /* set sa_uptime now in case we never receive anything from SA */ 637 saa_portp->saa_pt_sa_uptime = gethrtime(); 638 639 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_portp)) 640 641 /* Set new pointer in caller's */ 642 *saa_portpp = saa_portp; 643 644 bail: 645 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_create_port_end, 646 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port() exit\n"); 647 648 return (status); 649 } 650 651 /* 652 * ibmf_saa_impl_invalidate_port: 653 * invalidates port entry (assumes exist) and deletes kstat object 654 * kstat object is destroyed in order to allow creating port entry 655 * even if this entry is not purged 656 */ 657 static void 658 ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp) 659 { 660 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 661 ibmf_saa_impl_invalidate_port_start, 662 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() enter\n"); 663 664 ASSERT(saa_portp != NULL); 665 ASSERT(MUTEX_HELD(&saa_portp->saa_pt_mutex)); 666 667 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_INVALID; 668 ibmf_saa_impl_uninit_kstats(saa_portp); 669 670 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 671 ibmf_saa_impl_invalidate_port_end, 672 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() exit\n"); 673 } 674 675 /* 676 * ibmf_saa_impl_destroy_port: 677 * Frees the resources associated with an saa_portp structure. Assumes the 678 * saa_portp exists 679 * 680 * Input Arguments 681 * saa_portp pointer to saa_portp structure 682 * 683 * Output Arguments 684 * none 685 * 686 * Returns 687 * void 688 */ 689 static void 690 ibmf_saa_impl_destroy_port(saa_port_t *saa_portp) 691 { 692 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_start, 693 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() enter\n"); 694 695 ASSERT(saa_portp != NULL); 696 697 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 698 699 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 700 ibmf_saa_impl_destroy, IBMF_TNF_TRACE, "", 701 "ibmf_saa_impl_destroy(): destroying port_guid %016" PRIx64 "\n", 702 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 703 704 ibmf_saa_impl_uninit_kstats(saa_portp); 705 706 /* uninit synchronization variables used for registration */ 707 mutex_destroy(&saa_portp->saa_pt_mutex); 708 cv_destroy(&saa_portp->saa_pt_ibmf_reg_cv); 709 710 mutex_destroy(&saa_portp->saa_pt_event_sub_mutex); 711 mutex_destroy(&saa_portp->saa_pt_kstat_mutex); 712 713 kmem_free(saa_portp, sizeof (saa_port_t)); 714 715 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_end, 716 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() exit\n"); 717 } 718 719 /* 720 * ibmf_saa_impl_init_kstats: 721 * Create kstats structure. Should be called when memory is alloced for a new 722 * port entry. 723 */ 724 int 725 ibmf_saa_impl_init_kstats(saa_port_t *saa_portp) 726 { 727 char buf[128]; 728 ibmf_saa_kstat_t *ksp; 729 730 _NOTE(ASSUMING_PROTECTED(saa_portp->saa_pt_kstatp)) 731 732 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 733 ibmf_saa_impl_init_kstats_start, 734 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() enter\n"); 735 736 /* set up kstats structure */ 737 (void) sprintf(buf, "ibmf_saa_%016" PRIx64 "_stat", 738 saa_portp->saa_pt_port_guid); 739 740 saa_portp->saa_pt_kstatp = kstat_create("ibmf_saa", 741 0, buf, "misc", KSTAT_TYPE_NAMED, 742 sizeof (ibmf_saa_kstat_t) / sizeof (kstat_named_t), 743 KSTAT_FLAG_WRITABLE); 744 745 if (saa_portp->saa_pt_kstatp == NULL) 746 return (IBMF_NO_RESOURCES); 747 748 ksp = (ibmf_saa_kstat_t *)saa_portp->saa_pt_kstatp->ks_data; 749 750 kstat_named_init(&ksp->clients_registered, 751 "clients_registered", KSTAT_DATA_UINT32); 752 753 kstat_named_init(&ksp->clients_reg_failed, 754 "clients_reg_failed", KSTAT_DATA_UINT32); 755 756 kstat_named_init(&ksp->outstanding_requests, 757 "outstanding_requests", KSTAT_DATA_UINT32); 758 759 kstat_named_init(&ksp->total_requests, 760 "total_requests", KSTAT_DATA_UINT32); 761 762 kstat_named_init(&ksp->failed_requests, 763 "failed_requests", KSTAT_DATA_UINT32); 764 765 kstat_named_init(&ksp->requests_timedout, 766 "requests_timedout", KSTAT_DATA_UINT32); 767 768 kstat_install(saa_portp->saa_pt_kstatp); 769 770 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 771 ibmf_saa_impl_init_kstats_end, 772 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() exit\n"); 773 774 return (IBMF_SUCCESS); 775 } 776 777 /* 778 * ibmf_saa_impl_uninit_kstats: 779 * Free kstats context. Should be called when port is either destroyed 780 * or invalidated. 781 */ 782 static void 783 ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp) 784 { 785 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 786 ibmf_saa_impl_uninit_kstats_start, 787 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() enter\n"); 788 789 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 790 791 if (saa_portp->saa_pt_kstatp != NULL) { 792 kstat_delete(saa_portp->saa_pt_kstatp); 793 } 794 saa_portp->saa_pt_kstatp = NULL; 795 796 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 797 798 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 799 ibmf_saa_impl_uninit_kstats_end, 800 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() exit\n"); 801 } 802 803 /* 804 * ibmf_saa_impl_register_failed: 805 * invalidate entry and kick waiters 806 */ 807 void 808 ibmf_saa_impl_register_failed(saa_port_t *saa_portp) 809 { 810 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 811 ibmf_saa_impl_register_failed_start, 812 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() enter\n"); 813 814 mutex_enter(&saa_portp->saa_pt_mutex); 815 816 ibmf_saa_impl_invalidate_port(saa_portp); 817 818 cv_broadcast(&saa_portp->saa_pt_ibmf_reg_cv); 819 820 /* decrementing refcount is last thing we do on entry */ 821 822 ASSERT(saa_portp->saa_pt_reference_count > 0); 823 saa_portp->saa_pt_reference_count--; 824 825 mutex_exit(&saa_portp->saa_pt_mutex); 826 827 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 828 ibmf_saa_impl_register_failed_end, 829 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() exit\n"); 830 } 831 832 static int 833 ibmf_saa_impl_setup_qp_async_cb(saa_port_t *saa_portp, int setup_async_cb_only) 834 { 835 int status; 836 int unreg_status; 837 ib_pkey_t p_key; 838 ib_qkey_t q_key; 839 uint8_t portnum; 840 boolean_t qp_alloced = B_FALSE; 841 842 if (setup_async_cb_only == 0) { 843 844 /* allocate a qp through ibmf */ 845 status = ibmf_alloc_qp(saa_portp->saa_pt_ibmf_handle, 846 IB_PKEY_DEFAULT_LIMITED, IB_GSI_QKEY, 847 IBMF_ALT_QP_MAD_RMPP, &saa_portp->saa_pt_qp_handle); 848 849 if (status != IBMF_SUCCESS) { 850 851 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 852 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 853 "ibmf_saa_impl_setup_qp_async_cb: %s, " 854 "ibmf_status = %d\n", 855 tnf_string, msg, "Cannot alloc qp with ibmf", 856 tnf_int, status, status); 857 858 return (status); 859 } 860 861 qp_alloced = B_TRUE; 862 863 /* 864 * query the queue pair number; we will need it to unsubscribe 865 * from notice reports 866 */ 867 status = ibmf_query_qp(saa_portp->saa_pt_ibmf_handle, 868 saa_portp->saa_pt_qp_handle, &saa_portp->saa_pt_qpn, 869 &p_key, &q_key, &portnum, 0); 870 871 if (status != IBMF_SUCCESS) { 872 873 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 874 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 875 "ibmf_saa_impl_setup_qp_async_cb: %s, " 876 "ibmf_status = %d\n", 877 tnf_string, msg, 878 "Cannot query alt qp to get qp num", 879 tnf_int, status, status); 880 881 goto bail; 882 } 883 } 884 885 /* 886 * core ibmf is taking advantage of the fact that saa_portp is our 887 * callback arg. If this changes, the code in ibmf_recv would need to 888 * change as well 889 */ 890 status = ibmf_setup_async_cb(saa_portp->saa_pt_ibmf_handle, 891 saa_portp->saa_pt_qp_handle, ibmf_saa_report_cb, saa_portp, 0); 892 if (status != IBMF_SUCCESS) { 893 894 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 895 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 896 "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status = %d\n", 897 tnf_string, msg, "Cannot register async cb with ibmf", 898 tnf_int, status, status); 899 900 goto bail; 901 } 902 903 return (IBMF_SUCCESS); 904 905 bail: 906 if (qp_alloced == B_TRUE) { 907 /* free alternate qp */ 908 unreg_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle, 909 &saa_portp->saa_pt_qp_handle, 0); 910 if (unreg_status != IBMF_SUCCESS) { 911 912 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 913 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 914 "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status =" 915 " %d\n", tnf_string, msg, 916 "Cannot free alternate queue pair with ibmf", 917 tnf_int, unreg_status, unreg_status); 918 } 919 } 920 921 return (status); 922 } 923 924 /* 925 * ibmf_saa_impl_register_port: 926 */ 927 int 928 ibmf_saa_impl_register_port( 929 saa_port_t *saa_portp) 930 { 931 uint_t hca_count = 0; 932 ib_guid_t *hca_list = NULL; 933 int status = IBMF_SUCCESS; 934 int unreg_status = IBMF_SUCCESS; 935 int ibt_status = IBT_SUCCESS; 936 ibt_hca_portinfo_t *port_info_list = NULL; 937 uint_t port_count = 0; 938 uint_t port_size = 0; 939 int ihca, iport; 940 ib_guid_t port_guid; 941 boolean_t ibmf_reg = B_FALSE; 942 943 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 944 ibmf_saa_impl_register_port_start, IBMF_TNF_TRACE, "", 945 "ibmf_saa_impl_register_port() enter\n"); 946 947 ASSERT(saa_portp != NULL); 948 949 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 950 951 /* get the HCA list */ 952 953 hca_count = ibt_get_hca_list(&hca_list); 954 955 if (hca_count == 0) { 956 957 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 958 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 959 "ibmf_saa_impl_register_port: %s\n", 960 tnf_string, msg, "cannot register port (no HCAs).\n"); 961 962 status = IBMF_BAD_PORT; 963 goto bail; 964 } 965 966 /* lookup requested port guid in hca list */ 967 for (ihca = 0; ihca != hca_count; ihca++) { 968 969 ibt_status = ibt_query_hca_ports_byguid(hca_list[ihca], 970 0 /* all ports */, &port_info_list, 971 &port_count, &port_size); 972 973 if (ibt_status != IBT_SUCCESS) { 974 975 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 976 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 977 "ibmf_saa_impl_register_port: %s, %016" PRIx64 "\n", 978 tnf_string, msg, "Could not query hca. Exiting.", 979 tnf_opaque, guid, hca_list[ihca]); 980 981 status = IBMF_TRANSPORT_FAILURE; 982 break; 983 } 984 985 for (iport = 0; iport < port_count; iport++) { 986 987 /* get port guid associated with hca guid, port num */ 988 if (ibmf_saa_impl_get_port_guid( 989 port_info_list + iport, &port_guid) != IBMF_SUCCESS) 990 continue; 991 992 if (saa_portp->saa_pt_port_guid != port_guid) 993 continue; 994 995 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 996 ibmf_saa_impl_register_port, 997 IBMF_TNF_TRACE, "", 998 "ibmf_saa_impl_register_port: %s, hca_guid = %016" 999 PRIx64 ", port_guid = %016" PRIx64 1000 ", number = %d\n", 1001 tnf_string, msg, "found port", 1002 tnf_opaque, hca_guid, hca_list[ihca], 1003 tnf_opaque, port_guid, port_guid, 1004 tnf_uint, port, iport + 1); 1005 1006 /* 1007 * we're here? then we found our port: 1008 * fill in ibmf registration info 1009 * and address parameters from the portinfo 1010 */ 1011 1012 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid 1013 = hca_list[ihca]; 1014 saa_portp->saa_pt_ibmf_reginfo.ir_port_num = iport+1; 1015 saa_portp->saa_pt_ibmf_reginfo.ir_client_class 1016 = SUBN_ADM_MANAGER; 1017 1018 saa_portp->saa_pt_node_guid = hca_list[ihca]; 1019 saa_portp->saa_pt_port_num = iport + 1; 1020 1021 ibmf_saa_impl_set_transaction_params( 1022 saa_portp, port_info_list + iport); 1023 break; 1024 } 1025 1026 ibt_free_portinfo(port_info_list, port_size); 1027 1028 if (iport != port_count) 1029 break; /* found our port */ 1030 } 1031 1032 ibt_free_hca_list(hca_list, hca_count); 1033 1034 if (ihca == hca_count) { 1035 1036 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1037 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1038 "ibmf_saa_impl_register_port: %s, port_guid %016" 1039 PRIx64 "\n", 1040 tnf_string, msg, "Could not find port, exiting", 1041 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 1042 1043 status = IBMF_BAD_PORT; 1044 } 1045 1046 if (status != IBMF_SUCCESS) { 1047 1048 goto bail; 1049 } 1050 1051 /* 1052 * Now we found the port we searched for, 1053 * and open an ibmf session on that port. 1054 */ 1055 1056 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1057 ibmf_saa_impl_register_port, IBMF_TNF_TRACE, "", 1058 "ibmf_saa_impl_register_port: %s, port_guid = %016" PRIx64 1059 ", port = %d\n", tnf_string, msg, "Registering with ibmf", 1060 tnf_opaque, port_guid, saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid, 1061 tnf_uint, port, saa_portp->saa_pt_ibmf_reginfo.ir_port_num); 1062 1063 status = ibmf_register(&saa_portp->saa_pt_ibmf_reginfo, 1064 IBMF_VERSION, IBMF_REG_FLAG_RMPP, 1065 ibmf_saa_impl_async_event_cb, saa_portp, 1066 &saa_portp->saa_pt_ibmf_handle, 1067 &saa_portp->saa_pt_ibmf_impl_features); 1068 1069 if (status != IBMF_SUCCESS) { 1070 1071 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1072 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1073 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n", 1074 tnf_string, msg, "Could not register with ibmf", 1075 tnf_int, status, status); 1076 1077 goto bail; 1078 } 1079 1080 ibmf_reg = B_TRUE; 1081 1082 if (ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0) == IBMF_SUCCESS) 1083 return (IBMF_SUCCESS); 1084 1085 bail: 1086 if (ibmf_reg == B_TRUE) { 1087 /* unregister from ibmf */ 1088 unreg_status = ibmf_unregister( 1089 &saa_portp->saa_pt_ibmf_handle, 0); 1090 1091 if (unreg_status != IBMF_SUCCESS) { 1092 1093 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1094 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1095 "ibmf_saa_impl_register_port: %s, ibmf_status =" 1096 " %d\n", tnf_string, msg, 1097 "Cannot unregister from ibmf", 1098 tnf_int, unreg_status, unreg_status); 1099 } 1100 } 1101 1102 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_register_port_end, 1103 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_port() exit\n"); 1104 1105 return (status); 1106 } 1107 1108 /* 1109 * ibmf_saa_impl_getclassportinfo: 1110 */ 1111 void 1112 ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp) 1113 { 1114 int res; 1115 saa_impl_trans_info_t *trans_info; 1116 1117 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1118 ibmf_saa_impl_get_classportinfo_start, 1119 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() enter\n"); 1120 1121 /* 1122 * allocate memory for trans_info; send_request's callback will free up 1123 * memory since request is asynchronous 1124 */ 1125 trans_info = kmem_zalloc(sizeof (saa_impl_trans_info_t), KM_NOSLEEP); 1126 if (trans_info == NULL) { 1127 1128 mutex_enter(&saa_portp->saa_pt_mutex); 1129 1130 /* cpi transaction is handled as a client, decrement refcount */ 1131 ASSERT(saa_portp->saa_pt_reference_count > 0); 1132 saa_portp->saa_pt_reference_count--; 1133 1134 mutex_exit(&saa_portp->saa_pt_mutex); 1135 1136 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1137 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_ERROR, "", 1138 "ibmf_saa_impl_get_classportinfo: %s\n", tnf_string, msg, 1139 "Could not allocate memory for classportinfo trans_info"); 1140 1141 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1142 ibmf_saa_impl_get_classportinfo_end, IBMF_TNF_TRACE, "", 1143 "ibmf_saa_impl_get_classportinfo() exiting\n"); 1144 1145 return; 1146 } 1147 1148 /* no specific client associated with this transaction */ 1149 trans_info->si_trans_client_data = NULL; 1150 trans_info->si_trans_port = saa_portp; 1151 trans_info->si_trans_method = SA_SUBN_ADM_GET; 1152 trans_info->si_trans_attr_id = MAD_ATTR_ID_CLASSPORTINFO; 1153 1154 trans_info->si_trans_callback = ibmf_saa_impl_get_cpi_cb; 1155 trans_info->si_trans_callback_arg = saa_portp; 1156 1157 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 1158 1159 IBMF_SAA_ADD32_KSTATS(saa_portp, outstanding_requests, 1); 1160 IBMF_SAA_ADD32_KSTATS(saa_portp, total_requests, 1); 1161 1162 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 1163 1164 res = ibmf_saa_impl_send_request(trans_info); 1165 1166 if (res != IBMF_SUCCESS) { 1167 1168 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1169 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_TRACE, "", 1170 "ibmf_saa_impl_get_classportinfo: %s, res = 0x%x\n", 1171 tnf_string, msg, "ibmf_saa_impl_send_request failed", 1172 tnf_opaque, res, res); 1173 1174 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 1175 1176 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1); 1177 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1); 1178 1179 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 1180 1181 mutex_enter(&saa_portp->saa_pt_mutex); 1182 1183 /* cpi transaction is handled as a client, decrement refcount */ 1184 ASSERT(saa_portp->saa_pt_reference_count > 0); 1185 saa_portp->saa_pt_reference_count--; 1186 1187 mutex_exit(&saa_portp->saa_pt_mutex); 1188 1189 } 1190 1191 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1192 ibmf_saa_impl_get_classportinfo_end, 1193 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() exit\n"); 1194 } 1195 1196 /* 1197 * ibmf_saa_impl_get_cpi_cb: 1198 * 1199 * Called when the asynchronous getportinfo request receives its response. 1200 * Checks the status. If success, updates the times in the port's 1201 * ibmf_retrans structure that is used in ibmf_msg_transport calls. If failure, 1202 * just use default values. 1203 * 1204 * Input Arguments 1205 * arg user-specified pointer (points to the current port data) 1206 * length length of payload returned (should be size of classportinfo_t) 1207 * buffer pointer to classportinfo returned (should not be null) 1208 * status status of sa access request 1209 * 1210 * Output Arguments 1211 * none 1212 * 1213 * Returns void 1214 */ 1215 static void 1216 ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, int status) 1217 { 1218 saa_port_t *saa_portp; 1219 uint64_t base_time, resp_timeout, rttv_timeout; 1220 ib_mad_classportinfo_t *classportinfo; 1221 int resp_time_value; 1222 uint16_t sa_cap_mask; 1223 1224 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_start, 1225 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() enter\n"); 1226 1227 /* 1228 * access port entry: note that it may have become invalid 1229 * but we hold a ref count for cpi and the interactions on 1230 * the entry are harmless 1231 */ 1232 saa_portp = (saa_port_t *)arg; 1233 1234 /* process response */ 1235 1236 if ((status != IBMF_SUCCESS) || (buffer == NULL)) { 1237 1238 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1, 1239 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_ERROR, "", 1240 "ibmf_saa_impl_get_cpi_cb: %s, status = %d, buffer = " 1241 " 0x%p, length = %d\n", tnf_string, msg, 1242 "could not get classportinfo. Check node and path to sm" 1243 " lid", tnf_int, status, status, 1244 tnf_opaque, buffer, buffer, tnf_uint, length, length); 1245 1246 /* 1247 * IB spec (C13-13) indicates 20 can be used as default or 1248 * intial value for classportinfo->resptimeout value 1249 */ 1250 resp_time_value = 20; 1251 1252 sa_cap_mask = 0xFFFF; 1253 1254 } else if (buffer != NULL) { 1255 1256 classportinfo = (ib_mad_classportinfo_t *)buffer; 1257 1258 resp_time_value = classportinfo->RespTimeValue & 0x1f; 1259 1260 sa_cap_mask = classportinfo->CapabilityMask; 1261 1262 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1263 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "", 1264 "ibmf_saa_impl_get_cpi_cb: %s, timeout = 0x%x," 1265 " cap_mask = 0x%x\n", 1266 tnf_string, msg, "got classportinfo", 1267 tnf_opaque, timeout, resp_time_value, 1268 tnf_opaque, cap_mask, sa_cap_mask); 1269 1270 kmem_free(buffer, length); 1271 } 1272 1273 /* 1274 * using IB spec calculation from 13.4.6.2 1275 * use bit shifting for 2^x. 1276 */ 1277 base_time = (1 << resp_time_value); 1278 1279 resp_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000; 1280 1281 mutex_enter(&saa_portp->saa_pt_mutex); 1282 1283 base_time = 2 * (1 << saa_portp->saa_pt_timeout); 1284 1285 rttv_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000; 1286 1287 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = resp_timeout; 1288 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = rttv_timeout; 1289 saa_portp->saa_pt_sa_cap_mask = sa_cap_mask; 1290 1291 /* 1292 * cpi transaction is handled as a client, 1293 * decrement refcount; make sure it's the last 1294 * thing we do on this entry 1295 */ 1296 ASSERT(saa_portp->saa_pt_reference_count > 0); 1297 saa_portp->saa_pt_reference_count--; 1298 1299 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1300 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "", 1301 "ibmf_saa_impl_get_cpi_cb: %s, subnet_timeout = 0x%x, " 1302 "resp_time_value = 0x%x\n", 1303 tnf_string, msg, "updated resp timeout", 1304 tnf_opaque, subnet_timeout, saa_portp->saa_pt_timeout, 1305 tnf_opaque, resp_time_value, resp_time_value); 1306 1307 mutex_exit(&saa_portp->saa_pt_mutex); 1308 1309 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_end, 1310 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() exit\n"); 1311 } 1312 1313 /* 1314 * ibmf_saa_impl_send_request: 1315 * Sends a request to the sa. Can be used for both classportinfo and record 1316 * requests. Will set up all data structures for using the multi-packet 1317 * protocol, create the mad, and send it. Returns SA_SUCCESS if msg transport 1318 * worked, meaning succesful send for the async case and a succesful send and 1319 * recv for the sync case. 1320 */ 1321 int 1322 ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info) 1323 { 1324 uint16_t attr_id; 1325 saa_client_data_t *client_data; 1326 saa_port_t *saa_portp; 1327 uint32_t transport_flags; 1328 ibmf_msg_cb_t ibmf_callback; 1329 void *ibmf_callback_arg; 1330 ibmf_msg_t *msgp; 1331 ibmf_retrans_t ibmf_retrans; 1332 uint16_t sa_cap_mask; 1333 boolean_t sleep_flag; 1334 int ibmf_status = IBMF_SUCCESS; 1335 int retry_count; 1336 uint16_t mad_status; 1337 boolean_t sa_is_redirected = B_FALSE; 1338 1339 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1340 ibmf_saa_impl_send_request_start, 1341 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() enter\n"); 1342 1343 attr_id = trans_info->si_trans_attr_id; 1344 client_data = trans_info->si_trans_client_data; 1345 saa_portp = trans_info->si_trans_port; 1346 1347 /* 1348 * don't send on invalid entry 1349 * Note that there is a window where it could become 1350 * invalid after this test is done, but we'd rely on ibmf errors... 1351 */ 1352 if (ibmf_saa_is_valid(saa_portp, B_FALSE) == B_FALSE) { 1353 1354 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1, 1355 ibmf_saa_impl_send_request, 1356 IBMF_TNF_ERROR, "", 1357 "ibmf_saa_impl_send_request: %s, hca_guid = %016" 1358 PRIx64 ", port_guid = %016" PRIx64 1359 ", number = %d\n", 1360 tnf_string, msg, "sending on invalid port", 1361 tnf_opaque, hca_guid, 1362 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid, 1363 tnf_opaque, port_guid, 1364 saa_portp->saa_pt_port_guid, 1365 tnf_uint, port, 1366 saa_portp->saa_pt_ibmf_reginfo.ir_port_num); 1367 1368 ibmf_status = IBMF_REQ_INVALID; 1369 goto bail; 1370 } 1371 1372 /* check whether SA supports this attribute */ 1373 mutex_enter(&saa_portp->saa_pt_mutex); 1374 1375 sa_cap_mask = saa_portp->saa_pt_sa_cap_mask; 1376 sa_is_redirected = saa_portp->saa_pt_redirect_active; 1377 1378 mutex_exit(&saa_portp->saa_pt_mutex); 1379 1380 ibmf_status = ibmf_saa_impl_check_sa_support(sa_cap_mask, attr_id); 1381 1382 if (ibmf_status != IBMF_SUCCESS) { 1383 1384 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1385 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "", 1386 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1387 tnf_string, msg, "SA does not support attribute", 1388 tnf_int, ibmf_status, ibmf_status); 1389 1390 goto bail; 1391 } 1392 1393 /* make only non-blocking calls if this is an async request */ 1394 if ((trans_info->si_trans_callback == NULL) && 1395 (trans_info->si_trans_sub_callback == NULL)) { 1396 ibmf_callback = NULL; 1397 ibmf_callback_arg = NULL; 1398 sleep_flag = B_TRUE; 1399 } else { 1400 ibmf_callback = ibmf_saa_async_cb; 1401 ibmf_callback_arg = (void *)trans_info; 1402 sleep_flag = B_FALSE; 1403 } 1404 1405 ibmf_status = ibmf_saa_impl_init_msg(trans_info, sleep_flag, &msgp, 1406 &transport_flags, &ibmf_retrans); 1407 if (ibmf_status != IBMF_SUCCESS) { 1408 1409 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1410 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "", 1411 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1412 tnf_string, msg, "init_msg() failed", 1413 tnf_int, ibmf_status, ibmf_status); 1414 1415 goto bail; 1416 } 1417 1418 mutex_enter(&saa_portp->saa_pt_mutex); 1419 1420 saa_portp->saa_pt_num_outstanding_trans++; 1421 1422 mutex_exit(&saa_portp->saa_pt_mutex); 1423 1424 /* 1425 * increment the number of outstanding transaction so 1426 * ibmf_close_sa_session() will wait. classportinfo requests 1427 * don't have associated clients so check for valid clientp 1428 */ 1429 if (client_data != NULL) { 1430 1431 mutex_enter(&client_data->saa_client_mutex); 1432 1433 client_data->saa_client_num_pending_trans++; 1434 1435 mutex_exit(&client_data->saa_client_mutex); 1436 } 1437 1438 /* 1439 * make the call to msg_transport. If synchronous and success, 1440 * check that the response mad isn't status busy. If so, repeat the 1441 * call 1442 */ 1443 retry_count = 0; 1444 1445 /* 1446 * set the send time here. We only set this once at the beginning of 1447 * the transaction. Retrying because of busys or mastersmlid changes 1448 * does not change the original send time. It is meant to be an 1449 * absolute time out value and will only be used if there are other 1450 * problems (i.e. a buggy SA) 1451 */ 1452 trans_info->si_trans_send_time = gethrtime(); 1453 1454 for (;;) { 1455 1456 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 1457 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 1458 ibmf_callback, ibmf_callback_arg, transport_flags); 1459 1460 if (ibmf_callback != NULL) 1461 break; 1462 1463 /* 1464 * stop here for non-sequenced transactions since they wouldn't 1465 * receive a timeout or busy response 1466 */ 1467 if (!(transport_flags & IBMF_MSG_TRANS_FLAG_SEQ)) 1468 break; 1469 1470 /* 1471 * if the transaction timed out and this was a synchronous 1472 * request there's a possiblity we were talking to the wrong 1473 * master smlid or that the SA has stopped responding on the 1474 * redirected desination (if redirect is active). 1475 * Check this and retry if necessary. 1476 */ 1477 if ((ibmf_status == IBMF_TRANS_TIMEOUT) && 1478 (sleep_flag == B_TRUE)) { 1479 if (sa_is_redirected == B_TRUE) { 1480 ibmf_status = ibmf_saa_impl_revert_to_qp1( 1481 saa_portp, msgp, ibmf_callback, 1482 ibmf_callback_arg, transport_flags); 1483 } else { 1484 ibmf_status = ibmf_saa_impl_new_smlid_retry( 1485 saa_portp, msgp, ibmf_callback, 1486 ibmf_callback_arg, transport_flags); 1487 } 1488 } 1489 1490 /* 1491 * if the transaction timed out (and retrying with a new SM LID 1492 * didn't help) check how long it's been since we received an SA 1493 * packet. If it hasn't been max_wait_time then retry the 1494 * request. 1495 */ 1496 if ((ibmf_status == IBMF_TRANS_TIMEOUT) && 1497 (sleep_flag == B_TRUE)) { 1498 1499 ibmf_status = ibmf_saa_check_sa_and_retry( 1500 saa_portp, msgp, ibmf_callback, ibmf_callback_arg, 1501 trans_info->si_trans_send_time, transport_flags); 1502 } 1503 1504 if (ibmf_status != IBMF_SUCCESS) 1505 break; 1506 1507 if (retry_count >= IBMF_SAA_MAX_BUSY_RETRY_COUNT) 1508 break; 1509 1510 /* sync transaction with status SUCCESS should have response */ 1511 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL); 1512 1513 mad_status = b2h16(msgp->im_msgbufs_recv. 1514 im_bufs_mad_hdr->Status); 1515 1516 if ((mad_status != MAD_STATUS_BUSY) && 1517 (mad_status != MAD_STATUS_REDIRECT_REQUIRED)) 1518 break; 1519 1520 if (mad_status == MAD_STATUS_REDIRECT_REQUIRED) { 1521 1522 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1523 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1524 "ibmf_saa_impl_send_request: %s, retry_count %d\n", 1525 tnf_string, msg, 1526 "response returned redirect status", 1527 tnf_int, retry_count, retry_count); 1528 1529 /* update address info and copy it into msgp */ 1530 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp); 1531 } else { 1532 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1533 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1534 "ibmf_saa_impl_send_request: %s, retry_count %d\n", 1535 tnf_string, msg, "response returned busy status", 1536 tnf_int, retry_count, retry_count); 1537 } 1538 1539 retry_count++; 1540 1541 /* 1542 * since this is a blocking call, sleep for some time 1543 * to allow SA to transition from busy state (if busy) 1544 */ 1545 if (mad_status == MAD_STATUS_BUSY) 1546 delay(drv_usectohz( 1547 IBMF_SAA_BUSY_RETRY_SLEEP_SECS * 1000000)); 1548 } 1549 1550 if (ibmf_status != IBMF_SUCCESS) { 1551 1552 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1553 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1554 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1555 tnf_string, msg, "ibmf_msg_transport() failed", 1556 tnf_int, ibmf_status, ibmf_status); 1557 1558 ibmf_saa_impl_free_msg(saa_portp->saa_pt_ibmf_handle, msgp); 1559 1560 mutex_enter(&saa_portp->saa_pt_mutex); 1561 1562 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 1563 saa_portp->saa_pt_num_outstanding_trans--; 1564 1565 mutex_exit(&saa_portp->saa_pt_mutex); 1566 1567 if (client_data != NULL) { 1568 1569 mutex_enter(&client_data->saa_client_mutex); 1570 1571 ASSERT(client_data->saa_client_num_pending_trans > 0); 1572 client_data->saa_client_num_pending_trans--; 1573 1574 if ((client_data->saa_client_num_pending_trans == 0) && 1575 (client_data->saa_client_state == 1576 SAA_CLIENT_STATE_WAITING)) 1577 cv_signal(&client_data->saa_client_state_cv); 1578 1579 mutex_exit(&client_data->saa_client_mutex); 1580 } 1581 1582 } else if (sleep_flag == B_TRUE) { 1583 1584 mutex_enter(&saa_portp->saa_pt_mutex); 1585 1586 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 1587 saa_portp->saa_pt_num_outstanding_trans--; 1588 1589 mutex_exit(&saa_portp->saa_pt_mutex); 1590 1591 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1592 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1593 "ibmf_saa_impl_send_request: %s\n", 1594 tnf_string, msg, "Message sent and received successfully"); 1595 1596 /* fill in response values and free the message */ 1597 ibmf_saa_impl_prepare_response(saa_portp->saa_pt_ibmf_handle, 1598 msgp, B_FALSE, &trans_info->si_trans_status, 1599 &trans_info->si_trans_result, 1600 &trans_info->si_trans_length, sleep_flag); 1601 1602 if (client_data != NULL) { 1603 mutex_enter(&client_data->saa_client_mutex); 1604 1605 ASSERT(client_data->saa_client_num_pending_trans > 0); 1606 client_data->saa_client_num_pending_trans--; 1607 1608 if ((client_data->saa_client_num_pending_trans == 0) && 1609 (client_data->saa_client_state == 1610 SAA_CLIENT_STATE_WAITING)) 1611 cv_signal(&client_data->saa_client_state_cv); 1612 1613 mutex_exit(&client_data->saa_client_mutex); 1614 } 1615 } else { 1616 1617 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1618 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1619 "ibmf_saa_impl_send_request: %s\n", 1620 tnf_string, msg, "Message sent successfully"); 1621 } 1622 1623 bail: 1624 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1625 ibmf_saa_impl_send_request_end, 1626 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() exiting" 1627 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 1628 1629 return (ibmf_status); 1630 } 1631 1632 /* 1633 * ibmf_saa_impl_init_msg: 1634 * Allocates an ibmf message and fills out the header fields and formatted data 1635 * fields. Also sets up the correct transport_flags and retrans argument for 1636 * the message transport call based on the request information. 1637 * 1638 * Input Arguments 1639 * trans_info saa_trans_info structure passed to send_request 1640 * sleep_flag B_TRUE if init_msg can sleep in function calls 1641 * 1642 * Output Arguments 1643 * msgp ibmf message that should be given to msg_transport 1644 * transport_flagsp transport flags that should be given to msg_transport 1645 * ibmf_retrans_t retrans parameter that should be given to msg_transport 1646 * 1647 * Returns 1648 * ibmf_status 1649 */ 1650 static int 1651 ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, boolean_t sleep_flag, 1652 ibmf_msg_t **msgp, uint32_t *transport_flagsp, 1653 ibmf_retrans_t *ibmf_retransp) 1654 { 1655 int ibmf_status; 1656 ibmf_msg_bufs_t *req_mad; 1657 ib_mad_hdr_t *mad_hdr; 1658 int ibmf_sleep_flag, km_sleep_flag; 1659 int free_res; 1660 ib_sa_hdr_t sa_hdr; 1661 ibmf_msg_t *ibmf_msg; 1662 uint16_t attr_id, pack_attr_id; 1663 uint8_t method; 1664 saa_client_data_t *client_data; 1665 saa_port_t *saa_portp; 1666 sa_multipath_record_t *multipath_template; 1667 size_t payload_length; 1668 uint32_t transport_flags; 1669 1670 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1671 ibmf_saa_impl_init_msg_start, 1672 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() entering\n"); 1673 1674 attr_id = trans_info->si_trans_attr_id; 1675 method = trans_info->si_trans_method; 1676 client_data = trans_info->si_trans_client_data; 1677 saa_portp = trans_info->si_trans_port; 1678 1679 if (sleep_flag == B_TRUE) { 1680 ibmf_sleep_flag = IBMF_ALLOC_SLEEP; 1681 km_sleep_flag = KM_SLEEP; 1682 } else { 1683 ibmf_sleep_flag = IBMF_ALLOC_NOSLEEP; 1684 km_sleep_flag = KM_NOSLEEP; 1685 } 1686 1687 ibmf_status = ibmf_alloc_msg(saa_portp->saa_pt_ibmf_handle, 1688 ibmf_sleep_flag, &ibmf_msg); 1689 if (ibmf_status != IBMF_SUCCESS) { 1690 1691 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1692 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1693 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n", 1694 tnf_string, msg, "Cannot allocate msg_buf.", 1695 tnf_int, ibmf_status, ibmf_status); 1696 1697 goto bail; 1698 } 1699 1700 req_mad = &ibmf_msg->im_msgbufs_send; 1701 1702 /* create a template (SA MAD) */ 1703 mad_hdr = kmem_zalloc(sizeof (ib_mad_hdr_t), km_sleep_flag); 1704 1705 if (mad_hdr == NULL) { 1706 1707 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1708 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1709 "ibmf_saa_impl_init_msg: %s\n", 1710 tnf_string, msg, "Cannot allocate mad header."); 1711 1712 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1713 &ibmf_msg); 1714 ASSERT(free_res == IBMF_SUCCESS); 1715 1716 ibmf_status = IBMF_NO_MEMORY; 1717 goto bail; 1718 } 1719 1720 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mad_hdr, 1721 *req_mad)) 1722 1723 bzero(mad_hdr, sizeof (ib_mad_hdr_t)); 1724 mad_hdr->BaseVersion = SAA_MAD_BASE_VERSION; 1725 mad_hdr->MgmtClass = MAD_MGMT_CLASS_SUBN_ADM; 1726 mad_hdr->ClassVersion = SAA_MAD_CLASS_VERSION; 1727 mad_hdr->R_Method = method; 1728 mad_hdr->AttributeID = h2b16(attr_id); 1729 1730 /* attribute modifier is all Fs since RIDs are no longer used */ 1731 mad_hdr->AttributeModifier = h2b32(0xffffffff); 1732 1733 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 1734 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1735 "ibmf_saa_impl_init_msg: %s, class = 0x%x, method = 0x%x," 1736 " attr_id = 0x%x\n", tnf_string, msg, "Sending MAD", 1737 tnf_opaque, class, mad_hdr->MgmtClass, 1738 tnf_opaque, method, mad_hdr->R_Method, 1739 tnf_opaque, attr_id, attr_id); 1740 1741 bzero(&sa_hdr, sizeof (ib_sa_hdr_t)); 1742 sa_hdr.ComponentMask = trans_info->si_trans_component_mask; 1743 1744 if (client_data != NULL) 1745 sa_hdr.SM_KEY = client_data->saa_client_sm_key; 1746 1747 /* 1748 * pack data for IB wire format; req_mad will have different pointers to 1749 * sa header and payload, mad_hdr will be the same 1750 */ 1751 req_mad->im_bufs_mad_hdr = mad_hdr; 1752 1753 ibmf_status = ibmf_saa_utils_pack_sa_hdr(&sa_hdr, 1754 &req_mad->im_bufs_cl_hdr, &req_mad->im_bufs_cl_hdr_len, 1755 km_sleep_flag); 1756 1757 if (ibmf_status != IBMF_SUCCESS) { 1758 1759 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1760 ibmf_saa_impl_init_msg, IBMF_TNF_ERROR, "", 1761 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n", 1762 tnf_string, msg, "ibmf_saa_utils_pack_sa_hdr() failed", 1763 tnf_int, ibmf_status, ibmf_status); 1764 1765 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1766 1767 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1768 &ibmf_msg); 1769 ASSERT(free_res == IBMF_SUCCESS); 1770 1771 goto bail; 1772 } 1773 1774 if (attr_id == SA_MULTIPATHRECORD_ATTRID) { 1775 1776 multipath_template = 1777 (sa_multipath_record_t *)trans_info->si_trans_template; 1778 1779 payload_length = sizeof (sa_multipath_record_t) + 1780 ((multipath_template->SGIDCount + 1781 multipath_template->DGIDCount) * sizeof (ib_gid_t)); 1782 1783 pack_attr_id = attr_id; 1784 } else { 1785 1786 /* trace record template is a path record */ 1787 pack_attr_id = (attr_id == SA_TRACERECORD_ATTRID) ? 1788 SA_PATHRECORD_ATTRID : attr_id; 1789 1790 payload_length = ibmf_saa_impl_get_attr_id_length(pack_attr_id); 1791 1792 if (payload_length == 0) { 1793 payload_length = trans_info->si_trans_template_length; 1794 1795 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 1796 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1797 "ibmf_saa_impl_init_msg: %s, length = %d\n", 1798 tnf_string, msg, 1799 "Unknown attribute. Using user-defined length.", 1800 tnf_uint, length, payload_length) 1801 } 1802 } 1803 1804 /* transport type depends on method */ 1805 switch (method) { 1806 1807 case SA_SUBN_ADM_GET: 1808 case SA_SUBN_ADM_DELETE: 1809 case SA_SUBN_ADM_GET_TABLE: 1810 case SA_SUBN_ADM_GET_TRACE_TABLE: 1811 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ; 1812 break; 1813 case SA_SUBN_ADM_SET: 1814 /* unsubscribes can be sequenced or unsequenced */ 1815 if (trans_info->si_trans_unseq_unsubscribe == B_TRUE) { 1816 transport_flags = 0; 1817 } else { 1818 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ; 1819 } 1820 break; 1821 case SA_SUBN_ADM_GET_MULTI: 1822 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ | 1823 IBMF_MSG_TRANS_FLAG_RMPP; 1824 break; 1825 default : 1826 ibmf_status = IBMF_UNSUPP_METHOD; 1827 goto bail; 1828 } 1829 1830 trans_info->si_trans_transport_flags = transport_flags; 1831 1832 if (trans_info->si_trans_template != NULL) { 1833 1834 ibmf_status = ibmf_saa_utils_pack_payload( 1835 trans_info->si_trans_template, payload_length, pack_attr_id, 1836 &req_mad->im_bufs_cl_data, &req_mad->im_bufs_cl_data_len, 1837 km_sleep_flag); 1838 if (ibmf_status != IBMF_SUCCESS) { 1839 1840 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1841 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1842 "ibmf_saa_impl_init_msg: %s, ibmf_status =" 1843 " %d\n", tnf_string, msg, 1844 "ibmf_saa_utils_pack_payload() failed", 1845 tnf_int, ibmf_status, ibmf_status); 1846 1847 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1848 1849 kmem_free(req_mad->im_bufs_cl_hdr, 1850 req_mad->im_bufs_cl_hdr_len); 1851 1852 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1853 &ibmf_msg); 1854 ASSERT(free_res == IBMF_SUCCESS); 1855 1856 goto bail; 1857 } 1858 1859 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1860 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1861 "ibmf_saa_impl_init_msg: %s, attr_id = 0x%x, length =" 1862 " %d\n", tnf_string, msg, "Packed payload successfully", 1863 tnf_opaque, attr_id, attr_id, 1864 tnf_uint, length, req_mad->im_bufs_cl_data_len); 1865 1866 /* non-RMPP transactions have template size limit */ 1867 if (((transport_flags & IBMF_MSG_TRANS_FLAG_RMPP) == 0) && 1868 ((req_mad->im_bufs_cl_data_len + req_mad->im_bufs_cl_hdr_len 1869 + sizeof (ib_mad_hdr_t)) > IBMF_MAD_SIZE)) { 1870 1871 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1872 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1873 "ibmf_saa_impl_init_msg: %s\n", tnf_string, msg, 1874 "Template too large to fit in single packet"); 1875 1876 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1877 1878 kmem_free(req_mad->im_bufs_cl_hdr, 1879 req_mad->im_bufs_cl_hdr_len); 1880 1881 kmem_free(req_mad->im_bufs_cl_data, 1882 req_mad->im_bufs_cl_data_len); 1883 1884 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1885 &ibmf_msg); 1886 ASSERT(free_res == IBMF_SUCCESS); 1887 1888 ibmf_status = IBMF_REQ_INVALID; 1889 goto bail; 1890 } 1891 } 1892 1893 mutex_enter(&saa_portp->saa_pt_mutex); 1894 1895 mad_hdr->TransactionID = h2b64(saa_portp->saa_pt_current_tid++); 1896 1897 bcopy(&saa_portp->saa_pt_ibmf_retrans, ibmf_retransp, 1898 sizeof (ibmf_retrans_t)); 1899 1900 /* copy local addressing information to message */ 1901 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &ibmf_msg->im_local_addr, 1902 sizeof (ibmf_addr_info_t)); 1903 1904 /* copy global addressing information to message if in use */ 1905 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) { 1906 1907 ibmf_msg->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 1908 1909 bcopy(&saa_portp->saa_pt_ibmf_global_addr, 1910 &ibmf_msg->im_global_addr, 1911 sizeof (ibmf_global_addr_info_t)); 1912 } else { 1913 ibmf_msg->im_msg_flags = 0; 1914 } 1915 1916 mutex_exit(&saa_portp->saa_pt_mutex); 1917 1918 *msgp = ibmf_msg; 1919 *transport_flagsp = transport_flags; 1920 bail: 1921 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1922 ibmf_saa_impl_init_msg_end, 1923 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() exiting" 1924 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 1925 1926 return (ibmf_status); 1927 1928 } 1929 1930 /* 1931 * ibmf_saa_impl_new_smlid_retry: 1932 * 1933 * It's possible for the MasterSMLID to change while ibmf_saa is running. The 1934 * MasterSMLID is set when we first register with ibmf_saa. If a request 1935 * timesout, this function should be called to check whether the SM LID changed. 1936 * If so, it will call msg_transport again with the request. 1937 * 1938 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 1939 * same values passed to the original ibmf_msg_transport that timedout. The 1940 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 1941 * 1942 * If the lid did not change then this function returns IBMF_TRANS_TIMEOUT. 1943 * That way, callers can simply return the result of this function. 1944 * 1945 * Input Arguments 1946 * saa_portp pointer to saa_port structure 1947 * msgp ibmf message that timedout 1948 * ibmf_callback callback that should be called by msg_transport 1949 * ibmf_callback_arg args for ibmf_callback 1950 * transport_flags flags for ibmf_msg_transport 1951 * 1952 * Output Arguments 1953 * none 1954 * 1955 * Returns 1956 * IBMF_SUCCESS if lid changed and request was resent successfully, 1957 * IBMF_TRANS_TIMEOUT if lid did not change, 1958 * same values as ibmf_msg_transport() if lid changed but request could not be 1959 * resent. 1960 */ 1961 static int 1962 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 1963 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags) 1964 { 1965 ibt_hca_portinfo_t *ibt_portinfop; 1966 ib_lid_t master_sm_lid; 1967 int subnet_timeout; 1968 uint_t nports, size; 1969 ibmf_retrans_t ibmf_retrans; 1970 int ibmf_status; 1971 ibt_status_t ibt_status; 1972 1973 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1974 ibmf_saa_impl_new_smlid_retry_start, 1975 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() enter\n"); 1976 1977 _NOTE(ASSUMING_PROTECTED(*msgp)) 1978 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr)) 1979 1980 /* first query the portinfo to see if the lid changed */ 1981 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid, 1982 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size); 1983 1984 if (ibt_status != IBT_SUCCESS) { 1985 1986 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1987 ibmf_saa_impl_new_smlid_retry_err, IBMF_TNF_ERROR, "", 1988 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status =" 1989 " %d\n", tnf_string, msg, 1990 "ibt_query_hca_ports_byguid() failed", 1991 tnf_int, ibt_status, ibt_status); 1992 1993 ibmf_status = IBMF_TRANSPORT_FAILURE; 1994 1995 goto bail; 1996 } 1997 1998 master_sm_lid = ibt_portinfop->p_sm_lid; 1999 subnet_timeout = ibt_portinfop->p_subnet_timeout; 2000 2001 ibt_free_portinfo(ibt_portinfop, size); 2002 2003 /* if master smlid is different than the remote lid we sent to */ 2004 if (master_sm_lid != msgp->im_local_addr.ia_remote_lid) { 2005 2006 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L2, 2007 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2008 "ibmf_saa_impl_new_smlid_retry: %s, new_lid 0x%x," 2009 " old_lid 0x%x\n", tnf_string, msg, 2010 "master smlid has changed. retrying msg_transport", 2011 tnf_opaque, new_lid, master_sm_lid, 2012 tnf_opaque, old_lid, msgp->im_local_addr.ia_remote_lid); 2013 2014 mutex_enter(&saa_portp->saa_pt_mutex); 2015 2016 /* update the master sm lid value in ibmf_saa */ 2017 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2018 master_sm_lid; 2019 2020 /* new tid needed */ 2021 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2022 h2b64(saa_portp->saa_pt_current_tid++); 2023 2024 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2025 sizeof (ibmf_retrans_t)); 2026 2027 /* update the subnet timeout since this may be a new sm/sa */ 2028 saa_portp->saa_pt_timeout = subnet_timeout; 2029 2030 /* place upper bound on subnet timeout in case of faulty SM */ 2031 if (saa_portp->saa_pt_timeout > IBMF_SAA_MAX_SUBNET_TIMEOUT) 2032 saa_portp->saa_pt_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT; 2033 2034 /* increment the reference count to account for the cpi call */ 2035 saa_portp->saa_pt_reference_count++; 2036 2037 mutex_exit(&saa_portp->saa_pt_mutex); 2038 2039 /* update the remote lid for this particular message */ 2040 msgp->im_local_addr.ia_remote_lid = master_sm_lid; 2041 2042 /* get the classportinfo again since this may be a new sm/sa */ 2043 ibmf_saa_impl_get_classportinfo(saa_portp); 2044 2045 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 2046 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2047 ibmf_callback, ibmf_callback_arg, transport_flags); 2048 2049 if (ibmf_status != IBMF_SUCCESS) { 2050 2051 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2052 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2053 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status = " 2054 "%d\n", tnf_string, msg, 2055 "ibmf_msg_transport() failed", 2056 tnf_int, ibmf_status, ibmf_status); 2057 } 2058 2059 goto bail; 2060 } 2061 2062 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2063 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2064 "ibmf_saa_impl_new_smlid_retry: %s, master_smlid = 0x%x\n", 2065 tnf_string, msg, 2066 "master smlid did not change. returning failure", 2067 tnf_opaque, master_smlid, master_sm_lid); 2068 2069 /* mark status as timeout since that was original failure */ 2070 ibmf_status = IBMF_TRANS_TIMEOUT; 2071 2072 bail: 2073 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2074 ibmf_saa_impl_new_smlid_retry_end, 2075 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() exiting" 2076 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2077 2078 return (ibmf_status); 2079 } 2080 2081 /* 2082 * ibmf_saa_impl_revert_to_qp1() 2083 * 2084 * The SA that we had contact with via redirect may fail to respond. If this 2085 * occurs SA should revert back to qp1 and the SMLID set in the port. 2086 * msg_transport for the message that timed out will be retried with 2087 * these new parameters. 2088 * 2089 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 2090 * same values passed to the original ibmf_msg_transport that timedout. The 2091 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 2092 * 2093 * Input Arguments 2094 * saa_portp pointer to saa_port structure 2095 * msgp ibmf message that timedout 2096 * ibmf_callback callback that should be called by msg_transport 2097 * ibmf_callback_arg args for ibmf_callback 2098 * transport_flags flags for ibmf_msg_transport 2099 * 2100 * Output Arguments 2101 * none 2102 * 2103 * Returns 2104 * none 2105 */ 2106 static int 2107 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp, 2108 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags) 2109 { 2110 ibt_hca_portinfo_t *ibt_portinfop; 2111 ib_lid_t master_sm_lid, base_lid; 2112 uint8_t sm_sl; 2113 int subnet_timeout; 2114 uint_t nports, size; 2115 ibmf_retrans_t ibmf_retrans; 2116 int ibmf_status; 2117 ibt_status_t ibt_status; 2118 2119 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 2120 ibmf_saa_impl_revert_to_qp1_start, 2121 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() enter\n"); 2122 2123 _NOTE(ASSUMING_PROTECTED(*msgp)) 2124 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr)) 2125 2126 /* first query the portinfo to see if the lid changed */ 2127 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid, 2128 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size); 2129 2130 if (ibt_status != IBT_SUCCESS) { 2131 2132 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2133 ibmf_saa_impl_revert_to_qp1_err, IBMF_TNF_ERROR, "", 2134 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status =" 2135 " %d\n", tnf_string, msg, 2136 "ibt_query_hca_ports_byguid() failed", 2137 tnf_int, ibt_status, ibt_status); 2138 2139 ibmf_status = IBMF_TRANSPORT_FAILURE; 2140 2141 goto bail; 2142 } 2143 2144 master_sm_lid = ibt_portinfop->p_sm_lid; 2145 base_lid = ibt_portinfop->p_base_lid; 2146 sm_sl = ibt_portinfop->p_sm_sl; 2147 subnet_timeout = ibt_portinfop->p_subnet_timeout; 2148 2149 ibt_free_portinfo(ibt_portinfop, size); 2150 2151 2152 mutex_enter(&saa_portp->saa_pt_mutex); 2153 2154 saa_portp->saa_pt_redirect_active = B_FALSE; 2155 2156 /* update the address info in ibmf_saa */ 2157 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = base_lid; 2158 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = master_sm_lid; 2159 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = sm_sl; 2160 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1; 2161 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = IB_PKEY_DEFAULT_LIMITED; 2162 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY; 2163 saa_portp->saa_pt_ibmf_msg_flags = 0; 2164 2165 /* new tid needed */ 2166 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2167 h2b64(saa_portp->saa_pt_current_tid++); 2168 2169 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2170 sizeof (ibmf_retrans_t)); 2171 2172 /* update the subnet timeout since this may be a new sm/sa */ 2173 saa_portp->saa_pt_timeout = subnet_timeout; 2174 2175 /* place upper bound on subnet timeout in case of faulty SM */ 2176 if (saa_portp->saa_pt_timeout > IBMF_SAA_MAX_SUBNET_TIMEOUT) 2177 saa_portp->saa_pt_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT; 2178 2179 /* increment the reference count to account for the cpi call */ 2180 saa_portp->saa_pt_reference_count++; 2181 2182 mutex_exit(&saa_portp->saa_pt_mutex); 2183 2184 /* update the address info for this particular message */ 2185 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr, 2186 sizeof (ibmf_addr_info_t)); 2187 msgp->im_msg_flags = 0; /* No GRH */ 2188 2189 /* get the classportinfo again since this may be a new sm/sa */ 2190 ibmf_saa_impl_get_classportinfo(saa_portp); 2191 2192 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 2193 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2194 ibmf_callback, ibmf_callback_args, transport_flags); 2195 2196 if (ibmf_status != IBMF_SUCCESS) { 2197 2198 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2199 ibmf_saa_impl_revert_to_qp1, IBMF_TNF_TRACE, "", 2200 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status = " 2201 "%d\n", tnf_string, msg, 2202 "ibmf_msg_transport() failed", 2203 tnf_int, ibmf_status, ibmf_status); 2204 } 2205 2206 bail: 2207 2208 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2209 ibmf_saa_impl_revert_to_qp1_end, 2210 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() exiting" 2211 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2212 2213 return (ibmf_status); 2214 } 2215 2216 /* 2217 * ibmf_saa_impl_async_event_cb: 2218 * ibmf event callback, argument to ibmf_register 2219 * ibmf_handle is unused 2220 */ 2221 /* ARGSUSED */ 2222 static void 2223 ibmf_saa_impl_async_event_cb( 2224 ibmf_handle_t ibmf_handle, 2225 void *clnt_private, 2226 ibmf_async_event_t event_type) 2227 { 2228 saa_port_t *saa_portp; 2229 2230 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2231 ibmf_saa_impl_async_event_cb_start, IBMF_TNF_TRACE, "", 2232 "ibmf_saa_impl_async_event_cb: Handling event type 0x%x\n", 2233 tnf_opaque, event_type, event_type); 2234 2235 saa_portp = (saa_port_t *)clnt_private; 2236 ASSERT(saa_portp != NULL); 2237 2238 switch (event_type) { 2239 2240 case IBMF_CI_OFFLINE: 2241 ibmf_saa_impl_hca_detach(saa_portp); 2242 break; 2243 default: 2244 break; 2245 } 2246 2247 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, 2248 ibmf_saa_impl_async_event_cb_end, 2249 IBMF_TNF_TRACE, "", "ibmf_saa_impl_async_event_cb() exit\n"); 2250 } 2251 2252 2253 /* 2254 * ibmf_saa_impl_ibt_async_handler: 2255 * MUST NOT BE STATIC (referred from within IBMF) 2256 */ 2257 void 2258 ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event) 2259 { 2260 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2261 ibmf_saa_impl_ibt_async_handler_start, IBMF_TNF_TRACE, "", 2262 "ibmf_saa_impl_ibt_async_handler: Handling event code 0x%x\n", 2263 tnf_opaque, code, code); 2264 2265 switch (code) { 2266 2267 case IBT_EVENT_PORT_UP: 2268 ibmf_saa_impl_port_up(event->ev_hca_guid, event->ev_port); 2269 break; 2270 case IBT_ERROR_PORT_DOWN: 2271 ibmf_saa_impl_port_down(event->ev_hca_guid, event->ev_port); 2272 break; 2273 default: 2274 break; 2275 } 2276 2277 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_async_handler_end, 2278 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibt_async_handler() exit\n"); 2279 } 2280 2281 /* 2282 * ibmf_saa_impl_port_up: 2283 */ 2284 static void 2285 ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num) 2286 { 2287 saa_port_t *saa_portp = NULL; 2288 int is_ready; 2289 ibt_hca_portinfo_t *ibt_portinfop; 2290 ib_lid_t master_sm_lid; 2291 uint_t nports, size; 2292 ibt_status_t ibt_status; 2293 boolean_t event_subs = B_FALSE; 2294 2295 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_up_start, 2296 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up: Handling port up" 2297 " guid %016" PRIx64 " port %d\n", 2298 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2299 2300 /* Get classportinfo of corresponding entry */ 2301 mutex_enter(&saa_statep->saa_port_list_mutex); 2302 2303 saa_portp = saa_statep->saa_port_list; 2304 while (saa_portp != NULL) { 2305 2306 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid && 2307 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) { 2308 2309 mutex_enter(&saa_portp->saa_pt_mutex); 2310 2311 is_ready = (saa_portp->saa_pt_state 2312 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE; 2313 2314 /* 2315 * increment reference count to account for cpi and 2316 * informinfos. All 4 informinfo's sent are treated as 2317 * one port client reference 2318 */ 2319 if (is_ready == B_TRUE) 2320 saa_portp->saa_pt_reference_count += 2; 2321 2322 mutex_exit(&saa_portp->saa_pt_mutex); 2323 2324 if (is_ready == B_TRUE) 2325 break; /* normally, only 1 port entry */ 2326 } 2327 saa_portp = saa_portp->next; 2328 } 2329 2330 mutex_exit(&saa_statep->saa_port_list_mutex); 2331 2332 if (saa_portp != NULL && is_ready == B_TRUE) { 2333 2334 /* verify whether master sm lid changed */ 2335 2336 /* first query the portinfo to see if the lid changed */ 2337 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num, 2338 &ibt_portinfop, &nports, &size); 2339 2340 if (ibt_status != IBT_SUCCESS) { 2341 2342 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2343 ibmf_saa_impl_port_up_err, IBMF_TNF_ERROR, "", 2344 "ibmf_saa_impl_port_up: %s, ibmf_status =" 2345 " %d\n", tnf_string, msg, 2346 "ibt_query_hca_ports_byguid() failed", 2347 tnf_int, ibt_status, ibt_status); 2348 2349 goto bail; 2350 } 2351 2352 master_sm_lid = ibt_portinfop->p_sm_lid; 2353 2354 ibt_free_portinfo(ibt_portinfop, size); 2355 2356 /* check whether we need to subscribe for events */ 2357 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2358 2359 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2360 B_TRUE : B_FALSE; 2361 2362 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2363 2364 /* update the master smlid */ 2365 mutex_enter(&saa_portp->saa_pt_mutex); 2366 2367 /* update the master sm lid value in ibmf_saa */ 2368 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2369 master_sm_lid; 2370 2371 /* if we're not subscribed for events, dec reference count */ 2372 if (event_subs == B_FALSE) 2373 saa_portp->saa_pt_reference_count--; 2374 2375 mutex_exit(&saa_portp->saa_pt_mutex); 2376 2377 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2378 ibmf_saa_impl_port_up, IBMF_TNF_TRACE, "", 2379 "ibmf_saa_impl_port_up: %s, master_sm_lid = 0x%x\n", 2380 tnf_string, msg, 2381 "port is up. Sending classportinfo request", 2382 tnf_opaque, master_sm_lid, master_sm_lid); 2383 2384 /* get the classportinfo again */ 2385 ibmf_saa_impl_get_classportinfo(saa_portp); 2386 2387 /* 2388 * resubscribe to events if there are subscribers since SA may 2389 * have removed our subscription records when the port went down 2390 */ 2391 if (event_subs == B_TRUE) 2392 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE); 2393 } 2394 2395 bail: 2396 2397 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_up_end, 2398 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up() exit\n"); 2399 } 2400 2401 /* 2402 * ibmf_saa_impl_port_down: 2403 */ 2404 static void 2405 ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num) 2406 { 2407 2408 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_down_start, 2409 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down: Handling port down" 2410 " guid %016" PRIx64 " port %d\n", 2411 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2412 2413 2414 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_down_end, 2415 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down() exit\n"); 2416 } 2417 2418 /* 2419 * ibmf_saa_impl_hca_detach: 2420 * find entry, unregister if there are no clients 2421 * have to unregister since ibmf needs to close the hca and will only do this if 2422 * no clients are registered 2423 */ 2424 static void 2425 ibmf_saa_impl_hca_detach(saa_port_t *saa_removed) 2426 { 2427 saa_port_t *saa_portp; 2428 boolean_t must_unreg, must_unsub; 2429 2430 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_start, 2431 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: Detaching" 2432 " entry %016" PRIx64 "\n", tnf_opaque, entry, saa_removed); 2433 2434 /* find this entry */ 2435 mutex_enter(&saa_statep->saa_port_list_mutex); 2436 2437 saa_portp = saa_statep->saa_port_list; 2438 while (saa_portp != NULL) { 2439 2440 if (saa_portp == saa_removed) 2441 break; 2442 2443 saa_portp = saa_portp->next; 2444 } 2445 mutex_exit(&saa_statep->saa_port_list_mutex); 2446 2447 ASSERT(saa_portp != NULL); 2448 2449 if (saa_portp == NULL) { 2450 2451 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2452 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2453 "ibmf_saa_impl_hca_detach: %s, entry %016" 2454 PRIx64 "\n", 2455 tnf_string, msg, 2456 "Port entry NOT found", 2457 tnf_opaque, entryp, saa_removed); 2458 2459 goto bail; 2460 } 2461 2462 /* if there are clients expecting Reports(), unsusbscribe */ 2463 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2464 2465 must_unsub = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2466 B_TRUE : B_FALSE; 2467 2468 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2469 2470 /* fail if outstanding transactions */ 2471 mutex_enter(&saa_portp->saa_pt_mutex); 2472 2473 if (saa_portp->saa_pt_num_outstanding_trans > 0) { 2474 2475 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L1, 2476 ibmf_saa_impl_fini_err, IBMF_TNF_TRACE, "", 2477 "ibmf_saa_impl_fini: %s, port = %016" PRIx64 2478 ", num transactions = %d\n", 2479 tnf_string, msg, "Detaching HCA." 2480 " Outstanding transactions on port.", 2481 tnf_opaque, port, 2482 saa_portp->saa_pt_port_guid, 2483 tnf_uint, outstanding_transactions, 2484 saa_portp->saa_pt_num_outstanding_trans); 2485 2486 mutex_exit(&saa_portp->saa_pt_mutex); 2487 2488 goto bail; 2489 } 2490 2491 2492 /* 2493 * increment reference count by one to account for unsubscribe requests 2494 * that are about to be sent. All four informinfo's are treated as one 2495 * port client reference. The count will be decremented by 2496 * subscribe_events() before the call returns. 2497 */ 2498 if (must_unsub == B_TRUE) 2499 saa_portp->saa_pt_reference_count++; 2500 2501 mutex_exit(&saa_portp->saa_pt_mutex); 2502 2503 /* 2504 * try and unsubscribe from SA. Generate synchronous, unsequenced 2505 * unsubscribe requests. 2506 */ 2507 if (must_unsub == B_TRUE) 2508 ibmf_saa_subscribe_events(saa_portp, B_FALSE, B_TRUE); 2509 2510 /* warning if registered clients */ 2511 mutex_enter(&saa_portp->saa_pt_mutex); 2512 2513 if (saa_portp->saa_pt_reference_count > 0) { 2514 2515 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2516 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2517 "ibmf_saa_impl_hca_detach: %s, port %016" 2518 PRIx64 "\n", 2519 tnf_string, msg, 2520 "Detaching HCA for port with clients still" 2521 " registered", tnf_opaque, port, 2522 saa_portp->saa_pt_port_guid); 2523 } 2524 2525 /* synchronize on end of registration */ 2526 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) { 2527 2528 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1, 2529 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2530 "ibmf_saa_impl_hca_detach: %s\n", 2531 tnf_string, msg, "someone is registering. waiting" 2532 " for them to finish"); 2533 2534 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv, 2535 &saa_portp->saa_pt_mutex); 2536 2537 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1, 2538 ibmf_saa_impl_hca_detach, 2539 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: %s\n", 2540 tnf_string, msg, "done waiting"); 2541 } 2542 2543 /* unregister from ibmf */ 2544 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) { 2545 must_unreg = B_TRUE; 2546 } else 2547 must_unreg = B_FALSE; 2548 2549 ibmf_saa_impl_invalidate_port(saa_portp); 2550 2551 mutex_exit(&saa_portp->saa_pt_mutex); 2552 2553 if (must_unreg == B_TRUE) { 2554 if (ibmf_saa_impl_ibmf_unreg(saa_portp) != IBMF_SUCCESS) { 2555 mutex_enter(&saa_portp->saa_pt_mutex); 2556 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 2557 (void) ibmf_saa_impl_init_kstats(saa_portp); 2558 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 2559 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_READY; 2560 if (must_unsub == B_TRUE) 2561 saa_portp->saa_pt_reference_count++; 2562 mutex_exit(&saa_portp->saa_pt_mutex); 2563 2564 if (must_unsub == B_TRUE) { 2565 ibmf_saa_subscribe_events(saa_portp, B_TRUE, 2566 B_FALSE); 2567 } 2568 } 2569 } 2570 bail: 2571 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_end, 2572 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach() exit\n"); 2573 } 2574 2575 /* ARGSUSED */ 2576 void 2577 ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, void *args) 2578 { 2579 saa_impl_trans_info_t *trans_info; 2580 int status; 2581 size_t length; 2582 void *result; 2583 saa_port_t *saa_portp; 2584 saa_client_data_t *client_data; 2585 int ibmf_status; 2586 boolean_t ignore_data; 2587 ibmf_retrans_t ibmf_retrans; 2588 boolean_t sa_is_redirected = B_FALSE; 2589 2590 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_start, 2591 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() enter\n"); 2592 2593 trans_info = (saa_impl_trans_info_t *)args; 2594 2595 client_data = trans_info->si_trans_client_data; 2596 saa_portp = trans_info->si_trans_port; 2597 2598 mutex_enter(&saa_portp->saa_pt_mutex); 2599 sa_is_redirected = saa_portp->saa_pt_redirect_active; 2600 mutex_exit(&saa_portp->saa_pt_mutex); 2601 2602 if ((msgp->im_msg_status == IBMF_TRANS_TIMEOUT) && 2603 (sa_is_redirected == B_TRUE)) { 2604 2605 /* 2606 * We should retry the request using SM_LID and QP1 if we 2607 * have been using redirect up until now 2608 */ 2609 ibmf_status = ibmf_saa_impl_revert_to_qp1( 2610 saa_portp, msgp, ibmf_saa_async_cb, args, 2611 trans_info->si_trans_transport_flags); 2612 2613 /* 2614 * If revert_to_qp1 returns success msg was resent. 2615 * Otherwise msg could not be resent. Continue normally 2616 */ 2617 if (ibmf_status == IBMF_SUCCESS) 2618 goto bail; 2619 2620 } else if (msgp->im_msg_status == IBMF_TRANS_TIMEOUT) { 2621 2622 2623 ibmf_status = ibmf_saa_impl_new_smlid_retry(saa_portp, msgp, 2624 ibmf_saa_async_cb, args, 2625 trans_info->si_trans_transport_flags); 2626 2627 /* 2628 * if smlid_retry() returns success sm lid changed and msg 2629 * was resent. Otherwise, lid did not change or msg could not 2630 * be resent. Continue normally. 2631 */ 2632 if (ibmf_status == IBMF_SUCCESS) 2633 goto bail; 2634 2635 /* 2636 * check whether we've received anything from the SA in a while. 2637 * If we have, this function will retry and return success. If 2638 * we haven't continue normally so that we return a timeout to 2639 * the client 2640 */ 2641 ibmf_status = ibmf_saa_check_sa_and_retry( 2642 saa_portp, msgp, ibmf_saa_async_cb, args, 2643 trans_info->si_trans_send_time, 2644 trans_info->si_trans_transport_flags); 2645 2646 if (ibmf_status == IBMF_SUCCESS) 2647 goto bail; 2648 } 2649 2650 /* 2651 * If SA returned success but mad status is busy, retry a few times. 2652 * If SA returned success but mad status says redirect is required, 2653 * update the address info and retry the request to the new SA address 2654 */ 2655 if (msgp->im_msg_status == IBMF_SUCCESS) { 2656 2657 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL); 2658 2659 if ((b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) == 2660 MAD_STATUS_BUSY) && 2661 (trans_info->si_trans_retry_busy_count < 2662 IBMF_SAA_MAX_BUSY_RETRY_COUNT)) { 2663 2664 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2665 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2666 "ibmf_saa_async_cb: %s, retry_count = %d\n", 2667 tnf_string, msg, 2668 "async response returned busy status", 2669 tnf_int, retry_count, 2670 trans_info->si_trans_retry_busy_count); 2671 2672 trans_info->si_trans_retry_busy_count++; 2673 2674 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2675 sizeof (ibmf_retrans_t)); 2676 2677 ibmf_status = ibmf_msg_transport( 2678 saa_portp->saa_pt_ibmf_handle, 2679 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2680 ibmf_saa_async_cb, args, 2681 trans_info->si_trans_transport_flags); 2682 2683 /* 2684 * if retry is successful, quit here since async_cb will 2685 * get called again; otherwise, let this function call 2686 * handle the cleanup 2687 */ 2688 if (ibmf_status == IBMF_SUCCESS) 2689 goto bail; 2690 } else if (b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) 2691 == MAD_STATUS_REDIRECT_REQUIRED) { 2692 2693 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L2, 2694 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2695 "ibmf_saa_async_cb: " 2696 "async response returned redirect status\n"); 2697 2698 /* update address info and copy it into msgp */ 2699 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp); 2700 2701 /* retry with new address info */ 2702 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2703 sizeof (ibmf_retrans_t)); 2704 2705 ibmf_status = ibmf_msg_transport( 2706 saa_portp->saa_pt_ibmf_handle, 2707 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2708 ibmf_saa_async_cb, args, 2709 trans_info->si_trans_transport_flags); 2710 2711 /* 2712 * if retry is successful, quit here since async_cb will 2713 * get called again; otherwise, let this function call 2714 * handle the cleanup 2715 */ 2716 if (ibmf_status == IBMF_SUCCESS) 2717 goto bail; 2718 } 2719 } 2720 2721 mutex_enter(&saa_portp->saa_pt_mutex); 2722 2723 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 2724 saa_portp->saa_pt_num_outstanding_trans--; 2725 2726 mutex_exit(&saa_portp->saa_pt_mutex); 2727 2728 if ((trans_info->si_trans_callback == NULL) && 2729 (trans_info->si_trans_sub_callback == NULL)) 2730 ignore_data = B_TRUE; 2731 else 2732 ignore_data = B_FALSE; 2733 2734 ibmf_saa_impl_prepare_response(ibmf_handle, msgp, ignore_data, &status, 2735 &result, &length, B_FALSE); 2736 2737 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 2738 2739 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1); 2740 2741 if (status != IBMF_SUCCESS) 2742 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1); 2743 2744 if (status == IBMF_TRANS_TIMEOUT) 2745 IBMF_SAA_ADD32_KSTATS(saa_portp, requests_timedout, 1); 2746 2747 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 2748 2749 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2750 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2751 "ibmf_saa_async_cb: %s\n", tnf_string, msg, 2752 "Calling ibmf_saa client's callback"); 2753 2754 /* 2755 * there are three classes or trans_info users: ibmf_saa clients and 2756 * classportinfo requests; informinfo subscribe requests, and report 2757 * responses. For the first two, call the correct callback. For report 2758 * responses there's no need to notify anyone. 2759 */ 2760 if (trans_info->si_trans_callback != NULL) { 2761 /* ibmf_saa client or classportinfo request */ 2762 trans_info->si_trans_callback(trans_info->si_trans_callback_arg, 2763 length, result, status); 2764 } else if (trans_info->si_trans_sub_callback != NULL) { 2765 /* informinfo subscribe request */ 2766 trans_info->si_trans_sub_callback( 2767 trans_info->si_trans_callback_arg, length, result, status, 2768 trans_info->si_trans_sub_producer_type); 2769 } 2770 2771 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2772 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2773 "ibmf_saa_async_cb: %s\n", tnf_string, msg, 2774 "Returned from callback"); 2775 2776 if (client_data != NULL) { 2777 mutex_enter(&client_data->saa_client_mutex); 2778 2779 ASSERT(client_data->saa_client_num_pending_trans > 0); 2780 client_data->saa_client_num_pending_trans--; 2781 2782 if ((client_data->saa_client_num_pending_trans == 0) && 2783 (client_data->saa_client_state == SAA_CLIENT_STATE_WAITING)) 2784 cv_signal(&client_data->saa_client_state_cv); 2785 2786 mutex_exit(&client_data->saa_client_mutex); 2787 } 2788 2789 kmem_free(trans_info, sizeof (saa_impl_trans_info_t)); 2790 2791 bail: 2792 2793 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_end, 2794 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() exit\n"); 2795 } 2796 2797 /* 2798 * ibmf_saa_check_sa_and_retry: 2799 * 2800 * If a particular transaction times out, we don't want to give up if we know 2801 * the SA is responding. Check the time since we last received a response. If 2802 * it's less than ibmf_saa_max_wait_time retry the request. 2803 * 2804 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 2805 * same values passed to the original ibmf_msg_transport that timed out. The 2806 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 2807 * 2808 * If max_wait_time seconds have passed, this function returns IBMF_TIMEOUT. 2809 * That way, callers can simply return the result of this function. 2810 * 2811 * Input Arguments 2812 * saa_portp pointer to saa_port structure 2813 * msgp ibmf message that timedout 2814 * ibmf_callback callback that should be called by msg_transport 2815 * ibmf_callback_arg args for ibmf_callback 2816 * transport_flags flags for ibmf_msg_transport 2817 * 2818 * Output Arguments 2819 * none 2820 * 2821 * Returns 2822 * IBMF_SUCCESS if we've recently received data from the SA and request was 2823 * resent. 2824 * IBMF_TRANS_TIMEOUT if no data has been received from the SA in max_wait_time 2825 * same values as ibmf_msg_transport() if data has been received but request 2826 * could not be resent. 2827 */ 2828 static int 2829 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 2830 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, 2831 hrtime_t trans_send_time, int transport_flags) 2832 { 2833 hrtime_t curr_time, sa_uptime; 2834 ibmf_retrans_t ibmf_retrans; 2835 int ibmf_status; 2836 2837 do { 2838 2839 mutex_enter(&saa_portp->saa_pt_mutex); 2840 2841 sa_uptime = saa_portp->saa_pt_sa_uptime; 2842 2843 /* if nothing received from SA since we sent */ 2844 curr_time = gethrtime(); 2845 2846 /* 2847 * check if it's been a very long time since this 2848 * particular transaction was sent 2849 */ 2850 if (((curr_time - trans_send_time) / 1000000000) > 2851 ibmf_saa_trans_wait_time) { 2852 2853 mutex_exit(&saa_portp->saa_pt_mutex); 2854 2855 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1, 2856 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "", 2857 "ibmf_saa_check_sa_and_retry: %s, msgp = " 2858 "%p sa_uptime = %" PRIu64 ", trans send time = %" 2859 PRIu64 ", curr_time = %" PRIu64 "\n", 2860 tnf_string, msg, 2861 "Nothing received for this transaction", 2862 tnf_opaque, msgp, msgp, 2863 tnf_long, sa_uptime, sa_uptime, 2864 tnf_long, trans_send_time, trans_send_time, 2865 tnf_long, curr_time, curr_time); 2866 2867 ibmf_status = IBMF_TRANS_TIMEOUT; 2868 2869 break; 2870 } 2871 2872 /* 2873 * check time since we received something, 2874 * and make sure that it hasn't been an extra long 2875 * time for this particular transaction 2876 */ 2877 if (((curr_time - sa_uptime) / 1000000000) < 2878 ibmf_saa_max_wait_time) { 2879 2880 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L2, 2881 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "", 2882 "ibmf_saa_check_sa_and_retry: %s, msgp = " 2883 "%p sa_uptime = %" PRIu64 " trans_send_time = %" 2884 PRIu64 " curr_time = %" PRIu64 "\n", 2885 tnf_string, msg, "Something received. Retrying", 2886 tnf_opaque, msgp, msgp, 2887 tnf_long, sa_uptime, sa_uptime, 2888 tnf_long, trans_send_time, trans_send_time, 2889 tnf_long, curr_time, curr_time); 2890 2891 /* 2892 * something received in WAIT_TIME_IN_SECS; 2893 * resend request 2894 */ 2895 2896 /* new tid needed */ 2897 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2898 h2b64(saa_portp->saa_pt_current_tid++); 2899 2900 bcopy(&saa_portp->saa_pt_ibmf_retrans, 2901 &ibmf_retrans, sizeof (ibmf_retrans_t)); 2902 2903 mutex_exit(&saa_portp->saa_pt_mutex); 2904 2905 ibmf_status = ibmf_msg_transport( 2906 saa_portp->saa_pt_ibmf_handle, 2907 saa_portp->saa_pt_qp_handle, msgp, 2908 &ibmf_retrans, ibmf_callback, ibmf_callback_arg, 2909 transport_flags); 2910 2911 if (ibmf_status == IBMF_SUCCESS) 2912 goto bail; 2913 2914 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2915 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "", 2916 "ibmf_saa_check_sa_and_retry: %s, ibmf_status = " 2917 "%d\n", tnf_string, msg, 2918 "ibmf_msg_transport() failed", 2919 tnf_int, ibmf_status, ibmf_status); 2920 } else { 2921 2922 mutex_exit(&saa_portp->saa_pt_mutex); 2923 2924 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L1, 2925 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "", 2926 "ibmf_saa_check_sa_and_retry: %s, msgp = " 2927 "%p sa_uptime = %" PRIu64 " curr_time = %" 2928 PRIu64 "\n", tnf_string, msg, 2929 "Nothing received. Timing out", 2930 tnf_opaque, msgp, msgp, 2931 tnf_long, sa_uptime, sa_uptime, 2932 tnf_long, curr_time, curr_time); 2933 2934 ibmf_status = IBMF_TRANS_TIMEOUT; 2935 2936 break; 2937 } 2938 } while (ibmf_status == IBMF_TRANS_TIMEOUT); 2939 2940 bail: 2941 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2942 ibmf_saa_check_sa_and_retry_end, 2943 IBMF_TNF_TRACE, "", "ibmf_saa_check_sa_and_retry() exiting" 2944 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2945 2946 return (ibmf_status); 2947 } 2948 2949 2950 /* 2951 * ibmf_saa_impl_prepare_response: 2952 */ 2953 static void 2954 ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle, 2955 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result, 2956 size_t *length, boolean_t sleep_flag) 2957 { 2958 ibmf_msg_bufs_t *resp_buf; 2959 uint16_t attr_id; 2960 uint8_t method; 2961 boolean_t is_get_resp; 2962 uint16_t mad_status; 2963 uint16_t attr_offset; 2964 ib_sa_hdr_t *sa_hdr; 2965 2966 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 2967 ibmf_saa_impl_prepare_response_start, 2968 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() enter\n"); 2969 2970 _NOTE(ASSUMING_PROTECTED(*msgp)) 2971 2972 *result = NULL; 2973 *length = 0; 2974 sa_hdr = NULL; 2975 2976 resp_buf = &msgp->im_msgbufs_recv; 2977 2978 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*resp_buf)) 2979 2980 if (msgp->im_msg_status != IBMF_SUCCESS) { 2981 2982 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2983 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 2984 "ibmf_saa_impl_prepare_response: %s, msg_status = %d\n", 2985 tnf_string, msg, "Bad ibmf status", 2986 tnf_int, msg_status, msgp->im_msg_status); 2987 2988 *status = msgp->im_msg_status; 2989 2990 goto exit; 2991 } 2992 2993 if (resp_buf->im_bufs_mad_hdr == NULL) { 2994 2995 /* 2996 * this was an unsequenced transaction (from an unsubscribe for 2997 * following a CI_OFFLINE event) 2998 */ 2999 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3000 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3001 "ibmf_saa_impl_prepare_response: %s\n", 3002 tnf_string, msg, "Unsequenced transaction callback"); 3003 3004 goto exit; 3005 } 3006 3007 if ((mad_status = b2h16(resp_buf->im_bufs_mad_hdr->Status)) != 3008 MAD_STATUS_NO_INVALID_FIELDS) { 3009 3010 /* convert mad packet status to IBMF status */ 3011 switch (mad_status) { 3012 3013 case SA_STATUS_ERR_NO_RESOURCES: 3014 *status = IBMF_NO_RESOURCES; 3015 break; 3016 case SA_STATUS_ERR_REQ_INVALID: 3017 *status = IBMF_REQ_INVALID; 3018 break; 3019 case SA_STATUS_ERR_NO_RECORDS: 3020 *status = IBMF_NO_RECORDS; 3021 break; 3022 case SA_STATUS_ERR_TOO_MANY_RECORDS: 3023 *status = IBMF_TOO_MANY_RECORDS; 3024 break; 3025 case SA_STATUS_ERR_REQ_INVALID_GID: 3026 *status = IBMF_INVALID_GID; 3027 break; 3028 case SA_STATUS_ERR_REQ_INSUFFICIENT_COMPONENTS: 3029 *status = IBMF_INSUFF_COMPS; 3030 break; 3031 case MAD_STATUS_UNSUPP_METHOD: 3032 *status = IBMF_UNSUPP_METHOD; 3033 break; 3034 case MAD_STATUS_UNSUPP_METHOD_ATTR: 3035 *status = IBMF_UNSUPP_METHOD_ATTR; 3036 break; 3037 case MAD_STATUS_INVALID_FIELD: 3038 *status = IBMF_INVALID_FIELD; 3039 break; 3040 default: 3041 *status = IBMF_REQ_INVALID; 3042 break; 3043 } 3044 3045 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 3046 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3047 "ibmf_saa_impl_prepare_response: %s, mad_status = %x\n", 3048 tnf_string, msg, "Bad MAD status", 3049 tnf_int, mad_status, mad_status); 3050 3051 goto exit; 3052 } 3053 3054 attr_id = b2h16(resp_buf->im_bufs_mad_hdr->AttributeID); 3055 method = resp_buf->im_bufs_mad_hdr->R_Method; 3056 3057 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3058 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3059 "ibmf_saa_impl_prepare_response: attr_id = 0x%x, method = " 3060 "0x%x\n", 3061 tnf_opaque, attr_id, attr_id, 3062 tnf_opaque, method, method); 3063 3064 /* 3065 * ignore any data from deleteresp since there's no way to know whether 3066 * real data was returned; also ignore data if this was a Report 3067 * response 3068 */ 3069 if (method == SA_SUBN_ADM_DELETE_RESP) { 3070 3071 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3072 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3073 "impf_saa_impl_prepare_response: %s\n", 3074 tnf_string, msg, 3075 "DeleteResp or NoticeResp returned. " 3076 "Ignoring response data"); 3077 3078 *status = IBMF_SUCCESS; 3079 3080 *length = 0; 3081 *result = NULL; 3082 3083 goto exit; 3084 } 3085 3086 if (attr_id == SA_MULTIPATHRECORD_ATTRID) { 3087 3088 /* 3089 * getmulti is only for requests; attribute should not 3090 * be returned from SA 3091 */ 3092 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 3093 ibmf_saa_impl_prepare_response_err, IBMF_TNF_ERROR, 3094 "", "ibmf_saa_impl_prepare_response: %s\n", 3095 tnf_string, msg, "SA returned getmulti record"); 3096 3097 *status = IBMF_REQ_INVALID; 3098 3099 goto exit; 3100 } 3101 3102 /* if we are supposed to ignore data, stop here */ 3103 if (ignore_data == B_TRUE) { 3104 3105 *status = IBMF_SUCCESS; 3106 3107 goto exit; 3108 } 3109 3110 is_get_resp = resp_buf->im_bufs_mad_hdr->R_Method == 3111 SA_SUBN_ADM_GET_RESP ? B_TRUE: B_FALSE; 3112 3113 /* unpack the sa header to get the attribute offset */ 3114 *status = ibmf_saa_utils_unpack_sa_hdr(resp_buf->im_bufs_cl_hdr, 3115 resp_buf->im_bufs_cl_hdr_len, &sa_hdr, sleep_flag); 3116 if (*status != IBMF_SUCCESS) { 3117 3118 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3119 ibmf_saa_impl_prepare_response_err, 3120 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s," 3121 " ibmf_status = %d\n", tnf_string, msg, 3122 "Could not unpack sa hdr", tnf_int, ibmf_status, *status); 3123 3124 goto exit; 3125 } 3126 3127 attr_offset = sa_hdr->AttributeOffset; 3128 3129 /* 3130 * unpack data payload; if unpack function doesn't return success 3131 * (because it could not allocate memory) forward this status to waiting 3132 * client 3133 */ 3134 *status = ibmf_saa_utils_unpack_payload(resp_buf->im_bufs_cl_data, 3135 resp_buf->im_bufs_cl_data_len, attr_id, result, length, 3136 attr_offset, is_get_resp, sleep_flag); 3137 if (*status == IBMF_SUCCESS) { 3138 3139 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 3140 ibmf_saa_impl_prepare_response, 3141 IBMF_TNF_TRACE, "", 3142 "ibmf_saa_impl_prepare_response: attr_id = " 3143 "0x%x, attr_offset = %d, packed_payload_len = %d, " 3144 "unpacked_payload_len = %d\n", 3145 tnf_opaque, attr_id, attr_id, 3146 tnf_opaque, attr_offset, attr_offset, 3147 tnf_opaque, packed_payload_len, 3148 resp_buf->im_bufs_cl_data_len, 3149 tnf_opaque, unpacked_payload_len, *length); 3150 } else { 3151 3152 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1, 3153 ibmf_saa_impl_prepare_response_err, 3154 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s," 3155 "attr_id = 0x%x, attr_offset = %d, packed_payload_len = %d," 3156 "status = %d\n", 3157 tnf_string, msg, "Could not unpack payload", 3158 tnf_opaque, attr_id, attr_id, 3159 tnf_int, attr_offset, attr_offset, 3160 tnf_int, packed_payload_len, 3161 resp_buf->im_bufs_cl_data_len, 3162 tnf_int, status, *status); 3163 } 3164 exit: 3165 if (sa_hdr != NULL) 3166 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3167 3168 ibmf_saa_impl_free_msg(ibmf_handle, msgp); 3169 3170 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3171 ibmf_saa_impl_prepare_response_end, 3172 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() exit," 3173 " status = 0x%d\n", tnf_int, status, *status); 3174 } 3175 3176 3177 /* 3178 * ibmf_saa_impl_check_sa_support: 3179 * Checks the capability mask (returned from the SA classportinfo response) to 3180 * determine whether the sa supports the specified attribute ID. 3181 * 3182 * Input Arguments 3183 * cap_mask 16-bit capability mask returned in SA's classportinfo 3184 * attr_id attribute ID of current request 3185 * 3186 * Returns 3187 * IBMF_NOT_SUPPORTED if capability mask indicates SA does not support attribute 3188 * IBMF_SUCCESS otherwise 3189 */ 3190 static int 3191 ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id) 3192 { 3193 boolean_t attr_supported = B_TRUE; 3194 3195 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3196 ibmf_saa_impl_check_sa_support, IBMF_TNF_TRACE, "", 3197 "ibmf_saa_impl_check_sa_support: cap_mask = 0x%x, " 3198 "attr_id = 0x%x\n", tnf_opaque, cap_mask, cap_mask, 3199 tnf_opaque, attr_id, attr_id); 3200 3201 switch (attr_id) { 3202 3203 case SA_SWITCHINFORECORD_ATTRID: 3204 case SA_LINEARFDBRECORD_ATTRID: 3205 case SA_RANDOMFDBRECORD_ATTRID: 3206 case SA_MULTICASTFDBRECORD_ATTRID: 3207 case SA_SMINFORECORD_ATTRID: 3208 case SA_INFORMINFORECORD_ATTRID: 3209 case SA_LINKRECORD_ATTRID: 3210 case SA_GUIDINFORECORD_ATTRID: 3211 case SA_TRACERECORD_ATTRID: 3212 case SA_SERVICEASSNRECORD_ATTRID: 3213 3214 if ((cap_mask & 3215 SA_CAPMASK_OPT_RECORDS_SUPPORTED) == 0) { 3216 3217 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1, 3218 ibmf_saa_impl_check_sa_support, 3219 IBMF_TNF_ERROR, "", 3220 "ibmf_saa_impl_check_sa_support: %s, " 3221 "cap_mask = 0x%x\n", tnf_string, msg, 3222 "SA does not support optional records", 3223 tnf_opaque, cap_mask, cap_mask, 3224 tnf_opaque, attr_id, attr_id); 3225 3226 attr_supported = B_FALSE; 3227 } 3228 break; 3229 3230 case SA_MULTIPATHRECORD_ATTRID: 3231 3232 if ((cap_mask & SA_CAPMASK_MULTIPATH_SUPPORTED) == 0) { 3233 3234 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 3235 ibmf_saa_impl_check_sa_support, 3236 IBMF_TNF_ERROR, "", 3237 "ibmf_saa_impl_check_sa_support: %s, " 3238 "cap_mask = 0x%x\n", tnf_string, msg, 3239 "SA does not support multipath records", 3240 tnf_opaque, cap_mask, cap_mask); 3241 3242 attr_supported = B_FALSE; 3243 } 3244 break; 3245 3246 case SA_MCMEMBERRECORD_ATTRID: 3247 3248 if ((cap_mask & SA_CAPMASK_UD_MCAST_SUPPORTED) == 0) { 3249 3250 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 3251 ibmf_saa_impl_check_sa_support, 3252 IBMF_TNF_ERROR, "", 3253 "ibmf_saa_impl_check_sa_support: %s, " 3254 "cap_mask = 0x%x\n", tnf_string, msg, 3255 "SA does not support ud multicast", 3256 tnf_opaque, cap_mask, cap_mask); 3257 3258 attr_supported = B_FALSE; 3259 } 3260 break; 3261 3262 default: 3263 break; 3264 } /* switch */ 3265 3266 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3267 ibmf_saa_impl_check_sa_support_end, IBMF_TNF_TRACE, "", 3268 "ibmf_saa_impl_check_sa_support() exiting, attr_supported = %d\n", 3269 tnf_opaque, attr_supported, attr_supported); 3270 3271 if (attr_supported == B_FALSE) 3272 return (IBMF_UNSUPP_METHOD_ATTR); 3273 else 3274 return (IBMF_SUCCESS); 3275 } 3276 3277 /* 3278 * ibmf_saa_impl_get_attr_id_length: 3279 * 3280 * Returns the host size of the specified sa record. Returns 0 for unknown 3281 * attributes. multipath record size is a dynamic value given as a parameter 3282 * specified with the ibmf_sa_access() call. 3283 */ 3284 static uint_t 3285 ibmf_saa_impl_get_attr_id_length(uint16_t attr_id) 3286 { 3287 uint_t attr_length; 3288 3289 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3290 ibmf_saa_impl_get_attr_id_length_start, 3291 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length() enter\n"); 3292 3293 /* this function should not be used for multipath record */ 3294 ASSERT(attr_id != SA_MULTIPATHRECORD_ATTRID); 3295 3296 switch (attr_id) { 3297 case SA_CLASSPORTINFO_ATTRID: 3298 attr_length = sizeof (ib_mad_classportinfo_t); 3299 break; 3300 case SA_NOTICE_ATTRID: 3301 attr_length = sizeof (ib_mad_notice_t); 3302 break; 3303 case SA_INFORMINFO_ATTRID: 3304 attr_length = sizeof (ib_mad_informinfo_t); 3305 break; 3306 case SA_NODERECORD_ATTRID: 3307 attr_length = sizeof (sa_node_record_t); 3308 break; 3309 case SA_PORTINFORECORD_ATTRID: 3310 attr_length = sizeof (sa_portinfo_record_t); 3311 break; 3312 case SA_SLTOVLRECORD_ATTRID: 3313 attr_length = sizeof (sa_SLtoVLmapping_record_t); 3314 break; 3315 case SA_SWITCHINFORECORD_ATTRID: 3316 attr_length = sizeof (sa_switchinfo_record_t); 3317 break; 3318 case SA_LINEARFDBRECORD_ATTRID: 3319 attr_length = sizeof (sa_linearft_record_t); 3320 break; 3321 case SA_RANDOMFDBRECORD_ATTRID: 3322 attr_length = sizeof (sa_randomft_record_t); 3323 break; 3324 case SA_MULTICASTFDBRECORD_ATTRID: 3325 attr_length = sizeof (sa_multicastft_record_t); 3326 break; 3327 case SA_SMINFORECORD_ATTRID: 3328 attr_length = sizeof (sa_sminfo_record_t); 3329 break; 3330 case SA_INFORMINFORECORD_ATTRID: 3331 attr_length = sizeof (sa_informinfo_record_t); 3332 break; 3333 case SA_LINKRECORD_ATTRID: 3334 attr_length = sizeof (sa_link_record_t); 3335 break; 3336 case SA_GUIDINFORECORD_ATTRID: 3337 attr_length = sizeof (sa_guidinfo_record_t); 3338 break; 3339 case SA_SERVICERECORD_ATTRID: 3340 attr_length = sizeof (sa_service_record_t); 3341 break; 3342 case SA_PARTITIONRECORD_ATTRID: 3343 attr_length = sizeof (sa_pkey_table_record_t); 3344 break; 3345 case SA_PATHRECORD_ATTRID: 3346 attr_length = sizeof (sa_path_record_t); 3347 break; 3348 case SA_VLARBRECORD_ATTRID: 3349 attr_length = sizeof (sa_VLarb_table_record_t); 3350 break; 3351 case SA_MCMEMBERRECORD_ATTRID: 3352 attr_length = sizeof (sa_mcmember_record_t); 3353 break; 3354 case SA_TRACERECORD_ATTRID: 3355 attr_length = sizeof (sa_trace_record_t); 3356 break; 3357 case SA_SERVICEASSNRECORD_ATTRID: 3358 attr_length = sizeof (sa_service_assn_record_t); 3359 break; 3360 default: 3361 /* should only get the above type of packets */ 3362 attr_length = 0; 3363 break; 3364 } 3365 3366 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3367 ibmf_saa_impl_get_attr_id_length_end, 3368 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length():" 3369 " attr_id: 0x%x size %d\n", 3370 tnf_opaque, attr_id, attr_id, tnf_uint, attr_length, attr_length); 3371 3372 return (attr_length); 3373 } 3374 3375 /* 3376 * ibmf_saa_impl_free_msg: 3377 * Takes a completed message and free memory associated with the message, 3378 * including the individual fields of the im_msgbufs_send. 3379 * ibmf_free_msg, called at the end of this function, takes a pointer to the 3380 * message pointer so that it can set the message pointer to NULL. This 3381 * function takes just the message pointer so the msgp will not be NULL after 3382 * this function returns. 3383 * 3384 * Input Arguments 3385 * ibmf_hdl ibmf handle used in ibmf_msg_alloc 3386 * msgp pointer to ibmf_msg_t to free 3387 * 3388 * Returns 3389 * void 3390 */ 3391 static void 3392 ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp) 3393 { 3394 int res; 3395 3396 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3397 ibmf_saa_impl_free_msg_start, 3398 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() enter: msg %p\n", 3399 tnf_opaque, msg, msgp); 3400 3401 ASSERT(msgp != NULL); 3402 3403 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, 3404 sizeof (ib_mad_hdr_t)); 3405 3406 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_hdr, 3407 msgp->im_msgbufs_send.im_bufs_cl_hdr_len); 3408 3409 if (msgp->im_msgbufs_send.im_bufs_cl_data_len > 0) 3410 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_data, 3411 msgp->im_msgbufs_send.im_bufs_cl_data_len); 3412 3413 res = ibmf_free_msg(ibmf_hdl, &msgp); 3414 ASSERT(res == IBMF_SUCCESS); 3415 3416 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3417 ibmf_saa_impl_free_msg_end, 3418 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() exit\n"); 3419 } 3420 3421 /* 3422 * ibmf_saa_impl_get_port_guid: 3423 */ 3424 static int 3425 ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop, 3426 ib_guid_t *guid_ret) 3427 { 3428 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3429 ibmf_saa_impl_get_port_guid_start, 3430 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_port_guid() enter\n"); 3431 3432 if (ibt_portinfop->p_linkstate != IBT_PORT_ACTIVE) { 3433 3434 return (IBMF_BAD_PORT_STATE); 3435 } 3436 3437 if (ibt_portinfop->p_sgid_tbl_sz == 0) { 3438 3439 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L2, 3440 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "", 3441 "ibmf_saa_impl_get_port_guid: %s\n", tnf_string, msg, 3442 "portinfo sgid table size is 0. Exiting.\n"); 3443 3444 return (IBMF_TRANSPORT_FAILURE); 3445 } 3446 3447 *guid_ret = ibt_portinfop->p_sgid_tbl[0].gid_guid; 3448 3449 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3450 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "", 3451 "ibmf_saa_impl_get_port_guid: Returning port_guid %016" PRIx64 "\n", 3452 tnf_opaque, port_guid, *guid_ret); 3453 3454 return (IBMF_SUCCESS); 3455 } 3456 3457 /* 3458 * ibmf_saa_impl_set_transaction_params: 3459 */ 3460 static void 3461 ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp, 3462 ibt_hca_portinfo_t *portinfop) 3463 { 3464 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3465 ibmf_saa_impl_set_transaction_params_start, 3466 IBMF_TNF_TRACE, "", 3467 "ibmf_saa_impl_set_transaction_params() enter\n"); 3468 3469 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 3470 3471 saa_portp->saa_pt_ibmf_retrans.retrans_retries = 3472 IBMF_SAA_RETRANS_RETRIES; 3473 /* 3474 * For the first transaction (generally getting the 3475 * classportinfo) have ibmf pick our timeouts. It should be using the 3476 * default IB spec values. 3477 * Once we get the classportinfo we'll update the correct response time 3478 * value (rtv) and round-trip time (rttv). ibmf should always calculate 3479 * trans_to since it depends on the particular transaction's number of 3480 * packets. 3481 */ 3482 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = 0; 3483 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = 0; 3484 saa_portp->saa_pt_ibmf_retrans.retrans_trans_to = 0; 3485 3486 /* 3487 * Assume that the SA supports all optional records. If it 3488 * does not, the request will get returned with ERR_NOT_SUPP. When 3489 * the classportinfo response comes back we will update the cap mask 3490 * to prevent unnecessary unsupported requests. 3491 */ 3492 saa_portp->saa_pt_sa_cap_mask = 0xFFFF; 3493 3494 saa_portp->saa_pt_ibmf_msg_flags = 0; 3495 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1; 3496 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = 3497 IB_PKEY_DEFAULT_LIMITED; 3498 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY; 3499 3500 /* 3501 * fill out addr information for MADs that will be sent 3502 * to SA on this port 3503 */ 3504 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = portinfop->p_base_lid; 3505 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = portinfop->p_sm_lid; 3506 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = portinfop->p_sm_sl; 3507 3508 /* place upper bound on subnet timeout in case of faulty SM */ 3509 saa_portp->saa_pt_timeout = portinfop->p_subnet_timeout; 3510 3511 if (saa_portp->saa_pt_timeout > IBMF_SAA_MAX_SUBNET_TIMEOUT) { 3512 3513 saa_portp->saa_pt_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT; 3514 } 3515 3516 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 3517 ibmf_saa_impl_set_transaction_params, 3518 IBMF_TNF_TRACE, "", 3519 "ibmf_saa_impl_set_transaction_params: local_lid = 0x%x, " 3520 "sm_lid = 0x%x, sm_sl = 0x%x, sn_timeout = 0x%x\n", 3521 tnf_opaque, local_lid, portinfop->p_base_lid, 3522 tnf_opaque, sm_lid, portinfop->p_sm_lid, 3523 tnf_opaque, sm_sl, portinfop->p_sm_sl, 3524 tnf_opaque, subnet_timeout, portinfop->p_subnet_timeout); 3525 3526 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3527 ibmf_saa_impl_set_transaction_params_end, 3528 IBMF_TNF_TRACE, "", 3529 "ibmf_saa_impl_set_transaction_params() exit\n"); 3530 } 3531 3532 3533 /* 3534 * ibmf_saa_impl_update_sa_address_info 3535 */ 3536 static void 3537 ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, ibmf_msg_t *msgp) 3538 { 3539 void *result; 3540 ib_sa_hdr_t *sa_hdr; 3541 int rv; 3542 size_t length; 3543 uint16_t attr_id; 3544 ib_mad_classportinfo_t *cpi; 3545 ibmf_global_addr_info_t *gaddrp = &saa_portp->saa_pt_ibmf_global_addr; 3546 ibt_hca_portinfo_t *ibt_pinfo; 3547 uint_t nports, size; 3548 ibt_status_t ibt_status; 3549 3550 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3551 ibmf_saa_impl_update_sa_address_info, 3552 IBMF_TNF_TRACE, "", 3553 "ibmf_saa_impl_update_sa_address_info() enter\n"); 3554 3555 /* 3556 * decode the respons of msgp as a classportinfo attribute 3557 */ 3558 rv = ibmf_saa_utils_unpack_sa_hdr(msgp->im_msgbufs_recv.im_bufs_cl_hdr, 3559 msgp->im_msgbufs_recv.im_bufs_cl_hdr_len, &sa_hdr, KM_NOSLEEP); 3560 if (rv != IBMF_SUCCESS) { 3561 3562 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3563 ibmf_saa_impl_update_sa_address_err, 3564 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3565 "%s, ibmf_status = %d\n", tnf_string, msg, 3566 "Could not unpack sa hdr", tnf_int, ibmf_status, rv); 3567 3568 return; 3569 } 3570 3571 attr_id = b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID); 3572 if (attr_id != MAD_ATTR_ID_CLASSPORTINFO) { 3573 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3574 ibmf_saa_impl_update_sa_address_info_err, 3575 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3576 "%s, attrID = %x\n", tnf_string, msg, 3577 "Wrong attribute ID", tnf_int, ibmf_status, attr_id); 3578 3579 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3580 return; 3581 } 3582 rv = ibmf_saa_utils_unpack_payload( 3583 msgp->im_msgbufs_recv.im_bufs_cl_data, 3584 msgp->im_msgbufs_recv.im_bufs_cl_data_len, attr_id, &result, 3585 &length, sa_hdr->AttributeOffset, B_TRUE, KM_NOSLEEP); 3586 if (rv != IBMF_SUCCESS) { 3587 3588 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3589 ibmf_saa_impl_update_sa_address_err, 3590 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3591 "%s, ibmf_status = %d\n", tnf_string, msg, 3592 "Could not unpack payload", tnf_int, ibmf_status, rv); 3593 3594 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3595 return; 3596 } 3597 3598 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3599 3600 /* 3601 * Use the classportinfo contents to update the SA address info 3602 */ 3603 cpi = (ib_mad_classportinfo_t *)result; 3604 mutex_enter(&saa_portp->saa_pt_mutex); 3605 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = cpi->RedirectLID; 3606 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = cpi->RedirectQP; 3607 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = cpi->RedirectP_Key; 3608 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = cpi->RedirectQ_Key; 3609 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = cpi->RedirectSL; 3610 3611 saa_portp->saa_pt_redirect_active = B_TRUE; 3612 3613 if ((cpi->RedirectGID_hi != 0) || (cpi->RedirectGID_lo != 0)) { 3614 3615 mutex_exit(&saa_portp->saa_pt_mutex); 3616 ibt_status = ibt_query_hca_ports_byguid( 3617 saa_portp->saa_pt_node_guid, saa_portp->saa_pt_port_num, 3618 &ibt_pinfo, &nports, &size); 3619 if (ibt_status != IBT_SUCCESS) { 3620 3621 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3622 ibmf_saa_impl_update_sa_address_err, IBMF_TNF_TRACE, 3623 "", "ibmf_saa_impl_update_sa_address_info: " 3624 "%s, ibt_status = %d\n", tnf_string, msg, 3625 "Could not query hca port", 3626 tnf_int, ibt_status, ibt_status); 3627 3628 kmem_free(result, length); 3629 return; 3630 } 3631 3632 mutex_enter(&saa_portp->saa_pt_mutex); 3633 /* 3634 * Fill in global address info parameters 3635 * 3636 * NOTE: The HopLimit value is not specified through the 3637 * contents of ClassPortInfo. It may be possible to find 3638 * out the proper value to use even for SA beeing redirected 3639 * to another subnet. But we do only support redirect within 3640 * our local subnet 3641 */ 3642 gaddrp->ig_sender_gid.gid_prefix = 3643 ibt_pinfo->p_sgid_tbl[0].gid_prefix; 3644 gaddrp->ig_sender_gid.gid_guid = saa_portp->saa_pt_port_guid; 3645 gaddrp->ig_recver_gid.gid_prefix = cpi->RedirectGID_hi; 3646 gaddrp->ig_recver_gid.gid_guid = cpi->RedirectGID_lo; 3647 gaddrp->ig_flow_label = cpi->RedirectFL; 3648 gaddrp->ig_tclass = cpi->RedirectTC; 3649 gaddrp->ig_hop_limit = 0; 3650 3651 saa_portp->saa_pt_ibmf_msg_flags = 3652 IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 3653 3654 mutex_exit(&saa_portp->saa_pt_mutex); 3655 ibt_free_portinfo(ibt_pinfo, size); 3656 } else { 3657 saa_portp->saa_pt_ibmf_msg_flags = 0; 3658 mutex_exit(&saa_portp->saa_pt_mutex); 3659 } 3660 kmem_free(result, length); 3661 3662 /* 3663 * Update the address info of msgp with the new address parameters 3664 */ 3665 mutex_enter(&saa_portp->saa_pt_mutex); 3666 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr, 3667 sizeof (ibmf_addr_info_t)); 3668 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) { 3669 3670 msgp->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 3671 3672 bcopy(&saa_portp->saa_pt_ibmf_global_addr, 3673 &msgp->im_global_addr, sizeof (ibmf_global_addr_info_t)); 3674 } else { 3675 msgp->im_msg_flags = 0; 3676 } 3677 mutex_exit(&saa_portp->saa_pt_mutex); 3678 } 3679 3680 /* 3681 * ibmf_saa_impl_ibmf_unreg: 3682 */ 3683 static int 3684 ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp) 3685 { 3686 int ibmf_status; 3687 3688 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_start, 3689 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() enter\n"); 3690 3691 /* teardown async cb */ 3692 ibmf_status = ibmf_tear_down_async_cb(saa_portp->saa_pt_ibmf_handle, 3693 saa_portp->saa_pt_qp_handle, 0); 3694 if (ibmf_status != IBMF_SUCCESS) { 3695 3696 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3697 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3698 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3699 tnf_string, msg, "Could not tear down async cb", 3700 tnf_int, ibmf_status, ibmf_status); 3701 3702 goto bail; 3703 } 3704 3705 /* free qp */ 3706 ibmf_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle, 3707 &saa_portp->saa_pt_qp_handle, 0); 3708 3709 if (ibmf_status != IBMF_SUCCESS) { 3710 3711 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3712 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3713 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3714 tnf_string, msg, "Could not free queue pair", 3715 tnf_int, ibmf_status, ibmf_status); 3716 3717 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 1); 3718 goto bail; 3719 } 3720 3721 ibmf_status = ibmf_unregister(&saa_portp->saa_pt_ibmf_handle, 0); 3722 3723 if (ibmf_status != IBMF_SUCCESS) { 3724 3725 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3726 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3727 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3728 tnf_string, msg, "ibmf_unregister() failed", 3729 tnf_int, ibmf_status, ibmf_status); 3730 3731 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0); 3732 } 3733 3734 bail: 3735 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_end, 3736 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() exit\n"); 3737 3738 return (ibmf_status); 3739 } 3740