1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/ib/mgt/ibcm/ibcm_impl.h> 27 #include <sys/ib/ibtl/ibti.h> 28 #include <sys/ib/mgt/ibcm/ibcm_arp.h> 29 30 /* 31 * ibcm_ti.c 32 * These routines implement the Communication Manager's interfaces to IBTL. 33 */ 34 35 /* CM rc recycle task args structure definition */ 36 typedef struct ibcm_taskq_recycle_arg_s { 37 ibt_channel_hdl_t rc_chan; 38 ibt_cep_flags_t control; 39 uint8_t hca_port_num; 40 ibt_recycle_handler_t func; 41 void *arg; 42 } ibcm_taskq_recycle_arg_t; 43 44 _NOTE(READ_ONLY_DATA(ibcm_taskq_recycle_arg_s)) 45 46 static ibt_status_t ibcm_init_reply_addr(ibcm_hca_info_t *hcap, 47 ibcm_mad_addr_t *reply_addr, ibt_chan_open_args_t *chan_args, 48 ibt_chan_open_flags_t flags, ib_time_t *cm_pkt_lt, ib_lid_t prim_slid); 49 static void ibcm_process_abort_via_taskq(void *args); 50 static ibt_status_t ibcm_process_rc_recycle_ret(void *recycle_arg); 51 static ibt_status_t ibcm_process_join_mcg(void *taskq_arg); 52 static void ibcm_process_async_join_mcg(void *tq_arg); 53 54 ibt_status_t ibcm_get_node_rec(ibmf_saa_handle_t, sa_node_record_t *, 55 uint64_t c_mask, void *, size_t *); 56 57 static ibt_status_t ibcm_close_rc_channel(ibt_channel_hdl_t channel, 58 ibcm_state_data_t *statep, ibt_execution_mode_t mode); 59 60 /* Address Record management definitions */ 61 #define IBCM_DAPL_ATS_NAME "DAPL Address Translation Service" 62 #define IBCM_DAPL_ATS_SID 0x10000CE100415453ULL 63 #define IBCM_DAPL_ATS_NBYTES 16 64 ibcm_svc_info_t *ibcm_ar_svcinfop; 65 ibcm_ar_t *ibcm_ar_list; 66 67 /* 68 * Tunable parameter to turnoff the overriding of pi_path_mtu value. 69 * 1 By default override the path record's pi_path_mtu value to 70 * IB_MTU_1K for all RC channels. This is done only for the 71 * channels established on Tavor HCA and the path's pi_path_mtu 72 * is greater than IB_MTU_1K. 73 * 0 Do not override, use pi_path_mtu by default. 74 */ 75 int ibcm_override_path_mtu = 1; 76 77 #ifdef DEBUG 78 static void ibcm_print_reply_addr(ibt_channel_hdl_t channel, 79 ibcm_mad_addr_t *cm_reply_addr); 80 #endif 81 82 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibcm_port_info_s::{port_ibmf_hdl})) 83 84 /* access is controlled between ibcm_sm.c and ibcm_ti.c by CVs */ 85 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by CV", {ibt_rc_returns_t 86 ibt_ud_returns_t ibt_ap_returns_t ibt_ar_t})) 87 88 /* 89 * Typically, clients initialize these args in one api call, and use in 90 * another api 91 */ 92 _NOTE(SCHEME_PROTECTS_DATA("Expected usage of ibtl api by client", 93 {ibt_path_info_s ibt_cep_path_s ibt_adds_vect_s ibt_mcg_info_s ib_gid_s 94 ibt_ud_dest_attr_s ibt_ud_dest_s ibt_srv_data_s ibt_redirect_info_s})) 95 96 /* 97 * ibt_open_rc_channel() 98 * ibt_open_rc_channel opens a communication channel on the specified 99 * channel to the specified service. For connection service type qp's 100 * the CM initiates the CEP to establish the connection and transitions 101 * the QP/EEC to the "Ready to send" State modifying the QP/EEC's 102 * attributes as necessary. 103 * The implementation of this function assumes that alt path is different 104 * from primary path. It is assumed that the Path functions ensure that. 105 * 106 * RETURN VALUES: 107 * IBT_SUCCESS on success (or respective failure on error) 108 */ 109 ibt_status_t 110 ibt_open_rc_channel(ibt_channel_hdl_t channel, ibt_chan_open_flags_t flags, 111 ibt_execution_mode_t mode, ibt_chan_open_args_t *chan_args, 112 ibt_rc_returns_t *ret_args) 113 { 114 /* all fields that are related to REQ MAD formation */ 115 116 ib_pkey_t prim_pkey; 117 ib_lid_t primary_slid, alternate_slid; 118 ib_qpn_t local_qpn = 0; 119 ib_guid_t hca_guid; 120 ib_qkey_t local_qkey = 0; 121 ib_eecn_t local_eecn = 0; 122 ib_eecn_t remote_eecn = 0; 123 boolean_t primary_grh; 124 boolean_t alternate_grh = B_FALSE; 125 ib_lid_t base_lid; 126 ib_com_id_t local_comid; 127 ibmf_msg_t *ibmf_msg, *ibmf_msg_dreq; 128 ibcm_req_msg_t *req_msgp; 129 130 uint8_t rdma_in, rdma_out; 131 uint8_t cm_retries; 132 uint64_t local_cm_proc_time; /* In usec */ 133 uint8_t local_cm_resp_time; /* IB time */ 134 uint64_t remote_cm_resp_time; /* In usec */ 135 uint32_t starting_psn = 0; 136 137 /* CM path related fields */ 138 ibmf_handle_t ibmf_hdl; 139 ibcm_qp_list_t *cm_qp_entry; 140 ibcm_mad_addr_t cm_reply_addr; 141 142 uint8_t cm_pkt_lt; 143 144 /* Local args for ibtl/internal CM functions called within */ 145 ibt_status_t status; 146 ibcm_status_t lkup_status; 147 ibt_qp_query_attr_t qp_query_attr; 148 149 /* Other misc local args */ 150 ibt_priv_data_len_t len; 151 ibcm_hca_info_t *hcap; 152 ibcm_state_data_t *statep; 153 uint8_t port_no; 154 155 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel(chan %p, %X, %x, %p, %p)", 156 channel, flags, mode, chan_args, ret_args); 157 158 if (IBCM_INVALID_CHANNEL(channel)) { 159 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: invalid channel"); 160 return (IBT_CHAN_HDL_INVALID); 161 } 162 163 /* cm handler should always be specified */ 164 if (chan_args->oc_cm_handler == NULL) { 165 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 166 "CM handler is not be specified", channel); 167 return (IBT_INVALID_PARAM); 168 } 169 170 if (mode == IBT_NONBLOCKING) { 171 if (ret_args != NULL) { 172 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 173 " ret_args should be NULL when called in " 174 "non-blocking mode", channel); 175 return (IBT_INVALID_PARAM); 176 } 177 } else if (mode == IBT_BLOCKING) { 178 if (ret_args == NULL) { 179 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 180 " ret_args should be Non-NULL when called in " 181 "blocking mode", channel); 182 return (IBT_INVALID_PARAM); 183 } 184 if (ret_args->rc_priv_data_len > IBT_REP_PRIV_DATA_SZ) { 185 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 186 " private data length is too large", channel); 187 return (IBT_INVALID_PARAM); 188 } 189 if ((ret_args->rc_priv_data_len > 0) && 190 (ret_args->rc_priv_data == NULL)) { 191 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 192 " rc_priv_data_len > 0, but rc_priv_data NULL", 193 channel); 194 return (IBT_INVALID_PARAM); 195 } 196 } else { /* any other mode is not valid for ibt_open_rc_channel */ 197 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 198 "invalid mode %x specified", channel, mode); 199 return (IBT_INVALID_PARAM); 200 } 201 202 /* 203 * XXX: no support yet for ibt_chan_open_flags_t - IBT_OCHAN_DUP 204 */ 205 if (flags & IBT_OCHAN_DUP) { 206 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 207 "Unsupported Flags specified: 0x%X", channel, flags); 208 return (IBT_INVALID_PARAM); 209 } 210 211 if ((flags & IBT_OCHAN_REDIRECTED) && 212 (flags & IBT_OCHAN_PORT_REDIRECTED)) { 213 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 214 "Illegal to specify IBT_OCHAN_REDIRECTED and " 215 "IBT_OCHAN_PORT_REDIRECTED flags together", channel); 216 return (IBT_INVALID_PARAM); 217 } 218 219 if (((flags & IBT_OCHAN_REDIRECTED) && 220 (chan_args->oc_cm_redirect_info == NULL)) || 221 ((flags & IBT_OCHAN_PORT_REDIRECTED) && 222 (chan_args->oc_cm_cep_path == NULL))) { 223 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 224 "Redirect flag specified, but respective arg is NULL", 225 channel); 226 return (IBT_INVALID_PARAM); 227 } 228 229 if ((flags & IBT_OCHAN_REDIRECTED) && 230 (chan_args->oc_cm_redirect_info->rdi_dlid == 0) && 231 (chan_args->oc_cm_redirect_info->rdi_gid.gid_guid == 0)) { 232 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 233 "Either rdi_dlid or rdi_gid must be specified for" 234 " IBT_OCHAN_REDIRECTED", channel); 235 return (IBT_INVALID_PARAM); 236 } 237 238 /* primary dlid and hca_port_num should never be zero */ 239 port_no = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 240 241 if ((IBCM_PRIM_ADDS_VECT(chan_args).av_dlid == 0) && (port_no == 0)) { 242 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 243 "Primary Path's information is not valid", channel); 244 return (IBT_INVALID_PARAM); 245 } 246 247 /* validate SID */ 248 if (chan_args->oc_path->pi_sid == 0) { 249 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 250 "ERROR: Service ID in path information is 0", channel); 251 return (IBT_INVALID_PARAM); 252 } 253 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel: chan 0x%p SID %llX", 254 channel, chan_args->oc_path->pi_sid); 255 256 /* validate rnr_retry_cnt (enum has more than 3 bits) */ 257 if ((uint_t)chan_args->oc_path_rnr_retry_cnt > IBT_RNR_INFINITE_RETRY) { 258 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 259 "ERROR: oc_path_rnr_retry_cnt(%d) is out of range", 260 channel, chan_args->oc_path_rnr_retry_cnt); 261 return (IBT_INVALID_PARAM); 262 } 263 264 /* 265 * Ensure that client is not re-using a QP that is still associated 266 * with a statep 267 */ 268 IBCM_GET_CHAN_PRIVATE(channel, statep); 269 if (statep != NULL) { 270 IBCM_RELEASE_CHAN_PRIVATE(channel); 271 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 272 "Channel being re-used on active side", channel); 273 return (IBT_CHAN_IN_USE); 274 } 275 276 /* Get GUID from Channel */ 277 hca_guid = ibt_channel_to_hca_guid(channel); 278 279 /* validate QP's hca guid with that from primary path */ 280 if (hca_guid != chan_args->oc_path->pi_hca_guid) { 281 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 282 "GUID from Channel and primary path don't match", channel); 283 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 284 "Channel GUID %llX primary path GUID %llX", channel, 285 hca_guid, chan_args->oc_path->pi_hca_guid); 286 return (IBT_CHAN_HDL_INVALID); 287 } 288 289 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 290 "Local HCA GUID %llX", channel, hca_guid); 291 292 status = ibt_query_qp(channel, &qp_query_attr); 293 if (status != IBT_SUCCESS) { 294 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 295 "ibt_query_qp failed %d", channel, status); 296 return (status); 297 } 298 299 /* If client specified "no port change on QP" */ 300 if ((qp_query_attr.qp_info.qp_transport.rc.rc_path.cep_hca_port_num != 301 port_no) && (flags & IBT_OCHAN_PORT_FIXED)) { 302 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 303 "chan port %d and path port %d does not match", channel, 304 qp_query_attr.qp_info.qp_transport.rc.rc_path. \ 305 cep_hca_port_num, port_no); 306 return (IBT_INVALID_PARAM); 307 } 308 309 if (qp_query_attr.qp_info.qp_trans != IBT_RC_SRV) { 310 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 311 "Invalid Channel type: Applicable only to RC Channel", 312 channel); 313 return (IBT_CHAN_SRV_TYPE_INVALID); 314 } 315 316 /* Check if QP is in INIT state or not */ 317 if (qp_query_attr.qp_info.qp_state != IBT_STATE_INIT) { 318 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 319 "QP is not in INIT state %x", channel, 320 qp_query_attr.qp_info.qp_state); 321 return (IBT_CHAN_STATE_INVALID); 322 } 323 324 local_qpn = qp_query_attr.qp_qpn; 325 326 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p Active QPN 0x%x", 327 channel, local_qpn); 328 329 #ifdef NO_EEC_SUPPORT_YET 330 331 if (flags & IBT_OCHAN_RDC_EXISTS) { 332 ibt_eec_query_attr_t eec_query_attr; 333 334 local_qkey = qp_query_attr.qp_info.qp_transport.rd_qkey; 335 336 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: RD"); 337 338 status = ibt_query_eec(channel, &eec_query_attr); 339 if (status != IBT_SUCCESS) { 340 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 341 " ibt_query_eec failed %d", channel, status); 342 return (status); 343 } 344 local_eecn = eec_query_attr.eec_eecn; 345 } 346 347 #endif 348 if (chan_args->oc_path->pi_prim_pkt_lt > ibcm_max_ib_pkt_lt) { 349 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 350 "Huge PktLifeTime %d, Max is %d", channel, 351 chan_args->oc_path->pi_prim_pkt_lt, ibcm_max_ib_pkt_lt); 352 return (IBT_PATH_PKT_LT_TOO_HIGH); 353 } 354 355 /* If no HCA found return failure */ 356 if ((hcap = ibcm_find_hca_entry(hca_guid)) == NULL) { 357 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 358 "hcap is NULL. Probably hca is not in active state", 359 channel); 360 return (IBT_CHAN_HDL_INVALID); 361 } 362 363 rdma_out = chan_args->oc_rdma_ra_out; 364 rdma_in = chan_args->oc_rdma_ra_in; 365 366 if ((rdma_in > hcap->hca_max_rdma_in_qp) || 367 (rdma_out > hcap->hca_max_rdma_out_qp)) { 368 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 369 "rdma in %d/out %d values exceed hca limits(%d/%d)", 370 channel, rdma_in, rdma_out, hcap->hca_max_rdma_in_qp, 371 hcap->hca_max_rdma_out_qp); 372 ibcm_dec_hca_acc_cnt(hcap); 373 return (IBT_INVALID_PARAM); 374 } 375 376 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 377 "rdma_in %d rdma_out %d", channel, rdma_in, rdma_out); 378 379 status = ibt_get_port_state_byguid(hcap->hca_guid, port_no, 380 NULL, &base_lid); 381 if (status != IBT_SUCCESS) { 382 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 383 "primary port_num %d not active", channel, port_no); 384 ibcm_dec_hca_acc_cnt(hcap); 385 return (status); 386 } 387 388 /* Validate P_KEY Index */ 389 status = ibt_index2pkey_byguid(hcap->hca_guid, port_no, 390 IBCM_PRIM_CEP_PATH(chan_args).cep_pkey_ix, &prim_pkey); 391 if (status != IBT_SUCCESS) { 392 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 393 "Invalid Primary PKeyIx %x", channel, 394 IBCM_PRIM_CEP_PATH(chan_args).cep_pkey_ix); 395 ibcm_dec_hca_acc_cnt(hcap); 396 return (status); 397 } 398 399 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 400 "primary_port_num %d primary_pkey 0x%x", channel, port_no, 401 prim_pkey); 402 403 if ((hcap->hca_port_info[port_no - 1].port_ibmf_hdl == NULL) && 404 ((status = ibcm_hca_reinit_port(hcap, port_no - 1)) 405 != IBT_SUCCESS)) { 406 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 407 "ibmf reg or callback setup failed during re-initialize", 408 channel); 409 ibcm_dec_hca_acc_cnt(hcap); 410 return (status); 411 } 412 413 ibmf_hdl = hcap->hca_port_info[port_no - 1].port_ibmf_hdl; 414 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 415 "primary ibmf_hdl = 0x%p", channel, ibmf_hdl); 416 417 primary_slid = base_lid + IBCM_PRIM_ADDS_VECT(chan_args).av_src_path; 418 419 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: channel 0x%p " 420 "primary SLID = %x", channel, primary_slid); 421 422 /* check first if alternate path exists or not as it is OPTIONAL */ 423 if (IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num != 0) { 424 uint8_t alt_port_no; 425 426 alt_port_no = IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num; 427 428 if (chan_args->oc_path->pi_alt_pkt_lt > ibcm_max_ib_pkt_lt) { 429 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 430 "Huge Alt Pkt lt %d", channel, 431 chan_args->oc_path->pi_alt_pkt_lt); 432 ibcm_dec_hca_acc_cnt(hcap); 433 return (IBT_PATH_PKT_LT_TOO_HIGH); 434 } 435 436 if (port_no != alt_port_no) { 437 438 status = ibt_get_port_state_byguid(hcap->hca_guid, 439 alt_port_no, NULL, &base_lid); 440 if (status != IBT_SUCCESS) { 441 442 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 443 "chan 0x%p alt_port_num %d inactive %d", 444 channel, alt_port_no, status); 445 ibcm_dec_hca_acc_cnt(hcap); 446 return (status); 447 } 448 449 } 450 alternate_slid = 451 base_lid + IBCM_ALT_ADDS_VECT(chan_args).av_src_path; 452 453 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 454 "alternate SLID = %x", channel, alternate_slid); 455 } 456 457 /* 458 * only pkey needs to be zero'ed, because all other fields are set in 459 * in ibcm_init_reply_addr. But, let's bzero the complete struct for 460 * any future modifications. 461 */ 462 bzero(&cm_reply_addr, sizeof (cm_reply_addr)); 463 464 /* Initialize the MAD destination address in stored_reply_addr */ 465 if ((status = ibcm_init_reply_addr(hcap, &cm_reply_addr, chan_args, 466 flags, &cm_pkt_lt, primary_slid)) != IBT_SUCCESS) { 467 468 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 469 "ibcm_init_reply_addr failed status %d ", channel, status); 470 ibcm_dec_hca_acc_cnt(hcap); 471 return (status); 472 } 473 474 475 /* Initialize the pkey for CM MAD communication */ 476 if (cm_reply_addr.rcvd_addr.ia_p_key == 0) 477 cm_reply_addr.rcvd_addr.ia_p_key = prim_pkey; 478 479 #ifdef DEBUG 480 ibcm_print_reply_addr(channel, &cm_reply_addr); 481 #endif 482 483 /* Retrieve an ibmf qp for sending CM MADs */ 484 if ((cm_qp_entry = ibcm_find_qp(hcap, port_no, 485 cm_reply_addr.rcvd_addr.ia_p_key)) == NULL) { 486 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 487 "unable to allocate ibmf qp for CM MADs", channel); 488 ibcm_dec_hca_acc_cnt(hcap); 489 return (IBT_INSUFF_RESOURCE); 490 } 491 492 493 if (ibcm_alloc_comid(hcap, &local_comid) != IBCM_SUCCESS) { 494 ibcm_release_qp(cm_qp_entry); 495 ibcm_dec_hca_acc_cnt(hcap); 496 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 497 " Unable to allocate comid", channel); 498 return (IBT_INSUFF_KERNEL_RESOURCE); 499 } 500 501 /* allocate an IBMF mad buffer (REQ) */ 502 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg, 503 MAD_METHOD_SEND)) != IBT_SUCCESS) { 504 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 505 "chan 0x%p ibcm_alloc_out_msg failed", channel); 506 ibcm_release_qp(cm_qp_entry); 507 ibcm_free_comid(hcap, local_comid); 508 ibcm_dec_hca_acc_cnt(hcap); 509 return (status); 510 } 511 512 /* allocate an IBMF mad buffer (DREQ) */ 513 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg_dreq, 514 MAD_METHOD_SEND)) != IBT_SUCCESS) { 515 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 516 "chan 0x%p ibcm_alloc_out_msg failed", channel); 517 (void) ibcm_free_out_msg(ibmf_hdl, &ibmf_msg); 518 ibcm_release_qp(cm_qp_entry); 519 ibcm_free_comid(hcap, local_comid); 520 ibcm_dec_hca_acc_cnt(hcap); 521 return (status); 522 } 523 524 /* Init to Init, if QP's port does not match with path information */ 525 if (qp_query_attr.qp_info.qp_transport.rc.rc_path.cep_hca_port_num != 526 IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num) { 527 528 ibt_qp_info_t qp_info; 529 ibt_cep_modify_flags_t cep_flags; 530 531 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 532 "chan 0x%p chan port %d", channel, 533 qp_query_attr.qp_info.qp_transport.rc.rc_path.\ 534 cep_hca_port_num); 535 536 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 537 "chan 0x%p path port %d", channel, port_no); 538 539 bzero(&qp_info, sizeof (qp_info)); 540 /* For now, set it to RC type */ 541 542 qp_info.qp_trans = IBT_RC_SRV; 543 qp_info.qp_state = IBT_STATE_INIT; 544 qp_info.qp_transport.rc.rc_path.cep_hca_port_num = port_no; 545 546 cep_flags = IBT_CEP_SET_STATE | IBT_CEP_SET_PORT; 547 548 status = ibt_modify_qp(channel, cep_flags, &qp_info, NULL); 549 550 if (status != IBT_SUCCESS) { 551 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 552 "chan 0x%p ibt_modify_qp() = %d", channel, status); 553 ibcm_release_qp(cm_qp_entry); 554 ibcm_free_comid(hcap, local_comid); 555 ibcm_dec_hca_acc_cnt(hcap); 556 (void) ibcm_free_out_msg(ibmf_hdl, &ibmf_msg); 557 (void) ibcm_free_out_msg(ibmf_hdl, &ibmf_msg_dreq); 558 return (status); 559 } else 560 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 561 "chan 0x%p ibt_modify_qp() = %d", channel, status); 562 } 563 564 /* allocate ibcm_state_data_t before grabbing the WRITER lock */ 565 statep = kmem_zalloc(sizeof (ibcm_state_data_t), KM_SLEEP); 566 rw_enter(&hcap->hca_state_rwlock, RW_WRITER); 567 lkup_status = ibcm_lookup_msg(IBCM_OUTGOING_REQ, local_comid, 0, 0, 568 hcap, &statep); 569 rw_exit(&hcap->hca_state_rwlock); 570 571 /* CM should be seeing this for the first time */ 572 ASSERT(lkup_status == IBCM_LOOKUP_NEW); 573 574 /* Increment the hca's resource count */ 575 ibcm_inc_hca_res_cnt(hcap); 576 577 /* Once a resource created on hca, no need to hold the acc cnt */ 578 ibcm_dec_hca_acc_cnt(hcap); 579 580 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*statep)) 581 582 statep->timerid = 0; 583 statep->local_hca_guid = hca_guid; 584 statep->local_qpn = local_qpn; 585 statep->stored_reply_addr.cm_qp_entry = cm_qp_entry; 586 statep->prim_port = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 587 statep->alt_port = IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num; 588 589 590 /* Save "statep" as channel's CM private data. */ 591 statep->channel = channel; 592 IBCM_SET_CHAN_PRIVATE(statep->channel, statep); 593 594 statep->stored_msg = ibmf_msg; 595 statep->dreq_msg = ibmf_msg_dreq; 596 597 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*req_msgp)) 598 599 /* Start filling in the REQ MAD */ 600 req_msgp = (ibcm_req_msg_t *)IBCM_OUT_MSGP(statep->stored_msg); 601 req_msgp->req_local_comm_id = h2b32(local_comid); 602 req_msgp->req_svc_id = h2b64(chan_args->oc_path->pi_sid); 603 req_msgp->req_local_ca_guid = h2b64(hca_guid); 604 req_msgp->req_local_qkey = h2b32(local_qkey); /* for EEC/RD */ 605 606 /* Bytes 32-35 are req_local_qpn and req_off_resp_resources */ 607 req_msgp->req_local_qpn_plus = h2b32(local_qpn << 8 | rdma_in); 608 609 /* Bytes 36-39 are req_local_eec_no and req_off_initiator_depth */ 610 req_msgp->req_local_eec_no_plus = h2b32(local_eecn << 8 | rdma_out); 611 612 if (flags & IBT_OCHAN_REMOTE_CM_TM) 613 remote_cm_resp_time = chan_args->oc_remote_cm_time; 614 else 615 remote_cm_resp_time = ibcm_remote_response_time; 616 617 /* 618 * Bytes 40-43 - remote_eecn, remote_cm_resp_time, tran_type, 619 * IBT_CM_FLOW_CONTROL is always set by default. 620 */ 621 req_msgp->req_remote_eecn_plus = h2b32( 622 remote_eecn << 8 | (ibt_usec2ib(remote_cm_resp_time) & 0x1f) << 3 | 623 IBT_RC_SRV << 1 | IBT_CM_FLOW_CONTROL); 624 625 if (flags & IBT_OCHAN_LOCAL_CM_TM) 626 local_cm_proc_time = chan_args->oc_local_cm_time; 627 else 628 local_cm_proc_time = ibcm_local_processing_time; 629 630 local_cm_resp_time = ibt_usec2ib(local_cm_proc_time + 631 2 * ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt) + 632 ibcm_sw_delay); 633 634 /* save retry count */ 635 statep->cep_retry_cnt = chan_args->oc_path_retry_cnt; 636 637 if (flags & IBT_OCHAN_STARTING_PSN) 638 starting_psn = chan_args->oc_starting_psn; 639 640 if (local_cm_resp_time > 0x1f) 641 local_cm_resp_time = 0x1f; 642 643 /* Bytes 44-47 are req_starting_psn, local_cm_resp_time and retry_cnt */ 644 req_msgp->req_starting_psn_plus = h2b32(starting_psn << 8 | 645 local_cm_resp_time << 3 | statep->cep_retry_cnt); 646 647 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 648 "Prim Pkt lt (IB time) 0x%x", channel, 649 chan_args->oc_path->pi_prim_pkt_lt); 650 651 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 652 "local_cm_proc_time(usec) %d ", channel, local_cm_proc_time); 653 654 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 655 "local_cm_resp_time(ib_time) %d", channel, local_cm_resp_time); 656 657 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 658 "remote_cm_resp_time (usec) %d", channel, remote_cm_resp_time); 659 660 statep->starting_psn = starting_psn; 661 662 /* Pkey - bytes 48-49 */ 663 req_msgp->req_part_key = h2b16(prim_pkey); 664 665 if (flags & IBT_OCHAN_CM_RETRY) 666 cm_retries = chan_args->oc_cm_retry_cnt; 667 else 668 cm_retries = ibcm_max_retries; 669 670 statep->max_cm_retries = statep->remaining_retry_cnt = cm_retries; 671 req_msgp->req_max_cm_retries_plus = statep->max_cm_retries << 4; 672 673 /* 674 * Check whether SRQ is associated with this Channel, if yes, then 675 * set the SRQ Exists bit in the REQ. 676 */ 677 if (qp_query_attr.qp_srq != NULL) { 678 req_msgp->req_max_cm_retries_plus |= (1 << 3); 679 } 680 681 /* 682 * By default on Tavor, we override the PathMTU to 1K. 683 * To turn this off, set ibcm_override_path_mtu = 0. 684 */ 685 if (ibcm_override_path_mtu && IBCM_IS_HCA_TAVOR(hcap) && 686 (chan_args->oc_path->pi_path_mtu > IB_MTU_1K)) { 687 req_msgp->req_mtu_plus = IB_MTU_1K << 4 | 688 chan_args->oc_path_rnr_retry_cnt; 689 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel: chan 0x%p PathMTU" 690 " overridden to IB_MTU_1K(%d) from %d", channel, IB_MTU_1K, 691 chan_args->oc_path->pi_path_mtu); 692 } else 693 req_msgp->req_mtu_plus = chan_args->oc_path->pi_path_mtu << 4 | 694 chan_args->oc_path_rnr_retry_cnt; 695 696 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p CM retry cnt %d" 697 " staring PSN %x", channel, cm_retries, starting_psn); 698 699 700 #ifdef NO_EEC_SUPPORT_YET 701 if (flags & IBT_OCHAN_RDC_EXISTS) 702 req_msgp->req_mtu_plus |= 8; 703 #endif 704 705 /* Initialize the "primary" port stuff next - bytes 52-95 */ 706 req_msgp->req_primary_l_port_lid = h2b16(primary_slid); 707 req_msgp->req_primary_r_port_lid = 708 h2b16(IBCM_PRIM_ADDS_VECT(chan_args).av_dlid); 709 req_msgp->req_primary_l_port_gid.gid_prefix = 710 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_sgid.gid_prefix); 711 req_msgp->req_primary_l_port_gid.gid_guid = 712 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_sgid.gid_guid); 713 req_msgp->req_primary_r_port_gid.gid_prefix = 714 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_prefix); 715 req_msgp->req_primary_r_port_gid.gid_guid = 716 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid); 717 primary_grh = IBCM_PRIM_ADDS_VECT(chan_args).av_send_grh; 718 719 statep->remote_hca_guid = /* not correct, but helpful for debugging */ 720 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid; 721 722 /* Bytes 88-91 - primary_flowlbl, and primary_srate */ 723 req_msgp->req_primary_flow_label_plus = 724 h2b32(((primary_grh == B_TRUE) ? 725 (IBCM_PRIM_ADDS_VECT(chan_args).av_flow << 12) : 0) | 726 IBCM_PRIM_ADDS_VECT(chan_args).av_srate); 727 req_msgp->req_primary_traffic_class = (primary_grh == B_TRUE) ? 728 IBCM_PRIM_ADDS_VECT(chan_args).av_tclass : 0; 729 req_msgp->req_primary_hop_limit = (primary_grh == B_TRUE) ? 730 IBCM_PRIM_ADDS_VECT(chan_args).av_hop : 1; 731 req_msgp->req_primary_sl_plus = 732 IBCM_PRIM_ADDS_VECT(chan_args).av_srvl << 4 | 733 ((primary_grh == B_TRUE) ? 0 : 8); 734 735 req_msgp->req_primary_localtime_plus = 736 ibt_usec2ib((2 * ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt)) + 737 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 738 739 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan %p statep %p", 740 channel, statep); 741 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 742 "active hca_ack_delay (usec) %d", channel, 743 req_msgp->req_primary_localtime_plus); 744 745 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 746 "Sent primary cep timeout (IB Time) %d", channel, 747 hcap->hca_ack_delay); 748 749 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p prim_dlid %x ", 750 channel, IBCM_PRIM_ADDS_VECT(chan_args).av_dlid); 751 752 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 753 "prim GID %llX:%llX", channel, 754 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_prefix, 755 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid); 756 757 /* Initialize the "alternate" port stuff - optional */ 758 if (chan_args->oc_path->pi_alt_cep_path.cep_hca_port_num != 0) { 759 ib_gid_t tmp_gid; 760 761 req_msgp->req_alt_l_port_lid = h2b16(alternate_slid); 762 req_msgp->req_alt_r_port_lid = 763 h2b16(IBCM_ALT_ADDS_VECT(chan_args).av_dlid); 764 /* 765 * doing all this as req_alt_r/l_port_gid is at offset 766 * 100, 116 which is not divisible by 8 767 */ 768 769 tmp_gid.gid_prefix = 770 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_prefix); 771 tmp_gid.gid_guid = 772 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_guid); 773 bcopy(&tmp_gid, &req_msgp->req_alt_r_port_gid[0], 774 sizeof (ib_gid_t)); 775 tmp_gid.gid_prefix = 776 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_sgid.gid_prefix); 777 tmp_gid.gid_guid = 778 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_sgid.gid_guid); 779 780 bcopy(&tmp_gid, &req_msgp->req_alt_l_port_gid[0], 781 sizeof (ib_gid_t)); 782 alternate_grh = IBCM_ALT_ADDS_VECT(chan_args).av_send_grh; 783 784 /* Bytes 132-135 - alternate_flow_label, and alternate srate */ 785 req_msgp->req_alt_flow_label_plus = h2b32( 786 (((alternate_grh == B_TRUE) ? 787 (IBCM_ALT_ADDS_VECT(chan_args).av_flow << 12) : 0) | 788 IBCM_ALT_ADDS_VECT(chan_args).av_srate)); 789 req_msgp->req_alt_traffic_class = (alternate_grh == B_TRUE) ? 790 IBCM_ALT_ADDS_VECT(chan_args).av_tclass : 0; 791 req_msgp->req_alt_hop_limit = (alternate_grh == B_TRUE) ? 792 IBCM_ALT_ADDS_VECT(chan_args).av_hop : 1; 793 req_msgp->req_alt_sl_plus = 794 IBCM_ALT_ADDS_VECT(chan_args).av_srvl << 4 | 795 ((alternate_grh == B_TRUE) ? 0 : 8); 796 req_msgp->req_alt_localtime_plus = ibt_usec2ib((2 * 797 ibt_ib2usec(chan_args->oc_path->pi_alt_pkt_lt)) + 798 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 799 800 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 801 "alt_dlid %x ", channel, 802 IBCM_ALT_ADDS_VECT(chan_args).av_dlid); 803 804 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 805 "alt GID %llX:%llX", channel, 806 IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_prefix, 807 IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_guid); 808 } 809 810 len = min(chan_args->oc_priv_data_len, IBT_REQ_PRIV_DATA_SZ); 811 if ((len > 0) && chan_args->oc_priv_data) 812 bcopy(chan_args->oc_priv_data, req_msgp->req_private_data, len); 813 814 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*req_msgp)) 815 816 /* return_data is filled up in the state machine code */ 817 if (ret_args != NULL) { 818 statep->open_return_data = ret_args; 819 } 820 821 /* initialize some statep fields here */ 822 statep->mode = IBCM_ACTIVE_MODE; 823 statep->hcap = hcap; 824 825 statep->cm_handler = chan_args->oc_cm_handler; 826 statep->state_cm_private = chan_args->oc_cm_clnt_private; 827 828 statep->pkt_life_time = 829 ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt); 830 831 statep->timer_value = ibt_ib2usec(ibt_usec2ib( 832 2 * ibt_ib2usec(cm_pkt_lt) + remote_cm_resp_time)); 833 834 /* Initialize statep->stored_reply_addr */ 835 statep->stored_reply_addr.ibmf_hdl = ibmf_hdl; 836 837 /* Initialize stored reply addr fields */ 838 statep->stored_reply_addr.grh_hdr = cm_reply_addr.grh_hdr; 839 statep->stored_reply_addr.rcvd_addr = cm_reply_addr.rcvd_addr; 840 statep->stored_reply_addr.grh_exists = cm_reply_addr.grh_exists; 841 statep->stored_reply_addr.port_num = cm_reply_addr.port_num; 842 843 /* 844 * The IPD on local/active side is calculated by path functions, 845 * hence available in the args of ibt_open_rc_channel 846 */ 847 statep->local_srate = IBCM_PRIM_ADDS_VECT(chan_args).av_srate; 848 statep->local_alt_srate = IBCM_ALT_ADDS_VECT(chan_args).av_srate; 849 850 /* Store the source path bits for primary and alt paths */ 851 statep->prim_src_path_bits = IBCM_PRIM_ADDS_VECT(chan_args).av_src_path; 852 statep->alt_src_path_bits = IBCM_ALT_ADDS_VECT(chan_args).av_src_path; 853 854 statep->open_flow = 1; 855 statep->open_done = B_FALSE; 856 statep->state = statep->timer_stored_state = IBCM_STATE_REQ_SENT; 857 IBCM_REF_CNT_INCR(statep); /* Decremented before return */ 858 IBCM_REF_CNT_INCR(statep); /* Decremented after REQ is posted */ 859 statep->send_mad_flags |= IBCM_REQ_POST_BUSY; 860 861 IBCM_OUT_HDRP(statep->stored_msg)->AttributeID = 862 h2b16(IBCM_INCOMING_REQ + IBCM_ATTR_BASE_ID); 863 864 IBCM_OUT_HDRP(statep->stored_msg)->TransactionID = 865 h2b64(ibcm_generate_tranid(IBCM_INCOMING_REQ, statep->local_comid, 866 0)); 867 868 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*statep)) 869 870 ibcm_open_enqueue(statep); 871 872 mutex_enter(&statep->state_mutex); 873 874 if (mode == IBT_BLOCKING) { 875 876 /* wait for REQ/REP/RTU */ 877 while (statep->open_done != B_TRUE) { 878 cv_wait(&statep->block_client_cv, &statep->state_mutex); 879 } 880 881 /* 882 * In the case that open_channel() fails because of a 883 * REJ or timeout, change retval to IBT_CM_FAILURE 884 */ 885 if (statep->open_return_data->rc_status != IBT_CM_ACCEPT) 886 status = IBT_CM_FAILURE; 887 888 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel: chan 0x%p " 889 "ret status %d cm status %d", channel, status, 890 statep->open_return_data->rc_status); 891 } 892 893 /* decrement the ref-count before leaving here */ 894 IBCM_REF_CNT_DECR(statep); 895 896 mutex_exit(&statep->state_mutex); 897 898 IBTF_DPRINTF_L4(cmlog, "ibt_open_rc_channel: chan 0x%p done", channel); 899 return (status); 900 } 901 902 /* 903 * ibcm_init_reply_addr: 904 * 905 * The brief description of functionality below. 906 * 907 * For IBT_OCHAN_PORT_REDIRECTED (ie., port redirected case): 908 * Build CM path from chan_args->oc_cm_cep_path 909 * Set CM pkt lt (ie.,life time) to chan_args->oc_cm_pkt_lt 910 * 911 * For IBT_OCHAN_REDIRECTED (ie., port and CM redirected case): 912 * If Redirect LID is specified, 913 * If Redirect GID is not specified or specified to be on the same 914 * subnet, then 915 * Build CM path from chan_args->oc_cm_redirect_info 916 * Set CM pkt lt to subnet timeout 917 * Else (ie., GID specified, but on a different subnet) 918 * Do a path lookup to build CM Path and set CM pkt lt 919 * 920 */ 921 static ibt_status_t 922 ibcm_init_reply_addr(ibcm_hca_info_t *hcap, ibcm_mad_addr_t *reply_addr, 923 ibt_chan_open_args_t *chan_args, ibt_chan_open_flags_t flags, 924 ib_time_t *cm_pkt_lt, ib_lid_t prim_slid) 925 { 926 ibt_adds_vect_t *cm_adds; 927 ibt_path_info_t path; 928 boolean_t cm_grh; 929 ibt_status_t status; 930 931 IBTF_DPRINTF_L5(cmlog, "ibcm_init_reply_addr:"); 932 933 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*reply_addr)) 934 935 /* 936 * sending side CM lid/gid/port num are not based on any redirect 937 * params. These values are set to primary RC path lid/gid/port num. 938 * In the future, these values can be set based on framework policy 939 * decisions ensuring reachability. 940 */ 941 reply_addr->grh_hdr.ig_sender_gid = 942 IBCM_PRIM_ADDS_VECT(chan_args).av_sgid; 943 reply_addr->rcvd_addr.ia_local_lid = prim_slid; 944 reply_addr->port_num = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 945 946 if (flags & IBT_OCHAN_PORT_REDIRECTED) { 947 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: " 948 "IBT_OCHAN_PORT_REDIRECTED specified"); 949 950 status = ibt_index2pkey_byguid(hcap->hca_guid, 951 chan_args->oc_cm_cep_path->cep_hca_port_num, 952 chan_args->oc_cm_cep_path->cep_pkey_ix, 953 &reply_addr->rcvd_addr.ia_p_key); 954 955 if (status != IBT_SUCCESS) { 956 IBTF_DPRINTF_L2(cmlog, "ibcm_init_rely_addr: Invalid " 957 "CM PKeyIx %x port_num %x", 958 chan_args->oc_cm_cep_path->cep_pkey_ix, 959 chan_args->oc_cm_cep_path->cep_hca_port_num); 960 return (status); 961 } 962 963 cm_adds = &(chan_args->oc_cm_cep_path->cep_adds_vect); 964 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: dlid = %x", 965 cm_adds->av_dlid); 966 967 reply_addr->rcvd_addr.ia_q_key = IB_GSI_QKEY; 968 reply_addr->rcvd_addr.ia_remote_qno = 1; 969 *cm_pkt_lt = chan_args->oc_cm_pkt_lt; 970 971 } else if (flags & IBT_OCHAN_REDIRECTED) { 972 ibt_redirect_info_t *redirect_info; 973 ibt_hca_portinfo_t *port_infop; 974 uint_t psize, nports; 975 976 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: " 977 "IBT_OCHAN_REDIRECTED specified"); 978 979 redirect_info = chan_args->oc_cm_redirect_info; 980 981 if ((redirect_info->rdi_gid.gid_prefix == 0) || 982 (redirect_info->rdi_gid.gid_guid == 0)) { 983 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 984 "ERROR: Re-direct GID value NOT Provided."); 985 return (IBT_INVALID_PARAM); 986 } 987 988 /* As per spec definition 1.1, it's always IB_GSI_QKEY */ 989 reply_addr->rcvd_addr.ia_q_key = redirect_info->rdi_qkey; 990 reply_addr->rcvd_addr.ia_remote_qno = redirect_info->rdi_qpn; 991 reply_addr->rcvd_addr.ia_p_key = redirect_info->rdi_pkey; 992 993 /* 994 * if LID is non-zero in classportinfo then use classportinfo 995 * fields to form CM MAD destination address. 996 */ 997 if (redirect_info->rdi_dlid != 0) { 998 status = ibtl_cm_query_hca_ports_byguid(hcap->hca_guid, 999 reply_addr->port_num, &port_infop, &nports, &psize); 1000 if ((status != IBT_SUCCESS) || (nports == 0)) { 1001 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 1002 "Query Ports Failed: %d", status); 1003 return (status); 1004 } else if (port_infop->p_subnet_timeout > 1005 ibcm_max_ib_pkt_lt) { 1006 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 1007 "large subnet timeout %x port_no %x", 1008 port_infop->p_subnet_timeout, 1009 reply_addr->port_num); 1010 ibt_free_portinfo(port_infop, psize); 1011 return (IBT_PATH_PKT_LT_TOO_HIGH); 1012 } else { 1013 IBTF_DPRINTF_L3(cmlog, "ibcm_init_reply_addr: " 1014 "subnet timeout %x port_no %x", 1015 port_infop->p_subnet_timeout, 1016 reply_addr->port_num); 1017 1018 *cm_pkt_lt = 1019 ibt_ib2usec(min(ibcm_max_ib_mad_pkt_lt, 1020 port_infop->p_subnet_timeout)); 1021 1022 ibt_free_portinfo(port_infop, psize); 1023 } 1024 1025 reply_addr->rcvd_addr.ia_remote_lid = 1026 redirect_info->rdi_dlid; 1027 reply_addr->rcvd_addr.ia_service_level = 1028 redirect_info->rdi_sl; 1029 reply_addr->grh_exists = B_TRUE; 1030 reply_addr->grh_hdr.ig_recver_gid = 1031 redirect_info->rdi_gid; 1032 reply_addr->grh_hdr.ig_tclass = 1033 redirect_info->rdi_tclass; 1034 reply_addr->grh_hdr.ig_flow_label = 1035 redirect_info->rdi_flow; 1036 1037 /* Classportinfo doesn't have hoplimit field */ 1038 reply_addr->grh_hdr.ig_hop_limit = 1; 1039 return (IBT_SUCCESS); 1040 1041 } else { 1042 ibt_path_attr_t path_attr; 1043 ib_gid_t path_dgid[1]; 1044 1045 /* 1046 * If GID is specified, and LID is zero in classportinfo 1047 * do a path lookup using specified GID, Pkey, 1048 * in classportinfo 1049 */ 1050 1051 bzero(&path_attr, sizeof (path_attr)); 1052 1053 path_attr.pa_dgids = &path_dgid[0]; 1054 path_attr.pa_dgids[0] = redirect_info->rdi_gid; 1055 1056 /* 1057 * use reply_addr below, as sender_gid in reply_addr 1058 * may have been set above based on some policy decision 1059 * for originating end point for CM MADs above 1060 */ 1061 path_attr.pa_sgid = reply_addr->grh_hdr.ig_sender_gid; 1062 path_attr.pa_num_dgids = 1; 1063 path_attr.pa_pkey = redirect_info->rdi_pkey; 1064 1065 if ((status = ibt_get_paths(ibcm_ibt_handle, 1066 IBT_PATH_PKEY, &path_attr, 1, &path, NULL)) != 1067 IBT_SUCCESS) 1068 return (status); 1069 1070 /* Initialize cm_adds */ 1071 cm_adds = &path.pi_prim_cep_path.cep_adds_vect; 1072 *cm_pkt_lt = path.pi_prim_pkt_lt; 1073 } 1074 1075 } else { /* cm_pkey initialized in ibt_open_rc_channel */ 1076 reply_addr->rcvd_addr.ia_q_key = IB_GSI_QKEY; 1077 reply_addr->rcvd_addr.ia_remote_qno = 1; 1078 *cm_pkt_lt = chan_args->oc_path->pi_prim_pkt_lt; 1079 cm_adds = &(IBCM_PRIM_ADDS_VECT(chan_args)); 1080 } 1081 1082 1083 cm_grh = cm_adds->av_send_grh; 1084 reply_addr->grh_exists = cm_grh; 1085 1086 reply_addr->rcvd_addr.ia_remote_lid = 1087 cm_adds->av_dlid; 1088 reply_addr->grh_hdr.ig_recver_gid = 1089 cm_adds->av_dgid; 1090 reply_addr->grh_hdr.ig_flow_label = 1091 cm_adds->av_flow & IB_GRH_FLOW_LABEL_MASK; 1092 reply_addr->grh_hdr.ig_tclass = 1093 (cm_grh == B_TRUE) ? cm_adds->av_tclass : 0; 1094 reply_addr->grh_hdr.ig_hop_limit = 1095 (cm_grh == B_TRUE) ? cm_adds->av_hop : 1; 1096 reply_addr->rcvd_addr.ia_service_level = 1097 cm_adds->av_srvl; 1098 1099 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*reply_addr)) 1100 1101 return (IBT_SUCCESS); 1102 } 1103 1104 1105 /* 1106 * ibt_prime_close_rc_channel() 1107 * It allocates resources required for close channel operation, so 1108 * ibt_close_rc_channel can be called from interrupt routine. 1109 * 1110 * INPUTS: 1111 * channel The address of an ibt_channel_t struct that 1112 * specifies the channel to open. 1113 * 1114 * RETURN VALUES: 1115 * IBT_SUCCESS on success(or respective failure on error) 1116 * 1117 * Clients are typically expected to call this function in established state 1118 */ 1119 ibt_status_t 1120 ibt_prime_close_rc_channel(ibt_channel_hdl_t channel) 1121 { 1122 ibcm_state_data_t *statep; 1123 ibt_status_t status = IBT_SUCCESS; 1124 1125 IBTF_DPRINTF_L3(cmlog, "ibt_prime_close_rc_channel(%p)", channel); 1126 1127 /* validate channel, first */ 1128 if (IBCM_INVALID_CHANNEL(channel)) { 1129 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1130 "invalid channel", channel); 1131 return (IBT_CHAN_HDL_INVALID); 1132 } 1133 1134 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 1135 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1136 "Invalid Channel type: Applicable only to RC Channel", 1137 channel); 1138 return (IBT_CHAN_SRV_TYPE_INVALID); 1139 } 1140 1141 /* get the statep */ 1142 IBCM_GET_CHAN_PRIVATE(channel, statep); 1143 1144 /* 1145 * This can happen, if the statep is already gone by a DREQ from 1146 * the remote side 1147 */ 1148 1149 if (statep == NULL) { 1150 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1151 "statep NULL", channel); 1152 return (IBT_SUCCESS); 1153 } 1154 1155 mutex_enter(&statep->state_mutex); 1156 IBCM_RELEASE_CHAN_PRIVATE(channel); 1157 if (statep->state != IBCM_STATE_ESTABLISHED) { 1158 mutex_exit(&statep->state_mutex); 1159 return (IBT_CHAN_STATE_INVALID); 1160 } 1161 IBCM_REF_CNT_INCR(statep); 1162 IBTF_DPRINTF_L4(cmlog, "ibt_prime_close_rc_channel: chan 0x%p statep %p" 1163 " state %x", channel, statep, statep->state); 1164 mutex_exit(&statep->state_mutex); 1165 1166 /* clients could pre-allocate dreq mad, even before connection est */ 1167 if (statep->dreq_msg == NULL) 1168 status = ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl, 1169 &statep->dreq_msg, MAD_METHOD_SEND); 1170 1171 mutex_enter(&statep->state_mutex); 1172 IBCM_REF_CNT_DECR(statep); 1173 mutex_exit(&statep->state_mutex); 1174 1175 if (status != IBT_SUCCESS) { 1176 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1177 "ibcm_alloc_out_msg failed ", channel); 1178 return (status); 1179 } 1180 1181 /* If this message isn't seen then ibt_prime_close_rc_channel failed */ 1182 IBTF_DPRINTF_L5(cmlog, "ibt_prime_close_rc_channel: chan 0x%p done", 1183 channel); 1184 1185 return (IBT_SUCCESS); 1186 } 1187 1188 /* 1189 * ibt_close_rc_channel() 1190 * It closes an established channel. 1191 * 1192 * RETURN VALUES: 1193 * IBT_SUCCESS on success(or respective failure on error) 1194 */ 1195 ibt_status_t 1196 ibt_close_rc_channel(ibt_channel_hdl_t channel, ibt_execution_mode_t mode, 1197 void *priv_data, ibt_priv_data_len_t priv_data_len, uint8_t *ret_status, 1198 void *ret_priv_data, ibt_priv_data_len_t *ret_priv_data_len_p) 1199 { 1200 ibcm_state_data_t *statep; 1201 1202 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel(%p, %x, %p, %d, %p)", 1203 channel, mode, priv_data, priv_data_len, 1204 (ret_priv_data_len_p == NULL) ? 0 : *ret_priv_data_len_p); 1205 1206 /* validate channel, first */ 1207 if (IBCM_INVALID_CHANNEL(channel)) { 1208 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1209 "invalid channel", channel); 1210 return (IBT_CHAN_HDL_INVALID); 1211 } 1212 1213 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 1214 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1215 "Invalid Channel type: Applicable only to RC Channel", 1216 channel); 1217 return (IBT_CHAN_SRV_TYPE_INVALID); 1218 } 1219 1220 if (mode == IBT_BLOCKING) { 1221 /* valid only for BLOCKING MODE */ 1222 if ((ret_priv_data_len_p != NULL) && 1223 (*ret_priv_data_len_p > IBT_DREP_PRIV_DATA_SZ)) { 1224 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p" 1225 " private data len %d is too large", channel, 1226 *ret_priv_data_len_p); 1227 return (IBT_INVALID_PARAM); 1228 } 1229 } else if ((mode != IBT_NONBLOCKING) && (mode != IBT_NOCALLBACKS)) { 1230 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1231 "invalid mode %x specified", channel, mode); 1232 return (IBT_INVALID_PARAM); 1233 } 1234 1235 if (ibtl_cm_is_chan_closing(channel) || 1236 ibtl_cm_is_chan_closed(channel)) { 1237 if (ret_status) 1238 *ret_status = IBT_CM_CLOSED_ALREADY; 1239 1240 /* No private data to return to the client */ 1241 if (ret_priv_data_len_p != NULL) 1242 *ret_priv_data_len_p = 0; 1243 1244 if ((mode == IBT_BLOCKING) || 1245 (mode == IBT_NOCALLBACKS)) { 1246 IBCM_GET_CHAN_PRIVATE(channel, statep); 1247 if (statep == NULL) 1248 return (IBT_SUCCESS); 1249 mutex_enter(&statep->state_mutex); 1250 IBCM_RELEASE_CHAN_PRIVATE(channel); 1251 IBCM_REF_CNT_INCR(statep); 1252 while (statep->close_done != B_TRUE) 1253 cv_wait(&statep->block_client_cv, 1254 &statep->state_mutex); 1255 IBCM_REF_CNT_DECR(statep); 1256 mutex_exit(&statep->state_mutex); 1257 } 1258 1259 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel: chan 0x%p " 1260 "already marked for closing", channel); 1261 1262 return (IBT_SUCCESS); 1263 } 1264 1265 /* get the statep */ 1266 IBCM_GET_CHAN_PRIVATE(channel, statep); 1267 if (statep == NULL) { 1268 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1269 "statep NULL", channel); 1270 return (IBT_CHAN_STATE_INVALID); 1271 } 1272 1273 mutex_enter(&statep->state_mutex); 1274 1275 if (statep->dreq_msg == NULL) { 1276 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1277 "Fatal Error: dreq_msg is NULL", channel); 1278 IBCM_RELEASE_CHAN_PRIVATE(channel); 1279 mutex_exit(&statep->state_mutex); 1280 return (IBT_CHAN_STATE_INVALID); 1281 } 1282 1283 if ((ret_priv_data == NULL) || (ret_priv_data_len_p == NULL)) { 1284 statep->close_ret_priv_data = NULL; 1285 statep->close_ret_priv_data_len = NULL; 1286 } else { 1287 statep->close_ret_priv_data = ret_priv_data; 1288 statep->close_ret_priv_data_len = ret_priv_data_len_p; 1289 } 1290 1291 priv_data_len = min(priv_data_len, IBT_DREQ_PRIV_DATA_SZ); 1292 if ((priv_data != NULL) && (priv_data_len > 0)) { 1293 bcopy(priv_data, ((ibcm_dreq_msg_t *) 1294 IBCM_OUT_MSGP(statep->dreq_msg))->dreq_private_data, 1295 priv_data_len); 1296 } 1297 statep->close_ret_status = ret_status; 1298 1299 IBCM_RELEASE_CHAN_PRIVATE(channel); 1300 IBCM_REF_CNT_INCR(statep); 1301 1302 if (mode != IBT_NONBLOCKING) { 1303 return (ibcm_close_rc_channel(channel, statep, mode)); 1304 } 1305 1306 /* IBT_NONBLOCKING */ 1307 ibcm_close_enqueue(statep); 1308 mutex_exit(&statep->state_mutex); 1309 1310 return (IBT_SUCCESS); 1311 } 1312 1313 void 1314 ibcm_close_start(ibcm_state_data_t *statep) 1315 { 1316 mutex_enter(&statep->state_mutex); 1317 (void) ibcm_close_rc_channel(statep->channel, statep, IBT_NONBLOCKING); 1318 } 1319 1320 static 1321 ibt_status_t 1322 ibcm_close_rc_channel(ibt_channel_hdl_t channel, ibcm_state_data_t *statep, 1323 ibt_execution_mode_t mode) 1324 { 1325 ibcm_hca_info_t *hcap; 1326 1327 _NOTE(LOCK_RELEASED_AS_SIDE_EFFECT(&statep->state_mutex)); 1328 ASSERT(MUTEX_HELD(&statep->state_mutex)); 1329 1330 IBTF_DPRINTF_L3(cmlog, "ibcm_close_rc_channel: chan 0x%p statep %p", 1331 channel, statep); 1332 1333 hcap = statep->hcap; 1334 1335 /* HCA must have been in active state. If not, it's a client bug */ 1336 if (!IBCM_ACCESS_HCA_OK(hcap)) { 1337 IBTF_DPRINTF_L2(cmlog, "ibcm_close_rc_channel: chan 0x%p " 1338 "hcap 0x%p not active", channel, hcap); 1339 IBCM_REF_CNT_DECR(statep); 1340 mutex_exit(&statep->state_mutex); 1341 return (IBT_CHAN_HDL_INVALID); 1342 } 1343 1344 if (statep->state == IBCM_STATE_TRANSIENT_ESTABLISHED) { 1345 while (statep->cep_in_rts == IBCM_BLOCK) 1346 cv_wait(&statep->block_mad_cv, &statep->state_mutex); 1347 } 1348 1349 /* Do TRANSIENT_DREQ check after TRANSIENT_ESTABLISHED check */ 1350 while (statep->state == IBCM_STATE_TRANSIENT_DREQ_SENT) 1351 cv_wait(&statep->block_mad_cv, &statep->state_mutex); 1352 1353 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: chan 0x%p " 1354 "connection state is %x", channel, statep->state); 1355 1356 /* If state is in pre-established states, abort the connection est */ 1357 if (statep->state != IBCM_STATE_ESTABLISHED) { 1358 statep->cm_retries++; /* ensure connection trace is dumped */ 1359 1360 /* No DREP private data possible */ 1361 if (statep->close_ret_priv_data_len != NULL) 1362 *statep->close_ret_priv_data_len = 0; 1363 1364 /* 1365 * If waiting for a response mad, then cancel the timer, 1366 * and delete the connection 1367 */ 1368 if (statep->state == IBCM_STATE_REQ_SENT || 1369 statep->state == IBCM_STATE_REP_SENT || 1370 statep->state == IBCM_STATE_REP_WAIT || 1371 statep->state == IBCM_STATE_MRA_REP_RCVD) { 1372 timeout_id_t timer_val = statep->timerid; 1373 ibcm_conn_state_t old_state; 1374 1375 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: " 1376 "chan 0x%p connection aborted in state %x", channel, 1377 statep->state); 1378 1379 old_state = statep->state; 1380 statep->state = IBCM_STATE_DELETE; 1381 1382 if (mode == IBT_NONBLOCKING) { 1383 if (taskq_dispatch(ibcm_taskq, 1384 ibcm_process_abort_via_taskq, statep, 1385 TQ_NOSLEEP) == 0) { 1386 1387 IBCM_REF_CNT_DECR(statep); 1388 statep->state = old_state; 1389 mutex_exit(&statep->state_mutex); 1390 return (IBT_INSUFF_KERNEL_RESOURCE); 1391 } /* if taskq_dispatch succeeds */ 1392 /* Cancel the timer */ 1393 statep->timerid = 0; 1394 mutex_exit(&statep->state_mutex); 1395 } else { 1396 /* Cancel the timer */ 1397 statep->timerid = 0; 1398 mutex_exit(&statep->state_mutex); 1399 (void) taskq_dispatch(ibcm_taskq, 1400 ibcm_process_abort_via_taskq, statep, 1401 TQ_SLEEP); 1402 } 1403 1404 /* cancel the currently running timer */ 1405 if (timer_val != 0) 1406 (void) untimeout(timer_val); 1407 1408 /* wait until cm handler returns for BLOCKING cases */ 1409 mutex_enter(&statep->state_mutex); 1410 if ((mode == IBT_BLOCKING) || 1411 (mode == IBT_NOCALLBACKS)) { 1412 while (statep->close_done != B_TRUE) 1413 cv_wait(&statep->block_client_cv, 1414 &statep->state_mutex); 1415 } 1416 1417 if (statep->close_ret_status) 1418 *statep->close_ret_status = IBT_CM_CLOSED_ABORT; 1419 mutex_exit(&statep->state_mutex); 1420 1421 /* 1422 * It would ideal to post a REJ MAD, but that would 1423 * be non-conformance to spec. Hence, delete the state 1424 * data. Assuming that happens quickly, any retransmits 1425 * from the remote are replied by CM with reject 1426 * reason " no valid com id". That would stop remote 1427 * sending any more MADs. 1428 */ 1429 ibcm_delete_state_data(statep); 1430 return (IBT_SUCCESS); 1431 1432 /* if CM busy in cm handler, wait until cm handler returns */ 1433 } else if (statep->state == IBCM_STATE_REQ_RCVD || 1434 statep->state == IBCM_STATE_REP_RCVD || 1435 statep->state == IBCM_STATE_MRA_SENT || 1436 statep->state == IBCM_STATE_MRA_REP_SENT) { 1437 1438 /* take control of statep */ 1439 statep->abort_flag |= IBCM_ABORT_CLIENT; 1440 1441 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: " 1442 "chan 0x%p connection aborted in state = %x", 1443 channel, statep->state); 1444 1445 /* 1446 * wait until state machine modifies qp state to error, 1447 * including disassociating statep and QP 1448 */ 1449 if ((mode == IBT_BLOCKING) || (mode == IBT_NOCALLBACKS)) 1450 while (statep->close_done != B_TRUE) 1451 cv_wait(&statep->block_client_cv, 1452 &statep->state_mutex); 1453 1454 /* a sanity setting */ 1455 if (mode == IBT_NOCALLBACKS) 1456 statep->cm_handler = NULL; 1457 IBCM_REF_CNT_DECR(statep); 1458 1459 /* 1460 * In rare situations, connection attempt could be 1461 * terminated for some other reason, before abort is 1462 * processed, but CM still returns ret_status as abort 1463 */ 1464 if (statep->close_ret_status) 1465 *statep->close_ret_status = IBT_CM_CLOSED_ABORT; 1466 mutex_exit(&statep->state_mutex); 1467 1468 /* 1469 * REJ MAD is posted by the CM state machine for this 1470 * case, hence state structure is deleted in the 1471 * state machine processing. 1472 */ 1473 return (IBT_SUCCESS); 1474 1475 } else if ((statep->state == IBCM_STATE_TIMEWAIT) || 1476 (statep->state == IBCM_STATE_DELETE)) { 1477 1478 /* State already in timewait, so no return priv data */ 1479 IBCM_REF_CNT_DECR(statep); 1480 1481 /* The teardown has already been done */ 1482 if (statep->close_ret_status) 1483 *statep->close_ret_status = 1484 IBT_CM_CLOSED_ALREADY; 1485 mutex_exit(&statep->state_mutex); 1486 1487 return (IBT_SUCCESS); 1488 1489 } else if ((statep->state == IBCM_STATE_DREQ_RCVD) || 1490 (statep->state == IBCM_STATE_DREQ_SENT) || 1491 (statep->state == IBCM_STATE_DREP_RCVD) || 1492 ((statep->state == IBCM_STATE_TIMED_OUT) && 1493 (statep->timedout_state == IBCM_STATE_DREQ_SENT))) { 1494 1495 /* 1496 * Either the remote or local client has already 1497 * initiated the teardown. IBCM_STATE_DREP_RCVD is 1498 * possible, if CM initiated teardown without client's 1499 * knowledge, for stale handling, etc., 1500 */ 1501 if (mode == IBT_NOCALLBACKS) { 1502 if (statep->close_nocb_state == IBCM_UNBLOCK) { 1503 statep->close_nocb_state = IBCM_FAIL; 1504 /* enable free qp after return */ 1505 ibtl_cm_chan_is_closing( 1506 statep->channel); 1507 } else while (statep->close_nocb_state == 1508 IBCM_BLOCK) 1509 cv_wait(&statep->block_client_cv, 1510 &statep->state_mutex); 1511 statep->cm_handler = NULL; /* sanity setting */ 1512 if (statep->close_ret_status) 1513 *statep->close_ret_status = 1514 IBT_CM_CLOSED_ALREADY; 1515 } else if (mode == IBT_BLOCKING) { 1516 /* wait until state is moved to timewait */ 1517 while (statep->close_done != B_TRUE) 1518 cv_wait(&statep->block_client_cv, 1519 &statep->state_mutex); 1520 } 1521 1522 IBCM_REF_CNT_DECR(statep); 1523 mutex_exit(&statep->state_mutex); 1524 1525 /* ret_status is set in state machine code */ 1526 return (IBT_SUCCESS); 1527 1528 } else if (statep->state == IBCM_STATE_TIMED_OUT) { 1529 1530 if ((mode == IBT_BLOCKING) || 1531 (mode == IBT_NOCALLBACKS)) { 1532 1533 /* 1534 * wait until cm handler invocation and 1535 * disassociation between statep and channel 1536 * is complete 1537 */ 1538 while (statep->close_done != B_TRUE) 1539 cv_wait(&statep->block_client_cv, 1540 &statep->state_mutex); 1541 } 1542 1543 if (statep->close_ret_status) 1544 *statep->close_ret_status = IBT_CM_CLOSED_ABORT; 1545 IBCM_REF_CNT_DECR(statep); 1546 mutex_exit(&statep->state_mutex); 1547 1548 return (IBT_SUCCESS); 1549 } else { 1550 IBCM_REF_CNT_DECR(statep); 1551 mutex_exit(&statep->state_mutex); 1552 1553 return (IBT_CM_FAILURE); 1554 } 1555 } 1556 1557 ASSERT(statep->close_nocb_state != IBCM_BLOCK); 1558 1559 if (mode == IBT_NOCALLBACKS) { 1560 statep->close_nocb_state = IBCM_FAIL; 1561 statep->cm_handler = NULL; 1562 ibtl_cm_chan_is_closing(statep->channel); 1563 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: " 1564 "NOCALLBACKS on in statep = %p", statep); 1565 } 1566 1567 if (statep->state != IBCM_STATE_ESTABLISHED) { 1568 goto lost_race; 1569 } 1570 1571 /* 1572 * Cancel/wait for any pending ibt_set_alt_path, and 1573 * release state mutex 1574 */ 1575 ibcm_sync_lapr_idle(statep); 1576 1577 ibcm_close_enter(); 1578 1579 mutex_enter(&statep->state_mutex); 1580 if (statep->state != IBCM_STATE_ESTABLISHED) { 1581 ibcm_close_exit(); 1582 goto lost_race; 1583 } 1584 1585 statep->state = IBCM_STATE_TRANSIENT_DREQ_SENT; 1586 statep->timerid = 0; 1587 statep->close_done = B_FALSE; 1588 statep->close_flow = 1; 1589 mutex_exit(&statep->state_mutex); 1590 1591 ibcm_post_dreq_mad(statep); 1592 1593 mutex_enter(&statep->state_mutex); 1594 1595 lost_race: 1596 if (mode == IBT_BLOCKING) { 1597 1598 /* wait for DREP */ 1599 while (statep->close_done != B_TRUE) 1600 cv_wait(&statep->block_client_cv, 1601 &statep->state_mutex); 1602 1603 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: chan 0x%p " 1604 "done blocking", channel); 1605 } 1606 1607 IBCM_REF_CNT_DECR(statep); 1608 mutex_exit(&statep->state_mutex); 1609 1610 /* If this message isn't seen then ibt_close_rc_channel failed */ 1611 IBTF_DPRINTF_L5(cmlog, "ibcm_close_rc_channel: chan 0x%p done", 1612 channel); 1613 1614 return (IBT_SUCCESS); 1615 } 1616 1617 ibt_status_t 1618 ibt_recycle_rc(ibt_channel_hdl_t rc_chan, ibt_cep_flags_t control, 1619 uint8_t hca_port_num, ibt_recycle_handler_t func, void *arg) 1620 { 1621 ibcm_state_data_t *statep; 1622 ibcm_taskq_recycle_arg_t *ibcm_tq_recycle_arg; 1623 ibt_qp_query_attr_t qp_attr; 1624 ibt_status_t retval; 1625 1626 IBTF_DPRINTF_L3(cmlog, "ibt_recycle_rc (%p, 0x%X, %d, %p, %p)", rc_chan, 1627 control, hca_port_num, func, arg); 1628 1629 if (IBCM_INVALID_CHANNEL(rc_chan)) { 1630 IBTF_DPRINTF_L2(cmlog, "ibt_recycle_rc: invalid channel"); 1631 return (IBT_CHAN_HDL_INVALID); 1632 } 1633 1634 /* check qp state */ 1635 retval = ibt_query_qp(rc_chan, &qp_attr); 1636 1637 if (retval != IBT_SUCCESS) 1638 return (retval); 1639 1640 if (qp_attr.qp_info.qp_trans != IBT_RC_SRV) 1641 return (IBT_CHAN_SRV_TYPE_INVALID); 1642 1643 if (qp_attr.qp_info.qp_state != IBT_STATE_ERROR) 1644 return (IBT_CHAN_STATE_INVALID); 1645 1646 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ibcm_tq_recycle_arg)) 1647 1648 ibcm_tq_recycle_arg = kmem_alloc(sizeof (ibcm_taskq_recycle_arg_t), 1649 KM_SLEEP); 1650 1651 ibcm_tq_recycle_arg->rc_chan = rc_chan; 1652 ibcm_tq_recycle_arg->control = control; 1653 ibcm_tq_recycle_arg->hca_port_num = hca_port_num; 1654 ibcm_tq_recycle_arg->func = func; 1655 ibcm_tq_recycle_arg->arg = arg; 1656 1657 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*ibcm_tq_recycle_arg)) 1658 1659 IBCM_GET_CHAN_PRIVATE(rc_chan, statep); 1660 1661 /* 1662 * If non-blocking ie., func specified and channel has not yet completed 1663 * the timewait, then schedule the work for later 1664 */ 1665 if ((func != NULL) && (statep != NULL)) { 1666 IBCM_RELEASE_CHAN_PRIVATE(rc_chan); 1667 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(statep->recycle_arg)) 1668 statep->recycle_arg = ibcm_tq_recycle_arg; 1669 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(statep->recycle_arg)) 1670 return (IBT_SUCCESS); 1671 } 1672 1673 /* 1674 * if blocking ie., func specified, and channel has not yet completed 1675 * the timewait, then block until the channel completes the timewait 1676 */ 1677 if (statep != NULL) 1678 IBCM_RELEASE_CHAN_PRIVATE(rc_chan); 1679 IBCM_WAIT_CHAN_PRIVATE(rc_chan); 1680 1681 if (func) { /* NON BLOCKING case. Taskq for QP state change */ 1682 (void) taskq_dispatch(ibcm_taskq, ibcm_process_rc_recycle, 1683 ibcm_tq_recycle_arg, TQ_SLEEP); 1684 return (IBT_SUCCESS); 1685 } else /* BLOCKING case */ 1686 return (ibcm_process_rc_recycle_ret(ibcm_tq_recycle_arg)); 1687 } 1688 1689 void 1690 ibcm_process_rc_recycle(void *recycle_arg) 1691 { 1692 (void) ibcm_process_rc_recycle_ret(recycle_arg); 1693 } 1694 1695 static ibt_status_t 1696 ibcm_process_rc_recycle_ret(void *recycle_arg) 1697 { 1698 ibt_qp_info_t qp_info; 1699 ibt_status_t ibt_status = IBT_SUCCESS; 1700 ibt_cep_modify_flags_t cep_flags; 1701 ibt_qp_query_attr_t qp_attr; 1702 ibcm_taskq_recycle_arg_t *ibcm_tq_recycle_arg = 1703 (ibcm_taskq_recycle_arg_t *)recycle_arg; 1704 1705 /* QP must have been in error state */ 1706 ibt_status = ibt_query_qp(ibcm_tq_recycle_arg->rc_chan, &qp_attr); 1707 if (ibt_status != IBT_SUCCESS) 1708 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1709 "chanp %p ibt_query_qp() = %d", 1710 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1711 else { 1712 /* perform the QP state change from ERROR to RESET */ 1713 bzero(&qp_info, sizeof (qp_info)); 1714 1715 qp_info.qp_trans = IBT_RC_SRV; 1716 qp_info.qp_state = IBT_STATE_RESET; 1717 1718 /* Call modify_qp to move to RESET state */ 1719 ibt_status = ibt_modify_qp(ibcm_tq_recycle_arg->rc_chan, 1720 IBT_CEP_SET_STATE, &qp_info, NULL); 1721 1722 if (ibt_status != IBT_SUCCESS) 1723 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1724 "chanp %p ibt_modify_qp() = %d for ERROR to RESET", 1725 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1726 } 1727 1728 if (ibt_status == IBT_SUCCESS) { 1729 1730 qp_info.qp_state = IBT_STATE_INIT; 1731 1732 /* set flags for all mandatory args from RESET to INIT */ 1733 cep_flags = IBT_CEP_SET_STATE | IBT_CEP_SET_PORT; 1734 cep_flags |= IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W; 1735 cep_flags |= IBT_CEP_SET_ATOMIC; 1736 1737 qp_info.qp_transport.rc.rc_path.cep_hca_port_num = 1738 ibcm_tq_recycle_arg->hca_port_num; 1739 qp_info.qp_flags |= 1740 ibcm_tq_recycle_arg->control & IBT_CEP_RDMA_RD; 1741 qp_info.qp_flags |= 1742 ibcm_tq_recycle_arg->control & IBT_CEP_RDMA_WR; 1743 qp_info.qp_flags |= 1744 ibcm_tq_recycle_arg->control & IBT_CEP_ATOMIC; 1745 1746 /* Always use the existing pkey */ 1747 qp_info.qp_transport.rc.rc_path.cep_pkey_ix = 1748 qp_attr. qp_info.qp_transport.rc.rc_path.cep_pkey_ix; 1749 1750 /* Call modify_qp to move to INIT state */ 1751 ibt_status = ibt_modify_qp(ibcm_tq_recycle_arg->rc_chan, 1752 cep_flags, &qp_info, NULL); 1753 1754 if (ibt_status != IBT_SUCCESS) 1755 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1756 "chanp %p ibt_modify_qp() = %d for RESET to INIT", 1757 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1758 } 1759 1760 /* Change the QP CM state to indicate QP being re-used */ 1761 if (ibt_status == IBT_SUCCESS) 1762 ibtl_cm_chan_is_reused(ibcm_tq_recycle_arg->rc_chan); 1763 1764 /* Call func, if defined */ 1765 if (ibcm_tq_recycle_arg->func) 1766 (*(ibcm_tq_recycle_arg->func))(ibt_status, 1767 ibcm_tq_recycle_arg->arg); 1768 1769 kmem_free(ibcm_tq_recycle_arg, sizeof (ibcm_taskq_recycle_arg_t)); 1770 1771 return (ibt_status); 1772 } 1773 1774 static void 1775 ibcm_process_abort_via_taskq(void *args) 1776 { 1777 ibcm_state_data_t *statep = (ibcm_state_data_t *)args; 1778 1779 ibcm_process_abort(statep); 1780 mutex_enter(&statep->state_mutex); 1781 IBCM_REF_CNT_DECR(statep); 1782 mutex_exit(&statep->state_mutex); 1783 } 1784 1785 /* 1786 * Local UD CM Handler's private data, used during ibt_request_ud_dest() in 1787 * Non-Blocking mode operations. 1788 */ 1789 typedef struct ibcm_local_handler_s { 1790 ibt_cm_ud_handler_t actual_cm_handler; 1791 void *actual_cm_private; 1792 ibt_ud_dest_t *dest_hdl; 1793 } ibcm_local_handler_t; 1794 1795 _NOTE(READ_ONLY_DATA(ibcm_local_handler_s)) 1796 1797 /* 1798 * Local UD CM Handler, used when ibt_alloc_ud_dest() is issued in 1799 * NON-Blocking mode. 1800 * 1801 * Out here, we update the UD Destination handle with 1802 * the obtained DQPN and QKey (from SIDR REP) and invokes actual client 1803 * handler that was specified by the client. 1804 */ 1805 static ibt_cm_status_t 1806 ibcm_local_cm_handler(void *priv, ibt_cm_ud_event_t *event, 1807 ibt_cm_ud_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len) 1808 { 1809 ibcm_local_handler_t *handler_priv = (ibcm_local_handler_t *)priv; 1810 1811 IBTF_DPRINTF_L4(cmlog, "ibcm_local_cm_handler: event %d", 1812 event->cm_type); 1813 1814 ASSERT(handler_priv != NULL); 1815 1816 switch (event->cm_type) { 1817 case IBT_CM_UD_EVENT_SIDR_REP: 1818 /* Update QPN & QKey from event into destination handle. */ 1819 if (handler_priv->dest_hdl != NULL) { 1820 handler_priv->dest_hdl->ud_dst_qpn = 1821 event->cm_event.sidr_rep.srep_remote_qpn; 1822 handler_priv->dest_hdl->ud_qkey = 1823 event->cm_event.sidr_rep.srep_remote_qkey; 1824 } 1825 1826 /* Invoke the client handler - inform only, so ignore retval */ 1827 (void) handler_priv->actual_cm_handler( 1828 handler_priv->actual_cm_private, event, ret_args, priv_data, 1829 len); 1830 1831 /* Free memory allocated for local handler's private data. */ 1832 if (handler_priv != NULL) 1833 kmem_free(handler_priv, sizeof (*handler_priv)); 1834 1835 break; 1836 default: 1837 IBTF_DPRINTF_L2(cmlog, "ibcm_local_cm_handler: ERROR"); 1838 break; 1839 } 1840 1841 return (IBT_CM_ACCEPT); 1842 } 1843 1844 1845 /* Validate the input UD destination attributes. */ 1846 static ibt_status_t 1847 ibcm_validate_dqpn_data(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 1848 ibt_ud_returns_t *ret_args) 1849 { 1850 /* cm handler must always be specified */ 1851 if (mode == IBT_NONBLOCKING && attr->ud_cm_handler == NULL) { 1852 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1853 "CM handler is not specified "); 1854 return (IBT_INVALID_PARAM); 1855 } 1856 1857 if (mode == IBT_NONBLOCKING) { 1858 if (ret_args != NULL) { 1859 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1860 "ret_args should be NULL when called in " 1861 "non-blocking mode"); 1862 return (IBT_INVALID_PARAM); 1863 } 1864 } else if (mode == IBT_BLOCKING) { 1865 if (ret_args == NULL) { 1866 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1867 "ret_args should be Non-NULL when called in " 1868 "blocking mode"); 1869 return (IBT_INVALID_PARAM); 1870 } 1871 } else { 1872 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1873 "invalid mode %x specified ", mode); 1874 return (IBT_INVALID_PARAM); 1875 } 1876 1877 if (attr->ud_sid == 0) { 1878 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1879 "ServiceID must be specified. "); 1880 return (IBT_INVALID_PARAM); 1881 } 1882 1883 if (attr->ud_addr == NULL) { 1884 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1885 "Address Info NULL"); 1886 return (IBT_INVALID_PARAM); 1887 } 1888 1889 /* Validate SGID */ 1890 if ((attr->ud_addr->av_sgid.gid_prefix == 0) || 1891 (attr->ud_addr->av_sgid.gid_guid == 0)) { 1892 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: Invalid SGID"); 1893 return (IBT_INVALID_PARAM); 1894 } 1895 IBTF_DPRINTF_L3(cmlog, "ibcm_validate_dqpn_data: SGID<%llX:%llX>", 1896 attr->ud_addr->av_sgid.gid_prefix, 1897 attr->ud_addr->av_sgid.gid_guid); 1898 1899 /* Validate DGID */ 1900 if ((attr->ud_addr->av_dgid.gid_prefix == 0) || 1901 (attr->ud_addr->av_dgid.gid_guid == 0)) { 1902 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: Invalid DGID"); 1903 return (IBT_INVALID_PARAM); 1904 } 1905 IBTF_DPRINTF_L3(cmlog, "ibcm_validate_dqpn_data: DGID<%llX:%llX>", 1906 attr->ud_addr->av_dgid.gid_prefix, 1907 attr->ud_addr->av_dgid.gid_guid); 1908 1909 return (IBT_SUCCESS); 1910 } 1911 1912 1913 /* Perform SIDR to retrieve DQPN and QKey. */ 1914 static ibt_status_t 1915 ibcm_ud_get_dqpn(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 1916 ibt_ud_returns_t *ret_args) 1917 { 1918 ibt_status_t retval; 1919 ib_pkey_t ud_pkey; 1920 ibmf_handle_t ibmf_hdl; 1921 ibmf_msg_t *ibmf_msg; 1922 ibcm_hca_info_t *hcap; 1923 ibcm_sidr_req_msg_t *sidr_req_msgp; 1924 ibcm_ud_state_data_t *ud_statep; 1925 ibtl_cm_hca_port_t port; 1926 ibcm_sidr_srch_t sidr_entry; 1927 ibcm_qp_list_t *cm_qp_entry; 1928 1929 /* Retrieve HCA GUID value from the available SGID info. */ 1930 retval = ibtl_cm_get_hca_port(attr->ud_addr->av_sgid, 0, &port); 1931 if ((retval != IBT_SUCCESS) || (port.hp_port == 0)) { 1932 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1933 "ibtl_cm_get_hca_port failed: %d", retval); 1934 return (retval); 1935 } 1936 1937 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: " 1938 "HCA GUID:%llX, port_num:%d", port.hp_hca_guid, port.hp_port); 1939 1940 /* Lookup the HCA info for this GUID */ 1941 if ((hcap = ibcm_find_hca_entry(port.hp_hca_guid)) == NULL) { 1942 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: hcap is NULL"); 1943 return (IBT_HCA_INVALID); 1944 } 1945 1946 /* Return failure if the HCA device or Port is not operational */ 1947 1948 if ((retval = ibt_get_port_state_byguid(port.hp_hca_guid, port.hp_port, 1949 NULL, NULL)) != IBT_SUCCESS) { 1950 /* Device Port is not in good state, don't use it. */ 1951 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: Invalid " 1952 "port specified or port not active"); 1953 ibcm_dec_hca_acc_cnt(hcap); 1954 return (retval); 1955 } 1956 1957 retval = ibt_index2pkey_byguid(port.hp_hca_guid, port.hp_port, 1958 attr->ud_pkey_ix, &ud_pkey); 1959 if (retval != IBT_SUCCESS) { 1960 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1961 "Failed to convert index2pkey: %d", retval); 1962 ibcm_dec_hca_acc_cnt(hcap); 1963 return (retval); 1964 } 1965 1966 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(sidr_entry)) 1967 1968 /* Allocate a new request id */ 1969 if (ibcm_alloc_reqid(hcap, &sidr_entry.srch_req_id) == IBCM_FAILURE) { 1970 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1971 "no req id available"); 1972 ibcm_dec_hca_acc_cnt(hcap); 1973 return (IBT_INSUFF_KERNEL_RESOURCE); 1974 } 1975 1976 if ((hcap->hca_port_info[port.hp_port - 1].port_ibmf_hdl == NULL) && 1977 ((retval = ibcm_hca_reinit_port(hcap, port.hp_port - 1)) 1978 != IBT_SUCCESS)) { 1979 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1980 "ibmf reg or callback setup failed during re-initialize"); 1981 return (retval); 1982 } 1983 1984 ibmf_hdl = hcap->hca_port_info[port.hp_port - 1].port_ibmf_hdl; 1985 1986 /* find the ibmf QP to post the SIDR REQ */ 1987 if ((cm_qp_entry = ibcm_find_qp(hcap, port.hp_port, ud_pkey)) == 1988 NULL) { 1989 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: IBMF QP allocation" 1990 " failed"); 1991 ibcm_dec_hca_acc_cnt(hcap); 1992 return (IBT_INSUFF_RESOURCE); 1993 } 1994 1995 if ((retval = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg, MAD_METHOD_SEND)) 1996 != IBT_SUCCESS) { 1997 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: IBMF MSG allocation" 1998 " failed"); 1999 ibcm_release_qp(cm_qp_entry); 2000 ibcm_dec_hca_acc_cnt(hcap); 2001 return (retval); 2002 } 2003 2004 sidr_entry.srch_lid = port.hp_base_lid; 2005 sidr_entry.srch_gid = attr->ud_addr->av_sgid; 2006 sidr_entry.srch_grh_exists = attr->ud_addr->av_send_grh; 2007 sidr_entry.srch_mode = IBCM_ACTIVE_MODE; 2008 2009 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(sidr_entry)) 2010 2011 /* do various allocations needed here */ 2012 rw_enter(&hcap->hca_sidr_list_lock, RW_WRITER); 2013 2014 (void) ibcm_find_sidr_entry(&sidr_entry, hcap, &ud_statep, 2015 IBCM_FLAG_ADD); 2016 rw_exit(&hcap->hca_sidr_list_lock); 2017 2018 /* Increment hca's resource count */ 2019 ibcm_inc_hca_res_cnt(hcap); 2020 2021 /* After a resource created on hca, no need to hold the acc cnt */ 2022 ibcm_dec_hca_acc_cnt(hcap); 2023 2024 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ud_statep)) 2025 2026 /* Initialize some ud_statep fields */ 2027 ud_statep->ud_stored_msg = ibmf_msg; 2028 ud_statep->ud_svc_id = attr->ud_sid; 2029 ud_statep->ud_pkt_life_time = 2030 ibt_ib2usec(attr->ud_pkt_lt); 2031 ud_statep->ud_stored_reply_addr.cm_qp_entry = cm_qp_entry; 2032 2033 /* set remaining retry cnt */ 2034 ud_statep->ud_remaining_retry_cnt = ud_statep->ud_max_cm_retries; 2035 2036 /* 2037 * Get UD handler and corresponding args which is pass it back 2038 * as first argument for the handler. 2039 */ 2040 ud_statep->ud_state_cm_private = attr->ud_cm_private; 2041 2042 if (mode == IBT_BLOCKING) 2043 ud_statep->ud_return_data = ret_args; 2044 else 2045 ud_statep->ud_cm_handler = attr->ud_cm_handler; 2046 2047 /* Initialize the fields of ud_statep->ud_stored_reply_addr */ 2048 ud_statep->ud_stored_reply_addr.grh_exists = attr->ud_addr->av_send_grh; 2049 ud_statep->ud_stored_reply_addr.ibmf_hdl = ibmf_hdl; 2050 ud_statep->ud_stored_reply_addr.grh_hdr.ig_hop_limit = 2051 attr->ud_addr->av_hop; 2052 ud_statep->ud_stored_reply_addr.grh_hdr.ig_sender_gid = 2053 attr->ud_addr->av_sgid; 2054 ud_statep->ud_stored_reply_addr.grh_hdr.ig_recver_gid = 2055 attr->ud_addr->av_dgid; 2056 ud_statep->ud_stored_reply_addr.grh_hdr.ig_tclass = 2057 attr->ud_addr->av_tclass; 2058 ud_statep->ud_stored_reply_addr.grh_hdr.ig_flow_label = 2059 attr->ud_addr->av_flow & IB_GRH_FLOW_LABEL_MASK; 2060 2061 /* needs to be derived based on the base LID and path bits */ 2062 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_local_lid = 2063 port.hp_base_lid; 2064 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_remote_lid = 2065 attr->ud_addr->av_dlid; 2066 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_p_key = ud_pkey; 2067 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_q_key = IB_GSI_QKEY; 2068 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_service_level = 2069 attr->ud_addr->av_srvl; 2070 2071 /* 2072 * This may be enchanced later, to use a remote qno based on past 2073 * redirect rej mad responses. This would be the place to specify 2074 * appropriate remote qno 2075 */ 2076 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_remote_qno = 1; 2077 2078 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sidr_req_msgp)) 2079 2080 /* Initialize the SIDR REQ message fields */ 2081 sidr_req_msgp = 2082 (ibcm_sidr_req_msg_t *)IBCM_OUT_MSGP(ud_statep->ud_stored_msg); 2083 2084 sidr_req_msgp->sidr_req_request_id = h2b32(ud_statep->ud_req_id); 2085 sidr_req_msgp->sidr_req_service_id = h2b64(attr->ud_sid); 2086 sidr_req_msgp->sidr_req_pkey = h2b16(ud_pkey); 2087 IBCM_OUT_HDRP(ud_statep->ud_stored_msg)->AttributeID = 2088 h2b16(IBCM_INCOMING_SIDR_REQ + IBCM_ATTR_BASE_ID); 2089 2090 if ((attr->ud_priv_data != NULL) && (attr->ud_priv_data_len > 0)) { 2091 bcopy(attr->ud_priv_data, sidr_req_msgp->sidr_req_private_data, 2092 min(attr->ud_priv_data_len, IBT_SIDR_REQ_PRIV_DATA_SZ)); 2093 } 2094 2095 /* Send out the SIDR REQ message */ 2096 ud_statep->ud_state = IBCM_STATE_SIDR_REQ_SENT; 2097 ud_statep->ud_timer_stored_state = IBCM_STATE_SIDR_REQ_SENT; 2098 IBCM_UD_REF_CNT_INCR(ud_statep); /* for non-blocking SIDR REQ post */ 2099 ud_statep->ud_timer_value = ibt_ib2usec(ibcm_max_sidr_rep_proctime) + 2100 (ud_statep->ud_pkt_life_time * 2); 2101 2102 IBCM_OUT_HDRP(ud_statep->ud_stored_msg)->TransactionID = 2103 h2b64(ibcm_generate_tranid(IBCM_INCOMING_SIDR_REQ, 2104 ud_statep->ud_req_id, 0)); 2105 2106 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: timer_value in HZ = %x", 2107 ud_statep->ud_timer_value); 2108 2109 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*ud_statep)) 2110 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sidr_req_msgp)) 2111 2112 ibcm_post_ud_mad(ud_statep, ud_statep->ud_stored_msg, 2113 ibcm_post_sidr_req_complete, ud_statep); 2114 2115 mutex_enter(&ud_statep->ud_state_mutex); 2116 2117 /* Wait for SIDR_REP */ 2118 if (mode == IBT_BLOCKING) { 2119 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: blocking"); 2120 2121 while (ud_statep->ud_blocking_done != B_TRUE) { 2122 cv_wait(&ud_statep->ud_block_client_cv, 2123 &ud_statep->ud_state_mutex); 2124 } 2125 2126 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: finished blocking"); 2127 2128 if (ret_args->ud_status == IBT_CM_SREP_QPN_VALID) { 2129 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: DQPN = %x, " 2130 "status = %x, QKey = %x", ret_args->ud_dqpn, 2131 ret_args->ud_status, ret_args->ud_qkey); 2132 2133 } else { 2134 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: Status<%x>", 2135 ret_args->ud_status); 2136 retval = IBT_CM_FAILURE; 2137 } 2138 } 2139 2140 IBCM_UD_REF_CNT_DECR(ud_statep); 2141 mutex_exit(&ud_statep->ud_state_mutex); 2142 2143 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: done"); 2144 2145 return (retval); 2146 } 2147 2148 2149 /* 2150 * Function: 2151 * ibt_request_ud_dest 2152 * Input: 2153 * ud_dest A previously allocated UD destination handle. 2154 * mode This function can execute in blocking or non blocking 2155 * modes. 2156 * attr UD destination attributes to be modified. 2157 * Output: 2158 * ud_ret_args If the function is called in blocking mode, ud_ret_args 2159 * should be a pointer to an ibt_ud_returns_t struct. 2160 * Returns: 2161 * IBT_SUCCESS 2162 * Description: 2163 * Modify a previously allocated UD destination handle based on the 2164 * results of doing the SIDR protocol. 2165 */ 2166 ibt_status_t 2167 ibt_request_ud_dest(ibt_ud_dest_hdl_t ud_dest, ibt_execution_mode_t mode, 2168 ibt_ud_dest_attr_t *attr, ibt_ud_returns_t *ud_ret_args) 2169 { 2170 ibt_status_t retval; 2171 ibt_ud_dest_t *ud_destp; 2172 ibcm_local_handler_t *local_handler_priv = NULL; 2173 2174 IBTF_DPRINTF_L3(cmlog, "ibt_request_ud_dest(%p, %x, %p, %p)", 2175 ud_dest, mode, attr, ud_ret_args); 2176 2177 retval = ibcm_validate_dqpn_data(attr, mode, ud_ret_args); 2178 if (retval != IBT_SUCCESS) { 2179 return (retval); 2180 } 2181 2182 ud_destp = ud_dest; 2183 2184 /* Allocate an Address handle. */ 2185 retval = ibt_modify_ah(ud_destp->ud_dest_hca, ud_destp->ud_ah, 2186 attr->ud_addr); 2187 if (retval != IBT_SUCCESS) { 2188 IBTF_DPRINTF_L2(cmlog, "ibt_request_ud_dest: " 2189 "Address Handle Modification failed: %d", retval); 2190 return (retval); 2191 } 2192 2193 if (mode == IBT_NONBLOCKING) { 2194 /* 2195 * In NON-BLOCKING mode, and we need to update the destination 2196 * handle with the DQPN and QKey that are obtained from 2197 * SIDR REP, hook-up our own handler, so that we can catch 2198 * the event, and we ourselves call the actual client's 2199 * ud_cm_handler, in our handler. 2200 */ 2201 2202 /* Allocate memory for local handler's private data. */ 2203 local_handler_priv = 2204 kmem_alloc(sizeof (*local_handler_priv), KM_SLEEP); 2205 2206 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*local_handler_priv)) 2207 2208 local_handler_priv->actual_cm_handler = attr->ud_cm_handler; 2209 local_handler_priv->actual_cm_private = attr->ud_cm_private; 2210 local_handler_priv->dest_hdl = ud_destp; 2211 2212 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*local_handler_priv)) 2213 2214 attr->ud_cm_handler = ibcm_local_cm_handler; 2215 attr->ud_cm_private = local_handler_priv; 2216 } 2217 2218 /* In order to get DQPN and Destination QKey, perform SIDR */ 2219 retval = ibcm_ud_get_dqpn(attr, mode, ud_ret_args); 2220 if (retval != IBT_SUCCESS) { 2221 IBTF_DPRINTF_L2(cmlog, "ibt_request_ud_dest: " 2222 "Failed to get DQPN: %d", retval); 2223 2224 /* Free memory allocated for local handler's private data. */ 2225 if (local_handler_priv != NULL) 2226 kmem_free(local_handler_priv, 2227 sizeof (*local_handler_priv)); 2228 return (retval); 2229 } 2230 2231 /* 2232 * Fill in the dqpn and dqkey as obtained from ud_ret_args, 2233 * values will be valid only on BLOCKING mode. 2234 */ 2235 if (mode == IBT_BLOCKING) { 2236 ud_destp->ud_dst_qpn = ud_ret_args->ud_dqpn; 2237 ud_destp->ud_qkey = ud_ret_args->ud_qkey; 2238 } 2239 2240 return (retval); 2241 } 2242 2243 /* 2244 * Function: 2245 * ibt_ud_get_dqpn 2246 * Input: 2247 * attr A pointer to an ibt_ud_dest_attr_t struct that are 2248 * required for SIDR REQ message. Not specified attributes 2249 * should be set to "NULL" or "0". 2250 * ud_sid, ud_addr and ud_pkt_lt must be specified. 2251 * mode This function can execute in blocking or non blocking 2252 * modes. 2253 * Output: 2254 * returns If the function is called in blocking mode, returns 2255 * should be a pointer to an ibt_ud_returns_t struct. 2256 * Return: 2257 * IBT_SUCCESS on success or respective failure on error. 2258 * Description: 2259 * Finds the destination QPN at the specified destination that the 2260 * specified service can be reached on. The IBTF CM initiates the 2261 * service ID resolution protocol (SIDR) to determine a destination QPN. 2262 * 2263 * NOTE: SIDR_REQ is initiated from active side. 2264 */ 2265 ibt_status_t 2266 ibt_ud_get_dqpn(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 2267 ibt_ud_returns_t *returns) 2268 { 2269 ibt_status_t retval; 2270 2271 IBTF_DPRINTF_L3(cmlog, "ibt_ud_get_dqpn(%p, %x, %p)", 2272 attr, mode, returns); 2273 2274 retval = ibcm_validate_dqpn_data(attr, mode, returns); 2275 if (retval != IBT_SUCCESS) { 2276 return (retval); 2277 } 2278 2279 return (ibcm_ud_get_dqpn(attr, mode, returns)); 2280 } 2281 2282 2283 /* 2284 * ibt_cm_delay: 2285 * A client CM handler function can call this function 2286 * to extend its response time to a CM event. 2287 * INPUTS: 2288 * flags Indicates what CM message processing is being delayed 2289 * by the CM handler, valid values are: 2290 * IBT_CM_DELAY_REQ 2291 * IBT_CM_DELAY_REP 2292 * IBT_CM_DELAY_LAP 2293 * cm_session_id The session ID that was passed to client srv_handler 2294 * by the CM 2295 * service_time The extended service time 2296 * priv_data Vendor specific data to be sent in the CM generated 2297 * MRA message. Should be NULL if not specified. 2298 * len The number of bytes of data specified by priv_data. 2299 * 2300 * RETURN VALUES: 2301 * IBT_SUCCESS on success (or respective failure on error) 2302 */ 2303 ibt_status_t 2304 ibt_cm_delay(ibt_cmdelay_flags_t flags, void *cm_session_id, 2305 clock_t service_time, void *priv_data, ibt_priv_data_len_t len) 2306 { 2307 uint8_t msg_typ = 0; 2308 ibcm_mra_msg_t *mra_msgp; 2309 ibcm_state_data_t *statep; 2310 ibt_status_t status; 2311 2312 IBTF_DPRINTF_L3(cmlog, "ibt_cm_delay(0x%x, %p, 0x%x)", 2313 flags, cm_session_id, service_time); 2314 2315 /* 2316 * Make sure channel is associated with a statep 2317 */ 2318 statep = (ibcm_state_data_t *)cm_session_id; 2319 2320 if (statep == NULL) { 2321 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: statep NULL"); 2322 return (IBT_INVALID_PARAM); 2323 } 2324 2325 IBTF_DPRINTF_L4(cmlog, "ibt_cm_delay: statep %p", statep); 2326 2327 /* Allocate an ibmf msg for mra, if not allocated yet */ 2328 if (statep->mra_msg == NULL) { 2329 if ((status = ibcm_alloc_out_msg( 2330 statep->stored_reply_addr.ibmf_hdl, &statep->mra_msg, 2331 MAD_METHOD_SEND)) != IBT_SUCCESS) { 2332 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: chan 0x%p" 2333 "IBMF MSG allocation failed", statep->channel); 2334 return (status); 2335 } 2336 } 2337 2338 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mra_msgp)) 2339 2340 mra_msgp = (ibcm_mra_msg_t *)IBCM_OUT_MSGP(statep->mra_msg); 2341 mra_msgp->mra_local_comm_id = h2b32(statep->local_comid); 2342 mra_msgp->mra_remote_comm_id = h2b32(statep->remote_comid); 2343 2344 /* fill in rest of MRA's fields - Message MRAed and Service Timeout */ 2345 if (flags == IBT_CM_DELAY_REQ) { 2346 msg_typ = IBT_CM_MRA_TYPE_REQ; 2347 } else if (flags == IBT_CM_DELAY_REP) { 2348 msg_typ = IBT_CM_MRA_TYPE_REP; 2349 } else if (flags == IBT_CM_DELAY_LAP) { 2350 msg_typ = IBT_CM_MRA_TYPE_LAP; 2351 } 2352 2353 mra_msgp->mra_message_type_plus = msg_typ << 6; 2354 mra_msgp->mra_service_timeout_plus = ibt_usec2ib(service_time) << 3; 2355 2356 len = min(len, IBT_MRA_PRIV_DATA_SZ); 2357 if (priv_data && (len > 0)) 2358 bcopy(priv_data, mra_msgp->mra_private_data, len); 2359 2360 IBCM_OUT_HDRP(statep->mra_msg)->AttributeID = 2361 h2b16(IBCM_INCOMING_MRA + IBCM_ATTR_BASE_ID); 2362 2363 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mra_msgp)) 2364 2365 mutex_enter(&statep->state_mutex); 2366 2367 if ((statep->mode == IBCM_ACTIVE_MODE) && 2368 (statep->state == IBCM_STATE_REP_RCVD)) { 2369 statep->state = IBCM_STATE_MRA_REP_SENT; 2370 } else if (statep->mode == IBCM_PASSIVE_MODE) { 2371 if (statep->state == IBCM_STATE_REQ_RCVD) { 2372 statep->state = IBCM_STATE_MRA_SENT; 2373 } else if (statep->ap_state == IBCM_AP_STATE_LAP_RCVD) { 2374 statep->ap_state = IBCM_AP_STATE_MRA_LAP_RCVD; 2375 } else { 2376 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: invalid state " 2377 "/ap_state/mode %x, %x, %x", statep->state, 2378 statep->ap_state, statep->mode); 2379 mutex_exit(&statep->state_mutex); 2380 return (IBT_CHAN_STATE_INVALID); 2381 } 2382 } else { 2383 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: invalid state " 2384 "/ap_state/mode %x, %x, %x", statep->state, 2385 statep->ap_state, statep->mode); 2386 mutex_exit(&statep->state_mutex); 2387 2388 return (IBT_CHAN_STATE_INVALID); 2389 } 2390 /* service time is usecs, stale_clock is nsecs */ 2391 statep->stale_clock = gethrtime() + 2392 (hrtime_t)ibt_ib2usec(ibt_usec2ib(service_time)) * (1000 * 2393 statep->max_cm_retries); 2394 2395 statep->send_mad_flags |= IBCM_MRA_POST_BUSY; 2396 IBCM_REF_CNT_INCR(statep); /* for ibcm_post_mra_complete */ 2397 mutex_exit(&statep->state_mutex); 2398 2399 IBCM_OUT_HDRP(statep->mra_msg)->TransactionID = 2400 IBCM_OUT_HDRP(statep->stored_msg)->TransactionID; 2401 2402 /* post the MRA mad in blocking mode, as no timers involved */ 2403 ibcm_post_rc_mad(statep, statep->mra_msg, ibcm_post_mra_complete, 2404 statep); 2405 ibcm_insert_trace(statep, IBCM_TRACE_OUTGOING_MRA); 2406 /* If this message isn't seen then ibt_cm_delay failed */ 2407 IBTF_DPRINTF_L3(cmlog, "ibt_cm_delay: done !!"); 2408 2409 return (IBT_SUCCESS); 2410 } 2411 2412 2413 /* 2414 * ibt_register_service() 2415 * Register a service with the IBCM 2416 * 2417 * INPUTS: 2418 * ibt_hdl The IBT client handle returned to the client 2419 * on an ibt_attach() call. 2420 * 2421 * srv The address of a ibt_srv_desc_t that describes 2422 * the service, containing the following: 2423 * 2424 * sd_ud_handler The Service CM UD event Handler. 2425 * sd_handler The Service CM RC/UC/RD event Handler. 2426 * sd_flags Service flags (peer-to-peer, or not). 2427 * 2428 * sid This tells CM if the service is local (sid is 0) or 2429 * wellknown (sid is the starting service id of the range). 2430 * 2431 * num_sids The number of contiguous service-ids to reserve. 2432 * 2433 * srv_hdl The address of a service identification handle, used 2434 * to deregister a service, and to bind GIDs to. 2435 * 2436 * ret_sid The address to store the Service ID return value. 2437 * If num_sids > 1, ret_sid is the first Service ID 2438 * in the range. 2439 * 2440 * ibt_register_service() returns: 2441 * IBT_SUCCESS - added a service successfully. 2442 * IBT_INVALID_PARAM - invalid input parameter. 2443 * IBT_CM_FAILURE - failed to add the service. 2444 * IBT_CM_SERVICE_EXISTS - service already exists. 2445 * IBT_INSUFF_KERNEL_RESOURCE - ran out of local service ids (should 2446 * never happen). 2447 */ 2448 ibt_status_t 2449 ibt_register_service(ibt_clnt_hdl_t ibt_hdl, ibt_srv_desc_t *srv, 2450 ib_svc_id_t sid, int num_sids, ibt_srv_hdl_t *srv_hdl, ib_svc_id_t *ret_sid) 2451 { 2452 ibcm_svc_info_t *svcinfop; 2453 2454 IBTF_DPRINTF_L2(cmlog, "ibt_register_service(%p (%s), %p, 0x%llX, %d)", 2455 ibt_hdl, ibtl_cm_get_clnt_name(ibt_hdl), srv, (longlong_t)sid, 2456 num_sids); 2457 2458 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*svcinfop)) 2459 2460 *srv_hdl = NULL; 2461 2462 if (num_sids <= 0) { 2463 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2464 "Invalid number of service-ids specified (%d)", num_sids); 2465 return (IBT_INVALID_PARAM); 2466 } 2467 2468 if (sid == 0) { 2469 if (ret_sid == NULL) 2470 return (IBT_INVALID_PARAM); 2471 sid = ibcm_alloc_local_sids(num_sids); 2472 if (sid == 0) 2473 return (IBT_INSUFF_KERNEL_RESOURCE); 2474 2475 /* Make sure that the ServiceId specified is not of LOCAL AGN type. */ 2476 } else if ((sid & IB_SID_AGN_MASK) == IB_SID_AGN_LOCAL) { 2477 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2478 "Invalid non-LOCAL SID specified: 0x%llX", 2479 (longlong_t)sid); 2480 return (IBT_INVALID_PARAM); 2481 } 2482 2483 svcinfop = ibcm_create_svc_entry(sid, num_sids); 2484 2485 if (svcinfop == NULL) { 2486 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2487 "Service-ID 0x%llx already registered", (longlong_t)sid); 2488 return (IBT_CM_SERVICE_EXISTS); 2489 } 2490 2491 /* 2492 * 'sid' and 'num_sids' are filled in ibcm_create_svc_entry() 2493 */ 2494 svcinfop->svc_flags = srv->sd_flags; 2495 svcinfop->svc_rc_handler = srv->sd_handler; 2496 svcinfop->svc_ud_handler = srv->sd_ud_handler; 2497 2498 if (ret_sid != NULL) 2499 *ret_sid = sid; 2500 2501 *srv_hdl = svcinfop; 2502 2503 ibtl_cm_change_service_cnt(ibt_hdl, num_sids); 2504 2505 /* If this message isn't seen, then ibt_register_service failed. */ 2506 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: done (%p, %llX)", 2507 svcinfop, sid); 2508 2509 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*svcinfop)) 2510 2511 return (IBT_SUCCESS); 2512 } 2513 2514 2515 static ibt_status_t 2516 ibcm_write_service_record(ibmf_saa_handle_t saa_handle, 2517 sa_service_record_t *srv_recp, ibmf_saa_access_type_t saa_type) 2518 { 2519 int rval; 2520 int retry; 2521 2522 ibcm_sa_access_enter(); 2523 for (retry = 0; retry < ibcm_max_sa_retries; retry++) { 2524 rval = ibmf_saa_update_service_record( 2525 saa_handle, srv_recp, saa_type, 0); 2526 if (rval != IBMF_TRANS_TIMEOUT) { 2527 break; 2528 } 2529 IBTF_DPRINTF_L2(cmlog, "ibcm_write_service_record: " 2530 "ibmf_saa_update_service_record timed out" 2531 " SID = %llX, rval = %d, saa_type = %d", 2532 (longlong_t)srv_recp->ServiceID, rval, saa_type); 2533 delay(ibcm_sa_timeout_delay); 2534 } 2535 ibcm_sa_access_exit(); 2536 2537 if (rval != IBMF_SUCCESS) { 2538 IBTF_DPRINTF_L2(cmlog, "ibcm_write_service_record: " 2539 "ibmf_saa_update_service_record() : Failed - %d", rval); 2540 return (ibcm_ibmf_analyze_error(rval)); 2541 } else 2542 return (IBT_SUCCESS); 2543 } 2544 2545 2546 static void 2547 ibcm_rem_stale_srec(ibmf_saa_handle_t saa_handle, sa_service_record_t *srec) 2548 { 2549 ibt_status_t retval; 2550 uint_t num_found; 2551 size_t length; 2552 sa_service_record_t *srv_resp; 2553 void *results_p; 2554 uint_t i; 2555 uint64_t component_mask; 2556 ibmf_saa_access_args_t access_args; 2557 2558 component_mask = 2559 SA_SR_COMPMASK_PKEY | SA_SR_COMPMASK_NAME | SA_SR_COMPMASK_GID; 2560 2561 /* Call in SA Access retrieve routine to get Service Records. */ 2562 access_args.sq_attr_id = SA_SERVICERECORD_ATTRID; 2563 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 2564 access_args.sq_component_mask = component_mask; 2565 access_args.sq_template = srec; 2566 access_args.sq_template_length = sizeof (sa_service_record_t); 2567 access_args.sq_callback = NULL; 2568 access_args.sq_callback_arg = NULL; 2569 2570 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 2571 &results_p); 2572 if (retval != IBT_SUCCESS) { 2573 IBTF_DPRINTF_L2(cmlog, "ibcm_rem_stale_srec: " 2574 "SA Access Failure"); 2575 return; 2576 } 2577 2578 num_found = length / sizeof (sa_service_record_t); 2579 2580 if (num_found) 2581 IBTF_DPRINTF_L3(cmlog, "ibcm_rem_stale_srec: " 2582 "Found %d matching Service Records.", num_found); 2583 2584 /* Validate the returned number of records. */ 2585 if ((results_p != NULL) && (num_found > 0)) { 2586 2587 /* Remove all the records. */ 2588 for (i = 0; i < num_found; i++) { 2589 2590 srv_resp = (sa_service_record_t *) 2591 ((uchar_t *)results_p + 2592 i * sizeof (sa_service_record_t)); 2593 2594 /* 2595 * Found some matching records, but check out whether 2596 * this Record is really stale or just happens to match 2597 * the current session records. If yes, don't remove it. 2598 */ 2599 mutex_enter(&ibcm_svc_info_lock); 2600 if (ibcm_find_svc_entry(srv_resp->ServiceID) != NULL) { 2601 /* This record is NOT STALE. */ 2602 mutex_exit(&ibcm_svc_info_lock); 2603 IBTF_DPRINTF_L3(cmlog, "ibcm_rem_stale_srec: " 2604 "This is not Stale, it's an active record"); 2605 continue; 2606 } 2607 mutex_exit(&ibcm_svc_info_lock); 2608 2609 IBTF_DPRINTF_L2(cmlog, "ibcm_rem_stale_srec: " 2610 "Removing Stale Rec: %s, %llX", 2611 srv_resp->ServiceName, srv_resp->ServiceID); 2612 2613 IBCM_DUMP_SERVICE_REC(srv_resp); 2614 2615 /* 2616 * Remove the Service Record Entry from SA. 2617 * 2618 * Get ServiceID info from Response Buf, other 2619 * attributes are already filled-in. 2620 */ 2621 2622 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(srec->ServiceID)) 2623 2624 srec->ServiceID = srv_resp->ServiceID; 2625 2626 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(srec->ServiceID)) 2627 2628 (void) ibcm_write_service_record(saa_handle, srec, 2629 IBMF_SAA_DELETE); 2630 } 2631 2632 /* Deallocate the memory for results_p. */ 2633 kmem_free(results_p, length); 2634 } 2635 } 2636 2637 2638 2639 /* 2640 * ibt_bind_service() 2641 * Register a service with the IBCM 2642 * 2643 * INPUTS: 2644 * srv_hdl The service id handle returned to the client 2645 * on an ibt_service_register() call. 2646 * 2647 * gid The GID to which to bind the service. 2648 * 2649 * srv_bind The address of a ibt_srv_bind_t that describes 2650 * the service record. This should be NULL if there 2651 * is to be no service record. This contains: 2652 * 2653 * sb_lease Lease period 2654 * sb_pkey Partition 2655 * sb_name pointer to ASCII string Service Name, 2656 * NULL terminated. 2657 * sb_key[] Key to secure the service record. 2658 * sb_data Service Data structure (64-byte) 2659 * 2660 * cm_private First argument of Service handler. 2661 * 2662 * sb_hdl_p The address of a service bind handle, used 2663 * to undo the service binding. 2664 * 2665 * ibt_bind_service() returns: 2666 * IBT_SUCCESS - added a service successfully. 2667 * IBT_INVALID_PARAM - invalid input parameter. 2668 * IBT_CM_FAILURE - failed to add the service. 2669 * IBT_CM_SERVICE_EXISTS - service already exists. 2670 */ 2671 ibt_status_t 2672 ibt_bind_service(ibt_srv_hdl_t srv_hdl, ib_gid_t gid, ibt_srv_bind_t *srv_bind, 2673 void *cm_private, ibt_sbind_hdl_t *sb_hdl_p) 2674 { 2675 ibt_status_t status; 2676 ibtl_cm_hca_port_t port; 2677 ibcm_svc_bind_t *sbindp, *sbp; 2678 ibcm_hca_info_t *hcap; 2679 ib_svc_id_t sid, start_sid, end_sid; 2680 ibmf_saa_handle_t saa_handle; 2681 sa_service_record_t srv_rec; 2682 uint16_t pkey_ix; 2683 2684 if (sb_hdl_p != NULL) 2685 *sb_hdl_p = NULL; /* return value for error cases */ 2686 2687 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: srv_hdl %p, gid (%llX:%llX)", 2688 srv_hdl, (longlong_t)gid.gid_prefix, (longlong_t)gid.gid_guid); 2689 2690 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sbindp)) 2691 2692 /* Call ibtl_cm_get_hca_port to get the port number and the HCA GUID. */ 2693 if ((status = ibtl_cm_get_hca_port(gid, 0, &port)) != IBT_SUCCESS) { 2694 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2695 "ibtl_cm_get_hca_port failed: %d", status); 2696 return (status); 2697 } 2698 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: Port:%d HCA GUID:%llX", 2699 port.hp_port, port.hp_hca_guid); 2700 2701 hcap = ibcm_find_hca_entry(port.hp_hca_guid); 2702 if (hcap == NULL) { 2703 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: NO HCA found"); 2704 return (IBT_HCA_BUSY_DETACHING); 2705 } 2706 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: hcap = %p", hcap); 2707 2708 if (srv_bind != NULL) { 2709 saa_handle = ibcm_get_saa_handle(hcap, port.hp_port); 2710 if (saa_handle == NULL) { 2711 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2712 "saa_handle is NULL"); 2713 ibcm_dec_hca_acc_cnt(hcap); 2714 return (IBT_HCA_PORT_NOT_ACTIVE); 2715 } 2716 if (srv_bind->sb_pkey == 0) { 2717 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2718 "P_Key must not be 0"); 2719 ibcm_dec_hca_acc_cnt(hcap); 2720 return (IBT_INVALID_PARAM); 2721 } 2722 if (strlen(srv_bind->sb_name) >= IB_SVC_NAME_LEN) { 2723 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2724 "Service Name is too long"); 2725 ibcm_dec_hca_acc_cnt(hcap); 2726 return (IBT_INVALID_PARAM); 2727 } else 2728 IBTF_DPRINTF_L3(cmlog, "ibt_bind_service: " 2729 "Service Name='%s'", srv_bind->sb_name); 2730 status = ibt_pkey2index_byguid(port.hp_hca_guid, 2731 port.hp_port, srv_bind->sb_pkey, &pkey_ix); 2732 if (status != IBT_SUCCESS) { 2733 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2734 "P_Key 0x%x not found in P_Key_Table", 2735 srv_bind->sb_pkey); 2736 ibcm_dec_hca_acc_cnt(hcap); 2737 return (status); 2738 } 2739 } 2740 2741 /* assume success - allocate before locking */ 2742 sbindp = kmem_zalloc(sizeof (*sbindp), KM_SLEEP); 2743 sbindp->sbind_cm_private = cm_private; 2744 sbindp->sbind_gid = gid; 2745 sbindp->sbind_hcaguid = port.hp_hca_guid; 2746 sbindp->sbind_port = port.hp_port; 2747 2748 mutex_enter(&ibcm_svc_info_lock); 2749 2750 sbp = srv_hdl->svc_bind_list; 2751 while (sbp != NULL) { 2752 if (sbp->sbind_gid.gid_guid == gid.gid_guid && 2753 sbp->sbind_gid.gid_prefix == gid.gid_prefix) { 2754 if (srv_bind == NULL || 2755 srv_bind->sb_pkey == sbp->sbind_pkey) { 2756 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2757 "failed: GID %llX:%llX and PKEY %x is " 2758 "already bound", gid.gid_prefix, 2759 gid.gid_guid, sbp->sbind_pkey); 2760 mutex_exit(&ibcm_svc_info_lock); 2761 ibcm_dec_hca_acc_cnt(hcap); 2762 kmem_free(sbindp, sizeof (*sbindp)); 2763 return (IBT_CM_SERVICE_EXISTS); 2764 } 2765 } 2766 sbp = sbp->sbind_link; 2767 } 2768 /* no entry found */ 2769 2770 sbindp->sbind_link = srv_hdl->svc_bind_list; 2771 srv_hdl->svc_bind_list = sbindp; 2772 2773 mutex_exit(&ibcm_svc_info_lock); 2774 2775 if (srv_bind != NULL) { 2776 bzero(&srv_rec, sizeof (srv_rec)); 2777 2778 srv_rec.ServiceLease = 2779 sbindp->sbind_lease = srv_bind->sb_lease; 2780 srv_rec.ServiceP_Key = 2781 sbindp->sbind_pkey = srv_bind->sb_pkey; 2782 srv_rec.ServiceKey_hi = 2783 sbindp->sbind_key[0] = srv_bind->sb_key[0]; 2784 srv_rec.ServiceKey_lo = 2785 sbindp->sbind_key[1] = srv_bind->sb_key[1]; 2786 (void) strcpy(sbindp->sbind_name, srv_bind->sb_name); 2787 (void) strcpy((char *)srv_rec.ServiceName, srv_bind->sb_name); 2788 srv_rec.ServiceGID = gid; 2789 2790 /* 2791 * Find out whether we have any stale Local Service records 2792 * matching the current attributes. If yes, we shall try to 2793 * remove them from SA using the current request's ServiceKey. 2794 * 2795 * We will perform this operation only for Local Services, as 2796 * it is handled by SA automatically for WellKnown Services. 2797 * 2798 * Ofcourse, clients can specify NOT to do this clean-up by 2799 * setting IBT_SBIND_NO_CLEANUP flag (srv_bind->sb_flag). 2800 */ 2801 if ((srv_hdl->svc_id & IB_SID_AGN_LOCAL) && 2802 (!(srv_bind->sb_flag & IBT_SBIND_NO_CLEANUP))) { 2803 ibcm_rem_stale_srec(saa_handle, &srv_rec); 2804 } 2805 2806 /* Handle endianess for service data. */ 2807 ibcm_swizzle_from_srv(&srv_bind->sb_data, sbindp->sbind_data); 2808 2809 bcopy(sbindp->sbind_data, srv_rec.ServiceData, IB_SVC_DATA_LEN); 2810 2811 /* insert srv record into the SA */ 2812 start_sid = srv_hdl->svc_id; 2813 end_sid = start_sid + srv_hdl->svc_num_sids - 1; 2814 for (sid = start_sid; sid <= end_sid; sid++) { 2815 2816 srv_rec.ServiceID = sid; 2817 2818 IBCM_DUMP_SERVICE_REC(&srv_rec); 2819 2820 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: " 2821 "ibmf_saa_write_service_record, SvcId = %llX", 2822 (longlong_t)sid); 2823 2824 status = ibcm_write_service_record(saa_handle, &srv_rec, 2825 IBMF_SAA_UPDATE); 2826 if (status != IBT_SUCCESS) { 2827 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service:" 2828 " ibcm_write_service_record fails %d, " 2829 "sid %llX", status, (longlong_t)sid); 2830 2831 if (sid != start_sid) { 2832 /* 2833 * Bind failed while bind SID other than 2834 * first in the sid_range. So we need 2835 * to unbind those, which are passed. 2836 * 2837 * Need to increment svc count to 2838 * compensate for ibt_unbind_service(). 2839 */ 2840 ibcm_inc_hca_svc_cnt(hcap); 2841 ibcm_dec_hca_acc_cnt(hcap); 2842 2843 (void) ibt_unbind_service(srv_hdl, 2844 sbindp); 2845 } else { 2846 ibcm_svc_bind_t **sbpp; 2847 2848 /* 2849 * Bind failed for the first SID or the 2850 * only SID in question, then no need 2851 * to unbind, just free memory and 2852 * return error. 2853 */ 2854 mutex_enter(&ibcm_svc_info_lock); 2855 2856 sbpp = &srv_hdl->svc_bind_list; 2857 sbp = *sbpp; 2858 while (sbp != NULL) { 2859 if (sbp == sbindp) { 2860 *sbpp = sbp->sbind_link; 2861 break; 2862 } 2863 sbpp = &sbp->sbind_link; 2864 sbp = *sbpp; 2865 } 2866 mutex_exit(&ibcm_svc_info_lock); 2867 ibcm_dec_hca_acc_cnt(hcap); 2868 2869 kmem_free(sbindp, sizeof (*sbindp)); 2870 } 2871 return (status); 2872 } 2873 } 2874 } 2875 ibcm_inc_hca_svc_cnt(hcap); 2876 ibcm_dec_hca_acc_cnt(hcap); 2877 2878 /* If this message isn't seen then ibt_bind_service failed */ 2879 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: DONE (%p, %llX:%llX)", 2880 srv_hdl, gid.gid_prefix, gid.gid_guid); 2881 2882 if (sb_hdl_p != NULL) 2883 *sb_hdl_p = sbindp; 2884 2885 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sbindp)) 2886 2887 return (IBT_SUCCESS); 2888 } 2889 2890 ibt_status_t 2891 ibt_unbind_service(ibt_srv_hdl_t srv_hdl, ibt_sbind_hdl_t sbindp) 2892 { 2893 ib_svc_id_t sid, end_sid; 2894 ibt_status_t rval; 2895 ibcm_hca_info_t *hcap; 2896 ibcm_svc_bind_t *sbp, **sbpp; 2897 2898 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service(%p, %p)", 2899 srv_hdl, sbindp); 2900 2901 hcap = ibcm_find_hca_entry(sbindp->sbind_hcaguid); 2902 2903 /* If there is a service on hca, respective hcap cannot go away */ 2904 ASSERT(hcap != NULL); 2905 2906 mutex_enter(&ibcm_svc_info_lock); 2907 2908 sbpp = &srv_hdl->svc_bind_list; 2909 sbp = *sbpp; 2910 while (sbp != NULL) { 2911 if (sbp == sbindp) { 2912 *sbpp = sbp->sbind_link; 2913 break; 2914 } 2915 sbpp = &sbp->sbind_link; 2916 sbp = *sbpp; 2917 } 2918 sid = srv_hdl->svc_id; 2919 end_sid = srv_hdl->svc_id + srv_hdl->svc_num_sids - 1; 2920 if (sbp != NULL) 2921 while (sbp->sbind_rewrite_state == IBCM_REWRITE_BUSY) 2922 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 2923 mutex_exit(&ibcm_svc_info_lock); 2924 2925 if (sbp == NULL) { 2926 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2927 "service binding not found: srv_hdl %p, srv_bind %p", 2928 srv_hdl, sbindp); 2929 ibcm_dec_hca_acc_cnt(hcap); 2930 return (IBT_INVALID_PARAM); 2931 } 2932 2933 if (sbindp->sbind_pkey != 0) { /* Are there service records? */ 2934 ibtl_cm_hca_port_t port; 2935 sa_service_record_t srv_rec; 2936 ibmf_saa_handle_t saa_handle; 2937 ibt_status_t status; 2938 2939 /* get the default SGID of the port */ 2940 if ((status = ibtl_cm_get_hca_port(sbindp->sbind_gid, 0, &port)) 2941 != IBT_SUCCESS) { 2942 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2943 "ibtl_cm_get_hca_port failed: %d", status); 2944 /* we're done, but there may be stale service records */ 2945 goto done; 2946 } 2947 2948 saa_handle = ibcm_get_saa_handle(hcap, port.hp_port); 2949 if (saa_handle == NULL) { 2950 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2951 "saa_handle is NULL"); 2952 /* we're done, but there may be stale service records */ 2953 goto done; 2954 } 2955 2956 /* Fill in fields of srv_rec */ 2957 bzero(&srv_rec, sizeof (srv_rec)); 2958 2959 srv_rec.ServiceP_Key = sbindp->sbind_pkey; 2960 srv_rec.ServiceKey_hi = sbindp->sbind_key[0]; 2961 srv_rec.ServiceKey_lo = sbindp->sbind_key[1]; 2962 srv_rec.ServiceGID = sbindp->sbind_gid; 2963 (void) strcpy((char *)srv_rec.ServiceName, sbindp->sbind_name); 2964 2965 while (sid <= end_sid) { 2966 2967 srv_rec.ServiceID = sid; 2968 IBCM_DUMP_SERVICE_REC(&srv_rec); 2969 2970 rval = ibcm_write_service_record(saa_handle, &srv_rec, 2971 IBMF_SAA_DELETE); 2972 2973 IBTF_DPRINTF_L4(cmlog, "ibt_unbind_service: " 2974 "ibcm_write_service_record rval = %d, SID %llx", 2975 rval, sid); 2976 if (rval != IBT_SUCCESS) { 2977 /* this is not considered a reason to fail */ 2978 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2979 "ibcm_write_service_record fails %d, " 2980 "sid %llx", rval, sid); 2981 } 2982 sid++; 2983 } 2984 } 2985 done: 2986 ibcm_dec_hca_svc_cnt(hcap); 2987 ibcm_dec_hca_acc_cnt(hcap); 2988 kmem_free(sbindp, sizeof (*sbindp)); 2989 2990 /* If this message isn't seen then ibt_unbind_service failed */ 2991 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: done !!"); 2992 2993 return (IBT_SUCCESS); 2994 } 2995 2996 /* 2997 * Simply pull off each binding from the list and unbind it. 2998 * If any of the unbind calls fail, we fail. 2999 */ 3000 ibt_status_t 3001 ibt_unbind_all_services(ibt_srv_hdl_t srv_hdl) 3002 { 3003 ibt_status_t status; 3004 ibcm_svc_bind_t *sbp; 3005 3006 mutex_enter(&ibcm_svc_info_lock); 3007 sbp = NULL; 3008 3009 /* this compare keeps the loop from being infinite */ 3010 while (sbp != srv_hdl->svc_bind_list) { 3011 sbp = srv_hdl->svc_bind_list; 3012 mutex_exit(&ibcm_svc_info_lock); 3013 status = ibt_unbind_service(srv_hdl, sbp); 3014 if (status != IBT_SUCCESS) 3015 return (status); 3016 mutex_enter(&ibcm_svc_info_lock); 3017 if (srv_hdl->svc_bind_list == NULL) 3018 break; 3019 } 3020 mutex_exit(&ibcm_svc_info_lock); 3021 return (IBT_SUCCESS); 3022 } 3023 3024 /* 3025 * ibt_deregister_service() 3026 * Deregister a service with the IBCM 3027 * 3028 * INPUTS: 3029 * ibt_hdl The IBT client handle returned to the client 3030 * on an ibt_attach() call. 3031 * 3032 * srv_hdl The address of a service identification handle, used 3033 * to de-register a service. 3034 * RETURN VALUES: 3035 * IBT_SUCCESS on success (or respective failure on error) 3036 */ 3037 ibt_status_t 3038 ibt_deregister_service(ibt_clnt_hdl_t ibt_hdl, ibt_srv_hdl_t srv_hdl) 3039 { 3040 ibcm_svc_info_t *svcp; 3041 ibcm_svc_lookup_t svc; 3042 3043 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service(%p (%s), %p)", 3044 ibt_hdl, ibtl_cm_get_clnt_name(ibt_hdl), srv_hdl); 3045 3046 mutex_enter(&ibcm_svc_info_lock); 3047 3048 if (srv_hdl->svc_bind_list != NULL) { 3049 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service:" 3050 " srv_hdl %p still has bindings", srv_hdl); 3051 mutex_exit(&ibcm_svc_info_lock); 3052 return (IBT_CM_SERVICE_BUSY); 3053 } 3054 svc.sid = srv_hdl->svc_id; 3055 svc.num_sids = 1; 3056 IBTF_DPRINTF_L3(cmlog, "ibt_deregister_service: SID 0x%llX, numsids %d", 3057 srv_hdl->svc_id, srv_hdl->svc_num_sids); 3058 3059 #ifdef __lock_lint 3060 ibcm_svc_compare(NULL, NULL); 3061 #endif 3062 svcp = avl_find(&ibcm_svc_avl_tree, &svc, NULL); 3063 if (svcp != srv_hdl) { 3064 mutex_exit(&ibcm_svc_info_lock); 3065 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service(): " 3066 "srv_hdl %p not found", srv_hdl); 3067 return (IBT_INVALID_PARAM); 3068 } 3069 avl_remove(&ibcm_svc_avl_tree, svcp); 3070 3071 /* wait for active REQ/SREQ handling to be done */ 3072 svcp->svc_to_delete = 1; 3073 while (svcp->svc_ref_cnt != 0) 3074 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 3075 3076 mutex_exit(&ibcm_svc_info_lock); 3077 3078 if ((srv_hdl->svc_id & IB_SID_AGN_MASK) == IB_SID_AGN_LOCAL) 3079 ibcm_free_local_sids(srv_hdl->svc_id, srv_hdl->svc_num_sids); 3080 3081 ibtl_cm_change_service_cnt(ibt_hdl, -srv_hdl->svc_num_sids); 3082 kmem_free(srv_hdl, sizeof (*srv_hdl)); 3083 3084 /* If this message isn't seen then ibt_deregister_service failed */ 3085 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service: done !!"); 3086 3087 return (IBT_SUCCESS); 3088 } 3089 3090 ibcm_status_t 3091 ibcm_ar_init(void) 3092 { 3093 ib_svc_id_t sid = IBCM_DAPL_ATS_SID; 3094 ibcm_svc_info_t *tmp_svcp; 3095 3096 IBTF_DPRINTF_L3(cmlog, "ibcm_ar_init()"); 3097 3098 /* remove this special SID from the pool of available SIDs */ 3099 if ((tmp_svcp = ibcm_create_svc_entry(sid, 1)) == NULL) { 3100 IBTF_DPRINTF_L3(cmlog, "ibcm_ar_init: " 3101 "DAPL ATS SID 0x%llx already registered", (longlong_t)sid); 3102 return (IBCM_FAILURE); 3103 } 3104 mutex_enter(&ibcm_svc_info_lock); 3105 ibcm_ar_svcinfop = tmp_svcp; 3106 ibcm_ar_list = NULL; /* no address records registered yet */ 3107 mutex_exit(&ibcm_svc_info_lock); 3108 return (IBCM_SUCCESS); 3109 } 3110 3111 ibcm_status_t 3112 ibcm_ar_fini(void) 3113 { 3114 ibcm_ar_t *ar_list; 3115 ibcm_svc_info_t *tmp_svcp; 3116 3117 mutex_enter(&ibcm_svc_info_lock); 3118 ar_list = ibcm_ar_list; 3119 3120 if (ar_list == NULL && 3121 avl_numnodes(&ibcm_svc_avl_tree) == 1 && 3122 avl_first(&ibcm_svc_avl_tree) == ibcm_ar_svcinfop) { 3123 avl_remove(&ibcm_svc_avl_tree, ibcm_ar_svcinfop); 3124 tmp_svcp = ibcm_ar_svcinfop; 3125 mutex_exit(&ibcm_svc_info_lock); 3126 kmem_free(tmp_svcp, sizeof (*ibcm_ar_svcinfop)); 3127 return (IBCM_SUCCESS); 3128 } 3129 mutex_exit(&ibcm_svc_info_lock); 3130 return (IBCM_FAILURE); 3131 } 3132 3133 3134 /* 3135 * Return to the caller: 3136 * IBT_SUCCESS Found a perfect match. 3137 * *arpp is set to the record. 3138 * IBT_INCONSISTENT_AR Found a record that's inconsistent. 3139 * IBT_AR_NOT_REGISTERED Found no record with same GID/pkey and 3140 * found no record with same data. 3141 */ 3142 static ibt_status_t 3143 ibcm_search_ar(ibt_ar_t *arp, ibcm_ar_t **arpp) 3144 { 3145 ibcm_ar_t *tmp; 3146 int i; 3147 3148 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3149 tmp = ibcm_ar_list; 3150 while (tmp != NULL) { 3151 if (tmp->ar.ar_gid.gid_prefix == arp->ar_gid.gid_prefix && 3152 tmp->ar.ar_gid.gid_guid == arp->ar_gid.gid_guid && 3153 tmp->ar.ar_pkey == arp->ar_pkey) { 3154 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) 3155 if (tmp->ar.ar_data[i] != arp->ar_data[i]) 3156 return (IBT_INCONSISTENT_AR); 3157 *arpp = tmp; 3158 return (IBT_SUCCESS); 3159 } else { 3160 /* if all the data bytes match, we have inconsistency */ 3161 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) 3162 if (tmp->ar.ar_data[i] != arp->ar_data[i]) 3163 break; 3164 if (i == IBCM_DAPL_ATS_NBYTES) 3165 return (IBT_INCONSISTENT_AR); 3166 /* try next address record */ 3167 } 3168 tmp = tmp->ar_link; 3169 } 3170 return (IBT_AR_NOT_REGISTERED); 3171 } 3172 3173 ibt_status_t 3174 ibt_register_ar(ibt_clnt_hdl_t ibt_hdl, ibt_ar_t *arp) 3175 { 3176 ibcm_ar_t *found; 3177 ibcm_ar_t *tmp; 3178 ibt_status_t status; 3179 ibt_status_t s1, s2; 3180 char *s; 3181 ibcm_ar_ref_t *hdlp; 3182 ibcm_ar_t *new; 3183 ibcm_ar_t **linkp; 3184 ibtl_cm_hca_port_t cm_port; 3185 uint16_t pkey_ix; 3186 ibcm_hca_info_t *hcap; 3187 ibmf_saa_handle_t saa_handle; 3188 sa_service_record_t *srv_recp; 3189 uint64_t gid_ored; 3190 3191 IBTF_DPRINTF_L3(cmlog, "ibt_register_ar: PKey 0x%X GID %llX:%llX", 3192 arp->ar_pkey, (longlong_t)arp->ar_gid.gid_prefix, 3193 (longlong_t)arp->ar_gid.gid_guid); 3194 3195 /* 3196 * If P_Key is 0, but GID is not, this query is invalid. 3197 * If GID is 0, but P_Key is not, this query is invalid. 3198 */ 3199 gid_ored = arp->ar_gid.gid_guid | arp->ar_gid.gid_prefix; 3200 if ((arp->ar_pkey == 0 && gid_ored != 0ULL) || 3201 (arp->ar_pkey != 0 && gid_ored == 0ULL)) { 3202 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3203 "GID/P_Key is not valid"); 3204 return (IBT_INVALID_PARAM); 3205 } 3206 3207 /* assume success, so these might be needed */ 3208 hdlp = kmem_alloc(sizeof (*hdlp), KM_SLEEP); 3209 new = kmem_zalloc(sizeof (*new), KM_SLEEP); 3210 3211 mutex_enter(&ibcm_svc_info_lock); 3212 /* search for existing GID/pkey (there can be at most 1) */ 3213 status = ibcm_search_ar(arp, &found); 3214 if (status == IBT_INCONSISTENT_AR) { 3215 mutex_exit(&ibcm_svc_info_lock); 3216 kmem_free(new, sizeof (*new)); 3217 kmem_free(hdlp, sizeof (*hdlp)); 3218 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3219 "address record is inconsistent with a known one"); 3220 return (IBT_INCONSISTENT_AR); 3221 } else if (status == IBT_SUCCESS) { 3222 if (found->ar_flags == IBCM_AR_INITING) { 3223 found->ar_waiters++; 3224 cv_wait(&found->ar_cv, &ibcm_svc_info_lock); 3225 found->ar_waiters--; 3226 } 3227 if (found->ar_flags == IBCM_AR_FAILED) { 3228 if (found->ar_waiters == 0) { 3229 cv_destroy(&found->ar_cv); 3230 kmem_free(found, sizeof (*found)); 3231 } 3232 mutex_exit(&ibcm_svc_info_lock); 3233 kmem_free(new, sizeof (*new)); 3234 kmem_free(hdlp, sizeof (*hdlp)); 3235 return (ibt_get_module_failure(IBT_FAILURE_IBCM, 0)); 3236 } 3237 hdlp->ar_ibt_hdl = ibt_hdl; 3238 hdlp->ar_ref_link = found->ar_ibt_hdl_list; 3239 found->ar_ibt_hdl_list = hdlp; 3240 mutex_exit(&ibcm_svc_info_lock); 3241 kmem_free(new, sizeof (*new)); 3242 ibtl_cm_change_service_cnt(ibt_hdl, 1); 3243 return (IBT_SUCCESS); 3244 } else { 3245 ASSERT(status == IBT_AR_NOT_REGISTERED); 3246 } 3247 hdlp->ar_ref_link = NULL; 3248 hdlp->ar_ibt_hdl = ibt_hdl; 3249 new->ar_ibt_hdl_list = hdlp; 3250 new->ar = *arp; 3251 new->ar_flags = IBCM_AR_INITING; 3252 new->ar_waiters = 0; 3253 cv_init(&new->ar_cv, NULL, CV_DEFAULT, NULL); 3254 new->ar_link = ibcm_ar_list; 3255 ibcm_ar_list = new; 3256 3257 /* verify GID/pkey is valid for a local port, etc. */ 3258 hcap = NULL; 3259 if ((s1 = ibtl_cm_get_hca_port(arp->ar_gid, 0, &cm_port)) 3260 != IBT_SUCCESS || 3261 (s2 = ibt_pkey2index_byguid(cm_port.hp_hca_guid, cm_port.hp_port, 3262 arp->ar_pkey, &pkey_ix)) != IBT_SUCCESS || 3263 (hcap = ibcm_find_hca_entry(cm_port.hp_hca_guid)) == NULL) { 3264 cv_destroy(&new->ar_cv); 3265 ibcm_ar_list = new->ar_link; 3266 mutex_exit(&ibcm_svc_info_lock); 3267 kmem_free(new, sizeof (*new)); 3268 kmem_free(hdlp, sizeof (*hdlp)); 3269 status = IBT_INVALID_PARAM; 3270 if (s1 == IBT_HCA_PORT_NOT_ACTIVE) { 3271 s = "PORT DOWN"; 3272 status = IBT_HCA_PORT_NOT_ACTIVE; 3273 } else if (s1 != IBT_SUCCESS) 3274 s = "GID not found"; 3275 else if (s2 != IBT_SUCCESS) 3276 s = "PKEY not found"; 3277 else 3278 s = "CM could not find its HCA entry"; 3279 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: %s, status = %d", 3280 s, status); 3281 return (status); 3282 } 3283 mutex_exit(&ibcm_svc_info_lock); 3284 saa_handle = ibcm_get_saa_handle(hcap, cm_port.hp_port); 3285 3286 /* create service record */ 3287 srv_recp = kmem_zalloc(sizeof (*srv_recp), KM_SLEEP); 3288 srv_recp->ServiceLease = 0xFFFFFFFF; /* infinite */ 3289 srv_recp->ServiceP_Key = arp->ar_pkey; 3290 srv_recp->ServiceKey_hi = 0xDA410000ULL; /* DAPL */ 3291 srv_recp->ServiceKey_lo = 0xA7500000ULL; /* ATS */ 3292 (void) strcpy((char *)srv_recp->ServiceName, IBCM_DAPL_ATS_NAME); 3293 srv_recp->ServiceGID = arp->ar_gid; 3294 bcopy(arp->ar_data, srv_recp->ServiceData, IBCM_DAPL_ATS_NBYTES); 3295 srv_recp->ServiceID = IBCM_DAPL_ATS_SID; 3296 3297 /* insert service record into the SA */ 3298 3299 IBCM_DUMP_SERVICE_REC(srv_recp); 3300 3301 if (saa_handle != NULL) 3302 status = ibcm_write_service_record(saa_handle, srv_recp, 3303 IBMF_SAA_UPDATE); 3304 else 3305 status = IBT_HCA_PORT_NOT_ACTIVE; 3306 3307 if (status != IBT_SUCCESS) { 3308 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: sa access fails %d, " 3309 "sid %llX", status, (longlong_t)srv_recp->ServiceID); 3310 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: FAILED for gid " 3311 "%llX:%llX pkey 0x%X", (longlong_t)arp->ar_gid.gid_prefix, 3312 (longlong_t)arp->ar_gid.gid_guid, arp->ar_pkey); 3313 3314 kmem_free(srv_recp, sizeof (*srv_recp)); 3315 kmem_free(hdlp, sizeof (*hdlp)); 3316 3317 mutex_enter(&ibcm_svc_info_lock); 3318 linkp = &ibcm_ar_list; 3319 tmp = *linkp; 3320 while (tmp != NULL) { 3321 if (tmp == new) { 3322 *linkp = new->ar_link; 3323 break; 3324 } 3325 linkp = &tmp->ar_link; 3326 tmp = *linkp; 3327 } 3328 if (new->ar_waiters > 0) { 3329 new->ar_flags = IBCM_AR_FAILED; 3330 cv_broadcast(&new->ar_cv); 3331 mutex_exit(&ibcm_svc_info_lock); 3332 } else { 3333 cv_destroy(&new->ar_cv); 3334 mutex_exit(&ibcm_svc_info_lock); 3335 kmem_free(new, sizeof (*new)); 3336 } 3337 ibcm_dec_hca_acc_cnt(hcap); 3338 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3339 "IBMF_SAA failed to write address record"); 3340 } else { /* SUCCESS */ 3341 uint8_t *b; 3342 3343 IBTF_DPRINTF_L3(cmlog, "ibt_register_ar: SUCCESS for gid " 3344 "%llx:%llx pkey %x", (longlong_t)arp->ar_gid.gid_prefix, 3345 (longlong_t)arp->ar_gid.gid_guid, arp->ar_pkey); 3346 b = arp->ar_data; 3347 3348 IBTF_DPRINTF_L3(cmlog, "ibt_register_ar:" 3349 " data %d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d", 3350 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], 3351 b[10], b[11], b[12], b[13], b[14], b[15]); 3352 mutex_enter(&ibcm_svc_info_lock); 3353 new->ar_srv_recp = srv_recp; 3354 new->ar_saa_handle = saa_handle; 3355 new->ar_port = cm_port.hp_port; 3356 new->ar_hcap = hcap; 3357 new->ar_flags = IBCM_AR_SUCCESS; 3358 if (new->ar_waiters > 0) 3359 cv_broadcast(&new->ar_cv); 3360 mutex_exit(&ibcm_svc_info_lock); 3361 ibtl_cm_change_service_cnt(ibt_hdl, 1); 3362 /* do not call ibcm_dec_hca_acc_cnt(hcap) until deregister */ 3363 } 3364 return (status); 3365 } 3366 3367 ibt_status_t 3368 ibt_deregister_ar(ibt_clnt_hdl_t ibt_hdl, ibt_ar_t *arp) 3369 { 3370 ibcm_ar_t *found; 3371 ibcm_ar_t *tmp; 3372 ibcm_ar_t **linkp; 3373 ibcm_ar_ref_t *hdlp; 3374 ibcm_ar_ref_t **hdlpp; 3375 ibt_status_t status; 3376 ibmf_saa_handle_t saa_handle; 3377 sa_service_record_t *srv_recp; 3378 uint64_t gid_ored; 3379 3380 IBTF_DPRINTF_L3(cmlog, "ibt_deregister_ar: pkey %x", arp->ar_pkey); 3381 IBTF_DPRINTF_L3(cmlog, "ibt_deregister_ar: gid %llx:%llx", 3382 (longlong_t)arp->ar_gid.gid_prefix, 3383 (longlong_t)arp->ar_gid.gid_guid); 3384 3385 /* 3386 * If P_Key is 0, but GID is not, this query is invalid. 3387 * If GID is 0, but P_Key is not, this query is invalid. 3388 */ 3389 gid_ored = arp->ar_gid.gid_guid | arp->ar_gid.gid_prefix; 3390 if ((arp->ar_pkey == 0 && gid_ored != 0ULL) || 3391 (arp->ar_pkey != 0 && gid_ored == 0ULL)) { 3392 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3393 "GID/P_Key is not valid"); 3394 return (IBT_INVALID_PARAM); 3395 } 3396 3397 mutex_enter(&ibcm_svc_info_lock); 3398 /* search for existing GID/pkey (there can be at most 1) */ 3399 status = ibcm_search_ar(arp, &found); 3400 if (status == IBT_INCONSISTENT_AR || status == IBT_AR_NOT_REGISTERED) { 3401 mutex_exit(&ibcm_svc_info_lock); 3402 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3403 "address record not found"); 3404 return (IBT_AR_NOT_REGISTERED); 3405 } 3406 ASSERT(status == IBT_SUCCESS); 3407 3408 hdlpp = &found->ar_ibt_hdl_list; 3409 hdlp = *hdlpp; 3410 while (hdlp != NULL) { 3411 if (hdlp->ar_ibt_hdl == ibt_hdl) 3412 break; 3413 hdlpp = &hdlp->ar_ref_link; 3414 hdlp = *hdlpp; 3415 } 3416 if (hdlp == NULL) { /* could not find ibt_hdl on list */ 3417 mutex_exit(&ibcm_svc_info_lock); 3418 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3419 "address record found, but not for this client"); 3420 return (IBT_AR_NOT_REGISTERED); 3421 } 3422 *hdlpp = hdlp->ar_ref_link; /* remove ref for this client */ 3423 if (found->ar_ibt_hdl_list == NULL && found->ar_waiters == 0) { 3424 /* last entry was removed */ 3425 found->ar_flags = IBCM_AR_INITING; /* hold off register_ar */ 3426 saa_handle = found->ar_saa_handle; 3427 srv_recp = found->ar_srv_recp; 3428 3429 /* wait if this service record is being rewritten */ 3430 while (found->ar_rewrite_state == IBCM_REWRITE_BUSY) 3431 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 3432 mutex_exit(&ibcm_svc_info_lock); 3433 3434 /* remove service record */ 3435 status = ibcm_write_service_record(saa_handle, srv_recp, 3436 IBMF_SAA_DELETE); 3437 if (status != IBT_SUCCESS) 3438 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3439 "IBMF_SAA failed to delete address record"); 3440 mutex_enter(&ibcm_svc_info_lock); 3441 if (found->ar_waiters == 0) { /* still no waiters */ 3442 linkp = &ibcm_ar_list; 3443 tmp = *linkp; 3444 while (tmp != found) { 3445 linkp = &tmp->ar_link; 3446 tmp = *linkp; 3447 } 3448 *linkp = tmp->ar_link; 3449 ibcm_dec_hca_acc_cnt(found->ar_hcap); 3450 kmem_free(srv_recp, sizeof (*srv_recp)); 3451 cv_destroy(&found->ar_cv); 3452 kmem_free(found, sizeof (*found)); 3453 } else { 3454 /* add service record back in for the waiters */ 3455 mutex_exit(&ibcm_svc_info_lock); 3456 status = ibcm_write_service_record(saa_handle, srv_recp, 3457 IBMF_SAA_UPDATE); 3458 mutex_enter(&ibcm_svc_info_lock); 3459 if (status == IBT_SUCCESS) 3460 found->ar_flags = IBCM_AR_SUCCESS; 3461 else { 3462 found->ar_flags = IBCM_AR_FAILED; 3463 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3464 "IBMF_SAA failed to write address record"); 3465 } 3466 cv_broadcast(&found->ar_cv); 3467 } 3468 } 3469 mutex_exit(&ibcm_svc_info_lock); 3470 kmem_free(hdlp, sizeof (*hdlp)); 3471 ibtl_cm_change_service_cnt(ibt_hdl, -1); 3472 return (status); 3473 } 3474 3475 ibt_status_t 3476 ibt_query_ar(ib_gid_t *sgid, ibt_ar_t *queryp, ibt_ar_t *resultp) 3477 { 3478 sa_service_record_t svcrec_req; 3479 sa_service_record_t *svcrec_resp; 3480 void *results_p; 3481 uint64_t component_mask = 0; 3482 uint64_t gid_ored; 3483 size_t length; 3484 int num_rec; 3485 int i; 3486 ibmf_saa_access_args_t access_args; 3487 ibt_status_t retval; 3488 ibtl_cm_hca_port_t cm_port; 3489 ibcm_hca_info_t *hcap; 3490 ibmf_saa_handle_t saa_handle; 3491 3492 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar(%p, %p)", queryp, resultp); 3493 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: sgid %llx:%llx", 3494 (longlong_t)sgid->gid_prefix, (longlong_t)sgid->gid_guid); 3495 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: query_pkey %x", queryp->ar_pkey); 3496 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: query_gid %llx:%llx", 3497 (longlong_t)queryp->ar_gid.gid_prefix, 3498 (longlong_t)queryp->ar_gid.gid_guid); 3499 3500 /* 3501 * If P_Key is 0, but GID is not, this query is invalid. 3502 * If GID is 0, but P_Key is not, this query is invalid. 3503 */ 3504 gid_ored = queryp->ar_gid.gid_guid | queryp->ar_gid.gid_prefix; 3505 if ((queryp->ar_pkey == 0 && gid_ored != 0ULL) || 3506 (queryp->ar_pkey != 0 && gid_ored == 0ULL)) { 3507 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: GID/P_Key is not valid"); 3508 return (IBT_INVALID_PARAM); 3509 } 3510 3511 hcap = NULL; 3512 if (ibtl_cm_get_hca_port(*sgid, 0, &cm_port) != IBT_SUCCESS || 3513 (hcap = ibcm_find_hca_entry(cm_port.hp_hca_guid)) == NULL || 3514 (saa_handle = ibcm_get_saa_handle(hcap, cm_port.hp_port)) == NULL) { 3515 if (hcap != NULL) 3516 ibcm_dec_hca_acc_cnt(hcap); 3517 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: sgid is not valid"); 3518 return (IBT_INVALID_PARAM); 3519 } 3520 3521 bzero(&svcrec_req, sizeof (svcrec_req)); 3522 3523 /* Is GID/P_Key Specified. */ 3524 if (queryp->ar_pkey != 0) { /* GID is non-zero from check above */ 3525 svcrec_req.ServiceP_Key = queryp->ar_pkey; 3526 component_mask |= SA_SR_COMPMASK_PKEY; 3527 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: P_Key %X", 3528 queryp->ar_pkey); 3529 svcrec_req.ServiceGID = queryp->ar_gid; 3530 component_mask |= SA_SR_COMPMASK_GID; 3531 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: GID %llX:%llX", 3532 (longlong_t)queryp->ar_gid.gid_prefix, 3533 (longlong_t)queryp->ar_gid.gid_guid); 3534 } 3535 3536 /* Is ServiceData Specified. */ 3537 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) { 3538 if (queryp->ar_data[i] != 0) { 3539 bcopy(queryp->ar_data, svcrec_req.ServiceData, 3540 IBCM_DAPL_ATS_NBYTES); 3541 component_mask |= 0xFFFF << 7; /* all 16 Data8 */ 3542 /* components */ 3543 break; 3544 } 3545 } 3546 3547 /* Service Name */ 3548 (void) strcpy((char *)svcrec_req.ServiceName, IBCM_DAPL_ATS_NAME); 3549 component_mask |= SA_SR_COMPMASK_NAME; 3550 3551 svcrec_req.ServiceID = IBCM_DAPL_ATS_SID; 3552 component_mask |= SA_SR_COMPMASK_ID; 3553 3554 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: " 3555 "Perform SA Access: Mask: 0x%X", component_mask); 3556 3557 /* 3558 * Call in SA Access retrieve routine to get Service Records. 3559 * 3560 * SA Access framework allocated memory for the "results_p". 3561 * Make sure to deallocate once we are done with the results_p. 3562 * The size of the buffer allocated will be as returned in 3563 * "length" field. 3564 */ 3565 access_args.sq_attr_id = SA_SERVICERECORD_ATTRID; 3566 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 3567 access_args.sq_component_mask = component_mask; 3568 access_args.sq_template = &svcrec_req; 3569 access_args.sq_template_length = sizeof (sa_service_record_t); 3570 access_args.sq_callback = NULL; 3571 access_args.sq_callback_arg = NULL; 3572 3573 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 3574 &results_p); 3575 3576 ibcm_dec_hca_acc_cnt(hcap); 3577 if (retval != IBT_SUCCESS) { 3578 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: SA Access Failed"); 3579 return (retval); 3580 } 3581 3582 num_rec = length / sizeof (sa_service_record_t); 3583 3584 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: " 3585 "Found %d Service Records.", num_rec); 3586 3587 /* Validate the returned number of records. */ 3588 if ((results_p != NULL) && (num_rec > 0)) { 3589 uint8_t *b; 3590 3591 /* Just return info from the first service record. */ 3592 svcrec_resp = (sa_service_record_t *)results_p; 3593 3594 /* The Service GID and Service ID */ 3595 resultp->ar_gid = svcrec_resp->ServiceGID; 3596 resultp->ar_pkey = svcrec_resp->ServiceP_Key; 3597 bcopy(svcrec_resp->ServiceData, 3598 resultp->ar_data, IBCM_DAPL_ATS_NBYTES); 3599 3600 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: " 3601 "Found: pkey %x dgid %llX:%llX", resultp->ar_pkey, 3602 (longlong_t)resultp->ar_gid.gid_prefix, 3603 (longlong_t)resultp->ar_gid.gid_guid); 3604 b = resultp->ar_data; 3605 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar:" 3606 " data %d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d", 3607 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], 3608 b[10], b[11], b[12], b[13], b[14], b[15]); 3609 3610 /* Deallocate the memory for results_p. */ 3611 kmem_free(results_p, length); 3612 if (num_rec > 1) 3613 retval = IBT_MULTIPLE_AR; 3614 else 3615 retval = IBT_SUCCESS; 3616 } else { 3617 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: " 3618 "ibmf_sa_access found 0 matching records"); 3619 retval = IBT_AR_NOT_REGISTERED; 3620 } 3621 return (retval); 3622 } 3623 3624 /* mark all ATS service records associated with the port */ 3625 static void 3626 ibcm_mark_ar(ib_guid_t hca_guid, uint8_t port) 3627 { 3628 ibcm_ar_t *tmp; 3629 3630 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3631 for (tmp = ibcm_ar_list; tmp != NULL; tmp = tmp->ar_link) { 3632 if (tmp->ar_hcap == NULL) 3633 continue; 3634 if (tmp->ar_hcap->hca_guid == hca_guid && 3635 tmp->ar_port == port) { 3636 /* even if it's busy, we mark it for rewrite */ 3637 tmp->ar_rewrite_state = IBCM_REWRITE_NEEDED; 3638 } 3639 } 3640 } 3641 3642 /* rewrite all ATS service records */ 3643 static int 3644 ibcm_rewrite_ar(void) 3645 { 3646 ibcm_ar_t *tmp; 3647 ibmf_saa_handle_t saa_handle; 3648 sa_service_record_t *srv_recp; 3649 ibt_status_t rval; 3650 int did_something = 0; 3651 3652 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3653 check_for_work: 3654 for (tmp = ibcm_ar_list; tmp != NULL; tmp = tmp->ar_link) { 3655 if (tmp->ar_rewrite_state == IBCM_REWRITE_NEEDED) { 3656 tmp->ar_rewrite_state = IBCM_REWRITE_BUSY; 3657 saa_handle = tmp->ar_saa_handle; 3658 srv_recp = tmp->ar_srv_recp; 3659 mutex_exit(&ibcm_svc_info_lock); 3660 IBTF_DPRINTF_L3(cmlog, "ibcm_rewrite_ar: " 3661 "rewriting ar @ %p", tmp); 3662 did_something = 1; 3663 rval = ibcm_write_service_record(saa_handle, srv_recp, 3664 IBMF_SAA_UPDATE); 3665 if (rval != IBT_SUCCESS) 3666 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_ar: " 3667 "ibcm_write_service_record failed: " 3668 "status = %d", rval); 3669 mutex_enter(&ibcm_svc_info_lock); 3670 /* if it got marked again, then we want to rewrite */ 3671 if (tmp->ar_rewrite_state == IBCM_REWRITE_BUSY) 3672 tmp->ar_rewrite_state = IBCM_REWRITE_IDLE; 3673 /* in case there was a waiter... */ 3674 cv_broadcast(&ibcm_svc_info_cv); 3675 goto check_for_work; 3676 } 3677 } 3678 return (did_something); 3679 } 3680 3681 static void 3682 ibcm_rewrite_svc_record(ibcm_svc_info_t *srv_hdl, ibcm_svc_bind_t *sbindp) 3683 { 3684 ibcm_hca_info_t *hcap; 3685 ib_svc_id_t sid, start_sid, end_sid; 3686 ibmf_saa_handle_t saa_handle; 3687 sa_service_record_t srv_rec; 3688 ibt_status_t rval; 3689 3690 hcap = ibcm_find_hca_entry(sbindp->sbind_hcaguid); 3691 if (hcap == NULL) { 3692 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record: " 3693 "NO HCA found for HCA GUID %llX", sbindp->sbind_hcaguid); 3694 return; 3695 } 3696 3697 saa_handle = ibcm_get_saa_handle(hcap, sbindp->sbind_port); 3698 if (saa_handle == NULL) { 3699 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record: " 3700 "saa_handle is NULL"); 3701 ibcm_dec_hca_acc_cnt(hcap); 3702 return; 3703 } 3704 3705 IBTF_DPRINTF_L3(cmlog, "ibcm_rewrite_svc_record: " 3706 "rewriting svc '%s', port_guid = %llX", sbindp->sbind_name, 3707 sbindp->sbind_gid.gid_guid); 3708 3709 bzero(&srv_rec, sizeof (srv_rec)); 3710 3711 srv_rec.ServiceLease = sbindp->sbind_lease; 3712 srv_rec.ServiceP_Key = sbindp->sbind_pkey; 3713 srv_rec.ServiceKey_hi = sbindp->sbind_key[0]; 3714 srv_rec.ServiceKey_lo = sbindp->sbind_key[1]; 3715 (void) strcpy((char *)srv_rec.ServiceName, sbindp->sbind_name); 3716 srv_rec.ServiceGID = sbindp->sbind_gid; 3717 3718 bcopy(sbindp->sbind_data, srv_rec.ServiceData, IB_SVC_DATA_LEN); 3719 3720 /* insert srv record into the SA */ 3721 start_sid = srv_hdl->svc_id; 3722 end_sid = start_sid + srv_hdl->svc_num_sids - 1; 3723 for (sid = start_sid; sid <= end_sid; sid++) { 3724 srv_rec.ServiceID = sid; 3725 3726 rval = ibcm_write_service_record(saa_handle, &srv_rec, 3727 IBMF_SAA_UPDATE); 3728 3729 IBTF_DPRINTF_L4(cmlog, "ibcm_rewrite_svc_record: " 3730 "ibcm_write_service_record, SvcId = %llX, " 3731 "rval = %d", (longlong_t)sid, rval); 3732 if (rval != IBT_SUCCESS) { 3733 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record:" 3734 " ibcm_write_service_record fails %d sid %llX", 3735 rval, (longlong_t)sid); 3736 } 3737 } 3738 ibcm_dec_hca_acc_cnt(hcap); 3739 } 3740 3741 /* 3742 * Task to mark all service records as needing to be rewritten to the SM/SA. 3743 * This task does not return until all of them have been rewritten. 3744 */ 3745 void 3746 ibcm_service_record_rewrite_task(void *arg) 3747 { 3748 ibcm_port_up_t *pup = (ibcm_port_up_t *)arg; 3749 ib_guid_t hca_guid = pup->pup_hca_guid; 3750 uint8_t port = pup->pup_port; 3751 ibcm_svc_info_t *svcp; 3752 ibcm_svc_bind_t *sbp; 3753 avl_tree_t *avl_tree = &ibcm_svc_avl_tree; 3754 static int task_is_running = 0; 3755 3756 IBTF_DPRINTF_L3(cmlog, "ibcm_service_record_rewrite_task STARTED " 3757 "for hca_guid %llX, port %d", hca_guid, port); 3758 3759 mutex_enter(&ibcm_svc_info_lock); 3760 ibcm_mark_ar(hca_guid, port); 3761 for (svcp = avl_first(avl_tree); svcp != NULL; 3762 svcp = avl_walk(avl_tree, svcp, AVL_AFTER)) { 3763 sbp = svcp->svc_bind_list; 3764 while (sbp != NULL) { 3765 if (sbp->sbind_pkey != 0 && 3766 sbp->sbind_port == port && 3767 sbp->sbind_hcaguid == hca_guid) { 3768 /* even if it's busy, we mark it for rewrite */ 3769 sbp->sbind_rewrite_state = IBCM_REWRITE_NEEDED; 3770 } 3771 sbp = sbp->sbind_link; 3772 } 3773 } 3774 if (task_is_running) { 3775 /* let the other task thread finish the work */ 3776 mutex_exit(&ibcm_svc_info_lock); 3777 return; 3778 } 3779 task_is_running = 1; 3780 3781 (void) ibcm_rewrite_ar(); 3782 3783 check_for_work: 3784 for (svcp = avl_first(avl_tree); svcp != NULL; 3785 svcp = avl_walk(avl_tree, svcp, AVL_AFTER)) { 3786 sbp = svcp->svc_bind_list; 3787 while (sbp != NULL) { 3788 if (sbp->sbind_rewrite_state == IBCM_REWRITE_NEEDED) { 3789 sbp->sbind_rewrite_state = IBCM_REWRITE_BUSY; 3790 mutex_exit(&ibcm_svc_info_lock); 3791 ibcm_rewrite_svc_record(svcp, sbp); 3792 mutex_enter(&ibcm_svc_info_lock); 3793 /* if it got marked again, we want to rewrite */ 3794 if (sbp->sbind_rewrite_state == 3795 IBCM_REWRITE_BUSY) 3796 sbp->sbind_rewrite_state = 3797 IBCM_REWRITE_IDLE; 3798 /* in case there was a waiter... */ 3799 cv_broadcast(&ibcm_svc_info_cv); 3800 goto check_for_work; 3801 } 3802 sbp = sbp->sbind_link; 3803 } 3804 } 3805 /* 3806 * If there were no service records to write, and we failed to 3807 * have to rewrite any more ATS service records, then we're done. 3808 */ 3809 if (ibcm_rewrite_ar() != 0) 3810 goto check_for_work; 3811 task_is_running = 0; 3812 mutex_exit(&ibcm_svc_info_lock); 3813 3814 IBTF_DPRINTF_L3(cmlog, "ibcm_service_record_rewrite_task DONE"); 3815 kmem_free(pup, sizeof (ibcm_port_up_t)); 3816 } 3817 3818 ibt_status_t 3819 ibt_ofuvcm_get_req_data(void *session_id, ibt_ofuvcm_req_data_t *req_data) 3820 { 3821 ibcm_state_data_t *statep = (ibcm_state_data_t *)session_id; 3822 ibcm_req_msg_t *req_msgp; 3823 3824 IBTF_DPRINTF_L3(cmlog, "ibt_get_ofuvcm_req_data: session_id %p", 3825 session_id); 3826 mutex_enter(&statep->state_mutex); 3827 if ((statep->state != IBCM_STATE_REQ_RCVD) && 3828 (statep->state != IBCM_STATE_MRA_SENT)) { 3829 IBTF_DPRINTF_L2(cmlog, "ibt_get_ofuvcm_req_data: Invalid " 3830 "State %x", statep->state); 3831 mutex_exit(&statep->state_mutex); 3832 return (IBT_CHAN_STATE_INVALID); 3833 } 3834 if (statep->mode == IBCM_ACTIVE_MODE) { 3835 IBTF_DPRINTF_L2(cmlog, "ibt_get_ofuvcm_req_data: Active mode " 3836 "not supported"); 3837 mutex_exit(&statep->state_mutex); 3838 return (IBT_INVALID_PARAM); 3839 } 3840 ASSERT(statep->req_msgp); 3841 3842 /* 3843 * Fill in the additional req message values reqired for 3844 * RTR transition. 3845 * Should the PSN be same as the active side?? 3846 */ 3847 req_msgp = (ibcm_req_msg_t *)statep->req_msgp; 3848 req_data->req_rnr_nak_time = ibcm_default_rnr_nak_time; 3849 req_data->req_path_mtu = req_msgp->req_mtu_plus >> 4; 3850 req_data->req_rq_psn = b2h32(req_msgp->req_starting_psn_plus) >> 8; 3851 mutex_exit(&statep->state_mutex); 3852 return (IBT_SUCCESS); 3853 } 3854 3855 ibt_status_t 3856 ibt_ofuvcm_proceed(ibt_cm_event_type_t event, void *session_id, 3857 ibt_cm_status_t status, ibt_cm_proceed_reply_t *cm_event_data, 3858 void *priv_data, ibt_priv_data_len_t priv_data_len) 3859 { 3860 ibcm_state_data_t *statep = (ibcm_state_data_t *)session_id; 3861 ibt_status_t ret; 3862 3863 IBTF_DPRINTF_L3(cmlog, "ibt_ofuvcm_proceed chan 0x%p event %x " 3864 "status %x session_id %p", statep->channel, event, status, 3865 session_id); 3866 3867 IBTF_DPRINTF_L5(cmlog, "ibt_ofuvcm_proceed chan 0x%p " 3868 "cm_event_data %p, priv_data %p priv_data_len %x", 3869 statep->channel, cm_event_data, priv_data, priv_data_len); 3870 3871 /* validate session_id and status */ 3872 if ((statep == NULL) || (status == IBT_CM_DEFER)) { 3873 IBTF_DPRINTF_L2(cmlog, "ibt_ofuvcm_proceed : Invalid Args"); 3874 return (IBT_INVALID_PARAM); 3875 } 3876 3877 if (event != IBT_CM_EVENT_REQ_RCV) { 3878 IBTF_DPRINTF_L2(cmlog, "ibt_ofuvcm_proceed : only for REQ_RCV"); 3879 return (IBT_INVALID_PARAM); 3880 } 3881 mutex_enter(&statep->state_mutex); 3882 statep->skip_rtr = 1; 3883 mutex_exit(&statep->state_mutex); 3884 3885 ret = ibt_cm_proceed(event, session_id, status, cm_event_data, 3886 priv_data, priv_data_len); 3887 return (ret); 3888 } 3889 3890 /* 3891 * Function: 3892 * ibt_cm_proceed 3893 * 3894 * Verifies the arguments and dispatches the cm state machine processing 3895 * via taskq 3896 */ 3897 3898 ibt_status_t 3899 ibt_cm_proceed(ibt_cm_event_type_t event, void *session_id, 3900 ibt_cm_status_t status, ibt_cm_proceed_reply_t *cm_event_data, 3901 void *priv_data, ibt_priv_data_len_t priv_data_len) 3902 { 3903 ibcm_state_data_t *statep = (ibcm_state_data_t *)session_id; 3904 ibcm_proceed_targs_t *proceed_targs; 3905 ibcm_proceed_error_t proceed_error; 3906 3907 IBTF_DPRINTF_L3(cmlog, "ibt_cm_proceed chan 0x%p event %x status %x " 3908 "session_id %p", statep->channel, event, status, session_id); 3909 3910 IBTF_DPRINTF_L5(cmlog, "ibt_cm_proceed chan 0x%p cm_event_data %p, " 3911 "priv_data %p priv_data_len %x", statep->channel, cm_event_data, 3912 priv_data, priv_data_len); 3913 3914 /* validate session_id and status */ 3915 if ((statep == NULL) || (status == IBT_CM_DEFER)) { 3916 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : Invalid Args"); 3917 return (IBT_INVALID_PARAM); 3918 } 3919 3920 /* If priv data len specified, then priv_data cannot be NULL */ 3921 if ((priv_data_len > 0) && (priv_data == NULL)) 3922 return (IBT_INVALID_PARAM); 3923 3924 proceed_error = IBCM_PROCEED_INVALID_NONE; 3925 3926 mutex_enter(&statep->state_mutex); 3927 if (event == IBT_CM_EVENT_REQ_RCV) { 3928 3929 if ((statep->state != IBCM_STATE_REQ_RCVD) && 3930 (statep->state != IBCM_STATE_MRA_SENT)) 3931 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3932 else if (priv_data_len > IBT_REP_PRIV_DATA_SZ) 3933 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3934 3935 } else if (event == IBT_CM_EVENT_REP_RCV) { 3936 if ((statep->state != IBCM_STATE_REP_RCVD) && 3937 (statep->state != IBCM_STATE_MRA_REP_SENT)) 3938 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3939 else if (priv_data_len > IBT_RTU_PRIV_DATA_SZ) 3940 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3941 } else if (event == IBT_CM_EVENT_LAP_RCV) { 3942 if ((statep->ap_state != IBCM_AP_STATE_LAP_RCVD) && 3943 (statep->ap_state != IBCM_AP_STATE_MRA_LAP_SENT)) 3944 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3945 else if (priv_data_len > IBT_APR_PRIV_DATA_SZ) 3946 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3947 } else if (event == IBT_CM_EVENT_CONN_CLOSED) { 3948 if (statep->state != IBCM_STATE_DREQ_RCVD) 3949 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3950 else if (priv_data_len > IBT_DREP_PRIV_DATA_SZ) 3951 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3952 } else { 3953 proceed_error = IBCM_PROCEED_INVALID_EVENT; 3954 } 3955 3956 /* if there is an error, print an error message and return */ 3957 if (proceed_error != IBCM_PROCEED_INVALID_NONE) { 3958 mutex_exit(&statep->state_mutex); 3959 if (proceed_error == IBCM_PROCEED_INVALID_EVENT_STATE) { 3960 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3961 "Invalid Event/State combination specified", 3962 statep->channel); 3963 return (IBT_INVALID_PARAM); 3964 } else if (proceed_error == IBCM_PROCEED_INVALID_PRIV_SZ) { 3965 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3966 "Invalid Event/priv len combination specified", 3967 statep->channel); 3968 return (IBT_INVALID_PARAM); 3969 } else if (proceed_error == IBCM_PROCEED_INVALID_EVENT) { 3970 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3971 "Invalid Event specified", statep->channel); 3972 return (IBT_INVALID_PARAM); 3973 } else { 3974 ASSERT(proceed_error == IBCM_PROCEED_INVALID_LAP); 3975 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3976 "IBT_CM_EVENT_LAP_RCV not supported", 3977 statep->channel); 3978 /* UNTIL HCA DRIVER ENABLES AP SUPPORT, FAIL THE CALL */ 3979 return (IBT_APM_NOT_SUPPORTED); 3980 } 3981 } 3982 3983 3984 /* wait until client's CM handler returns DEFER status back to CM */ 3985 3986 while (statep->clnt_proceed == IBCM_BLOCK) { 3987 IBTF_DPRINTF_L5(cmlog, "ibt_cm_proceed : chan 0x%p blocked for " 3988 "return of client's cm handler", statep->channel); 3989 cv_wait(&statep->block_client_cv, &statep->state_mutex); 3990 } 3991 3992 if (statep->clnt_proceed == IBCM_FAIL) { 3993 mutex_exit(&statep->state_mutex); 3994 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p Failed as " 3995 "client returned non-DEFER status from cm handler", 3996 statep->channel); 3997 return (IBT_CHAN_STATE_INVALID); 3998 } 3999 4000 ASSERT(statep->clnt_proceed == IBCM_UNBLOCK); 4001 statep->clnt_proceed = IBCM_FAIL; 4002 mutex_exit(&statep->state_mutex); 4003 4004 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*proceed_targs)) 4005 4006 /* the state machine processing is done in a separate thread */ 4007 4008 /* proceed_targs is freed in ibcm_proceed_via_taskq */ 4009 proceed_targs = kmem_alloc(sizeof (ibcm_proceed_targs_t), 4010 KM_SLEEP); 4011 4012 proceed_targs->event = event; 4013 proceed_targs->status = status; 4014 proceed_targs->priv_data_len = priv_data_len; 4015 4016 bcopy(priv_data, proceed_targs->priv_data, priv_data_len); 4017 4018 proceed_targs->tst.rc.statep = statep; 4019 bcopy(cm_event_data, &proceed_targs->tst.rc.rc_cm_event_data, 4020 sizeof (ibt_cm_proceed_reply_t)); 4021 4022 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*proceed_targs)) 4023 4024 (void) taskq_dispatch(ibcm_taskq, ibcm_proceed_via_taskq, 4025 proceed_targs, TQ_SLEEP); 4026 4027 return (IBT_SUCCESS); 4028 } 4029 4030 /* 4031 * Function: 4032 * ibcm_proceed_via_taskq 4033 * 4034 * Called from taskq, dispatched by ibt_cm_proceed 4035 * Completes the cm state processing for ibt_cm_proceed 4036 */ 4037 void 4038 ibcm_proceed_via_taskq(void *targs) 4039 { 4040 ibcm_proceed_targs_t *proceed_targs = (ibcm_proceed_targs_t *)targs; 4041 ibcm_state_data_t *statep = proceed_targs->tst.rc.statep; 4042 ibt_cm_reason_t reject_reason; 4043 uint8_t arej_len; 4044 ibcm_status_t response; 4045 ibcm_clnt_reply_info_t clnt_info; 4046 4047 clnt_info.reply_event = &proceed_targs->tst.rc.rc_cm_event_data; 4048 clnt_info.priv_data = proceed_targs->priv_data; 4049 clnt_info.priv_data_len = proceed_targs->priv_data_len; 4050 4051 IBTF_DPRINTF_L4(cmlog, "ibcm_proceed_via_taskq chan 0x%p targs %x", 4052 statep->channel, targs); 4053 4054 if (proceed_targs->event == IBT_CM_EVENT_REQ_RCV) { 4055 response = 4056 ibcm_process_cep_req_cm_hdlr(statep, proceed_targs->status, 4057 &clnt_info, &reject_reason, &arej_len, 4058 (ibcm_req_msg_t *)statep->defer_cm_msg); 4059 4060 ibcm_handle_cep_req_response(statep, response, reject_reason, 4061 arej_len); 4062 4063 mutex_enter(&statep->state_mutex); 4064 statep->skip_rtr = 0; 4065 mutex_exit(&statep->state_mutex); 4066 4067 } else if (proceed_targs->event == IBT_CM_EVENT_REP_RCV) { 4068 response = 4069 ibcm_process_cep_rep_cm_hdlr(statep, proceed_targs->status, 4070 &clnt_info, &reject_reason, &arej_len, 4071 (ibcm_rep_msg_t *)statep->defer_cm_msg); 4072 4073 ibcm_handle_cep_rep_response(statep, response, reject_reason, 4074 arej_len, (ibcm_rep_msg_t *)statep->defer_cm_msg); 4075 4076 } else if (proceed_targs->event == IBT_CM_EVENT_LAP_RCV) { 4077 ibcm_process_cep_lap_cm_hdlr(statep, proceed_targs->status, 4078 &clnt_info, (ibcm_lap_msg_t *)statep->defer_cm_msg, 4079 (ibcm_apr_msg_t *)IBCM_OUT_MSGP(statep->lapr_msg)); 4080 4081 ibcm_post_apr_mad(statep); 4082 4083 } else { 4084 ASSERT(proceed_targs->event == IBT_CM_EVENT_CONN_CLOSED); 4085 ibcm_handle_cep_dreq_response(statep, proceed_targs->priv_data, 4086 proceed_targs->priv_data_len); 4087 } 4088 4089 kmem_free(targs, sizeof (ibcm_proceed_targs_t)); 4090 } 4091 4092 /* 4093 * Function: 4094 * ibt_cm_ud_proceed 4095 * 4096 * Verifies the arguments and dispatches the cm state machine processing 4097 * via taskq 4098 */ 4099 ibt_status_t 4100 ibt_cm_ud_proceed(void *session_id, ibt_channel_hdl_t ud_channel, 4101 ibt_cm_status_t status, ibt_redirect_info_t *redirect_infop, 4102 void *priv_data, ibt_priv_data_len_t priv_data_len) 4103 { 4104 ibcm_ud_state_data_t *ud_statep = (ibcm_ud_state_data_t *)session_id; 4105 ibcm_proceed_targs_t *proceed_targs; 4106 ibt_qp_query_attr_t qp_attr; 4107 ibt_status_t retval; 4108 4109 IBTF_DPRINTF_L3(cmlog, "ibt_cm_ud_proceed session_id %p " 4110 "ud_channel %p ", session_id, ud_channel); 4111 4112 IBTF_DPRINTF_L4(cmlog, "ibt_cm_ud_proceed status %x priv_data %p " 4113 "priv_data_len %x", status, priv_data, priv_data_len); 4114 4115 /* validate session_id and status */ 4116 if ((ud_statep == NULL) || (status == IBT_CM_DEFER)) { 4117 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Invalid Args"); 4118 return (IBT_INVALID_PARAM); 4119 } 4120 4121 /* If priv data len specified, then priv_data cannot be NULL */ 4122 if ((priv_data_len > 0) && (priv_data == NULL)) 4123 return (IBT_INVALID_PARAM); 4124 4125 if (priv_data_len > IBT_SIDR_REP_PRIV_DATA_SZ) 4126 return (IBT_INVALID_PARAM); 4127 4128 /* retrieve qpn and qkey from ud channel */ 4129 4130 /* validate event and statep's state */ 4131 4132 if (status == IBT_CM_ACCEPT) { 4133 retval = ibt_query_qp(ud_channel, &qp_attr); 4134 if ((retval != IBT_SUCCESS) || 4135 (qp_attr.qp_info.qp_trans != IBT_UD_SRV)) { 4136 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed: " 4137 "Failed to retrieve QPN from the channel: %d", 4138 retval); 4139 return (IBT_INVALID_PARAM); 4140 } 4141 } 4142 4143 4144 mutex_enter(&ud_statep->ud_state_mutex); 4145 4146 if (ud_statep->ud_state != IBCM_STATE_SIDR_REQ_RCVD) { 4147 mutex_exit(&ud_statep->ud_state_mutex); 4148 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Invalid State " 4149 "specified"); 4150 return (IBT_INVALID_PARAM); 4151 } 4152 4153 /* wait until client's CM handler returns DEFER status back to CM */ 4154 4155 while (ud_statep->ud_clnt_proceed == IBCM_BLOCK) { 4156 IBTF_DPRINTF_L5(cmlog, "ibt_cm_ud_proceed : Blocked for return" 4157 " of client's ud cm handler"); 4158 cv_wait(&ud_statep->ud_block_client_cv, 4159 &ud_statep->ud_state_mutex); 4160 } 4161 4162 if (ud_statep->ud_clnt_proceed == IBCM_FAIL) { 4163 mutex_exit(&ud_statep->ud_state_mutex); 4164 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Failed as client " 4165 "returned non-DEFER status from cm handler"); 4166 return (IBT_INVALID_PARAM); 4167 } 4168 4169 ASSERT(ud_statep->ud_clnt_proceed == IBCM_UNBLOCK); 4170 ud_statep->ud_clnt_proceed = IBCM_FAIL; 4171 mutex_exit(&ud_statep->ud_state_mutex); 4172 4173 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*proceed_targs)) 4174 4175 /* the state machine processing is done in a separate thread */ 4176 4177 /* proceed_targs is freed in ibcm_proceed_via_taskq */ 4178 proceed_targs = kmem_zalloc(sizeof (ibcm_proceed_targs_t), 4179 KM_SLEEP); 4180 4181 proceed_targs->status = status; 4182 proceed_targs->priv_data_len = priv_data_len; 4183 4184 bcopy(priv_data, proceed_targs->priv_data, priv_data_len); 4185 4186 if (status == IBT_CM_ACCEPT) { 4187 proceed_targs->tst.ud.ud_qkey = 4188 qp_attr.qp_info.qp_transport.ud.ud_qkey; 4189 proceed_targs->tst.ud.ud_qpn = qp_attr.qp_qpn; 4190 } 4191 4192 proceed_targs->tst.ud.ud_statep = ud_statep; 4193 4194 /* copy redirect info based on status */ 4195 if (status == IBT_CM_REDIRECT) 4196 bcopy(redirect_infop, &proceed_targs->tst.ud.ud_redirect_info, 4197 sizeof (ibt_redirect_info_t)); 4198 4199 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*proceed_targs)) 4200 4201 (void) taskq_dispatch(ibcm_taskq, ibcm_ud_proceed_via_taskq, 4202 proceed_targs, TQ_SLEEP); 4203 4204 return (IBT_SUCCESS); 4205 } 4206 4207 /* 4208 * Function: 4209 * ibcm_ud_proceed_via_taskq 4210 * 4211 * Called from taskq, dispatched by ibt_cm_ud_proceed 4212 * Completes the cm state processing for ibt_cm_ud_proceed 4213 */ 4214 void 4215 ibcm_ud_proceed_via_taskq(void *targs) 4216 { 4217 ibcm_proceed_targs_t *proceed_targs = (ibcm_proceed_targs_t *)targs; 4218 ibcm_ud_state_data_t *ud_statep = proceed_targs->tst.ud.ud_statep; 4219 ibcm_ud_clnt_reply_info_t ud_clnt_info; 4220 ibt_sidr_status_t sidr_status; 4221 4222 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_proceed_via_taskq(%p)", targs); 4223 4224 ud_clnt_info.ud_qpn = proceed_targs->tst.ud.ud_qpn; 4225 ud_clnt_info.ud_qkey = proceed_targs->tst.ud.ud_qkey; 4226 ud_clnt_info.priv_data = proceed_targs->priv_data; 4227 ud_clnt_info.priv_data_len = proceed_targs->priv_data_len; 4228 ud_clnt_info.redirect_infop = &proceed_targs->tst.ud.ud_redirect_info; 4229 4230 /* validate event and statep's state */ 4231 ibcm_process_sidr_req_cm_hdlr(ud_statep, proceed_targs->status, 4232 &ud_clnt_info, &sidr_status, 4233 (ibcm_sidr_rep_msg_t *)IBCM_OUT_MSGP(ud_statep->ud_stored_msg)); 4234 4235 ibcm_post_sidr_rep_mad(ud_statep, sidr_status); 4236 4237 /* decr the statep ref cnt incremented in ibcm_process_sidr_req_msg */ 4238 mutex_enter(&ud_statep->ud_state_mutex); 4239 IBCM_UD_REF_CNT_DECR(ud_statep); 4240 mutex_exit(&ud_statep->ud_state_mutex); 4241 4242 kmem_free(targs, sizeof (ibcm_proceed_targs_t)); 4243 } 4244 4245 /* 4246 * Function: 4247 * ibt_set_alt_path 4248 * Input: 4249 * channel Channel handle returned from ibt_alloc_rc_channel(9F). 4250 * 4251 * mode Execute in blocking or non blocking mode. 4252 * 4253 * alt_path A pointer to an ibt_alt_path_info_t as returned from an 4254 * ibt_get_alt_path(9F) call that specifies the new 4255 * alternate path. 4256 * 4257 * priv_data A pointer to a buffer specified by caller for the 4258 * private data in the outgoing CM Load Alternate Path 4259 * (LAP) message sent to the remote host. This can be NULL 4260 * if no private data is available to communicate to the 4261 * remote node. 4262 * 4263 * priv_data_len Length of valid data in priv_data, this should be less 4264 * than or equal to IBT_LAP_PRIV_DATA_SZ. 4265 * 4266 * Output: 4267 * ret_args If called in blocking mode, points to a return argument 4268 * structure of type ibt_ap_returns_t. 4269 * 4270 * Returns: 4271 * IBT_SUCCESS on Success else appropriate error. 4272 * Description: 4273 * Load the specified alternate path. Causes the CM to send an LAP message 4274 * to the remote node. 4275 * Can only be called on a previously opened RC channel. 4276 */ 4277 ibt_status_t 4278 ibt_set_alt_path(ibt_channel_hdl_t channel, ibt_execution_mode_t mode, 4279 ibt_alt_path_info_t *alt_path, void *priv_data, 4280 ibt_priv_data_len_t priv_data_len, ibt_ap_returns_t *ret_args) 4281 { 4282 ibmf_handle_t ibmf_hdl; 4283 ibt_status_t status = IBT_SUCCESS; 4284 ibcm_lap_msg_t *lap_msgp; 4285 ibcm_hca_info_t *hcap; 4286 ibcm_state_data_t *statep; 4287 uint8_t port_no; 4288 ib_lid_t alternate_slid; 4289 ibt_priv_data_len_t len; 4290 ib_lid_t base_lid; 4291 boolean_t alt_grh; 4292 4293 IBTF_DPRINTF_L3(cmlog, "ibt_set_alt_path(%p, %x, %p, %p, %x, %p)", 4294 channel, mode, alt_path, priv_data, priv_data_len, ret_args); 4295 4296 /* validate channel */ 4297 if (IBCM_INVALID_CHANNEL(channel)) { 4298 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: invalid channel"); 4299 return (IBT_CHAN_HDL_INVALID); 4300 } 4301 4302 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 4303 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4304 "Invalid Channel type: Applicable only to RC Channel"); 4305 return (IBT_CHAN_SRV_TYPE_INVALID); 4306 } 4307 4308 if (mode == IBT_NONBLOCKING) { 4309 if (ret_args != NULL) { 4310 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4311 "ret_args should be NULL when called in " 4312 "non-blocking mode"); 4313 return (IBT_INVALID_PARAM); 4314 } 4315 } else if (mode == IBT_BLOCKING) { 4316 if (ret_args == NULL) { 4317 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4318 "ret_args should be Non-NULL when called in " 4319 "blocking mode"); 4320 return (IBT_INVALID_PARAM); 4321 } 4322 if (ret_args->ap_priv_data_len > IBT_APR_PRIV_DATA_SZ) { 4323 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4324 "expected private data length is too large"); 4325 return (IBT_INVALID_PARAM); 4326 } 4327 if ((ret_args->ap_priv_data_len > 0) && 4328 (ret_args->ap_priv_data == NULL)) { 4329 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4330 "apr_priv_data_len > 0, but apr_priv_data NULL"); 4331 return (IBT_INVALID_PARAM); 4332 } 4333 } else { /* any other mode is not valid for ibt_set_alt_path */ 4334 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4335 "invalid mode %x specified", mode); 4336 return (IBT_INVALID_PARAM); 4337 } 4338 4339 if ((port_no = alt_path->ap_alt_cep_path.cep_hca_port_num) == 0) 4340 return (IBT_INVALID_PARAM); 4341 4342 /* get the statep */ 4343 IBCM_GET_CHAN_PRIVATE(channel, statep); 4344 if (statep == NULL) { 4345 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: statep NULL"); 4346 return (IBT_CM_FAILURE); 4347 } 4348 4349 mutex_enter(&statep->state_mutex); 4350 IBCM_RELEASE_CHAN_PRIVATE(channel); 4351 IBCM_REF_CNT_INCR(statep); 4352 mutex_exit(&statep->state_mutex); 4353 4354 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: statep %p", statep); 4355 4356 hcap = statep->hcap; 4357 4358 /* HCA must have been in active state. If not, it's a client bug */ 4359 if (!IBCM_ACCESS_HCA_OK(hcap)) 4360 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: hca in error state"); 4361 4362 ASSERT(statep->cm_handler != NULL); 4363 4364 /* Check Alternate port */ 4365 status = ibt_get_port_state_byguid(hcap->hca_guid, port_no, NULL, 4366 &base_lid); 4367 if (status != IBT_SUCCESS) { 4368 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4369 "ibt_get_port_state_byguid status %d ", status); 4370 mutex_enter(&statep->state_mutex); 4371 IBCM_REF_CNT_DECR(statep); 4372 mutex_exit(&statep->state_mutex); 4373 return (status); 4374 } 4375 4376 if ((hcap->hca_port_info[port_no - 1].port_ibmf_hdl == NULL) && 4377 ((status = ibcm_hca_reinit_port(hcap, port_no - 1)) 4378 != IBT_SUCCESS)) { 4379 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4380 "ibmf reg or callback setup failed during re-initialize"); 4381 mutex_enter(&statep->state_mutex); 4382 IBCM_REF_CNT_DECR(statep); 4383 mutex_exit(&statep->state_mutex); 4384 return (status); 4385 } 4386 4387 ibmf_hdl = statep->stored_reply_addr.ibmf_hdl; 4388 4389 alternate_slid = base_lid + 4390 alt_path->ap_alt_cep_path.cep_adds_vect.av_src_path; 4391 4392 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: alternate SLID = %x", 4393 h2b16(alternate_slid)); 4394 4395 ibcm_lapr_enter(); /* limit how many run simultaneously */ 4396 4397 /* Allocate MAD for LAP */ 4398 if (statep->lapr_msg == NULL) 4399 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &statep->lapr_msg, 4400 MAD_METHOD_SEND)) != IBT_SUCCESS) { 4401 ibcm_lapr_exit(); 4402 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4403 "chan 0x%p ibcm_alloc_out_msg failed", channel); 4404 mutex_enter(&statep->state_mutex); 4405 IBCM_REF_CNT_DECR(statep); 4406 mutex_exit(&statep->state_mutex); 4407 return (status); 4408 } 4409 4410 mutex_enter(&statep->state_mutex); 4411 4412 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: connection state is" 4413 " %x", statep->state); 4414 4415 /* Check state */ 4416 if ((statep->state != IBCM_STATE_ESTABLISHED) || 4417 (statep->ap_state != IBCM_AP_STATE_IDLE)) { 4418 IBCM_REF_CNT_DECR(statep); 4419 mutex_exit(&statep->state_mutex); 4420 (void) ibcm_free_out_msg(ibmf_hdl, &statep->lapr_msg); 4421 ibcm_lapr_exit(); 4422 return (IBT_CHAN_STATE_INVALID); 4423 } else { 4424 /* Set to LAP Sent state */ 4425 statep->ap_state = IBCM_AP_STATE_LAP_SENT; 4426 statep->ap_done = B_FALSE; 4427 statep->remaining_retry_cnt = statep->max_cm_retries; 4428 statep->timer_stored_state = statep->state; 4429 statep->timer_stored_ap_state = statep->ap_state; 4430 IBCM_REF_CNT_INCR(statep); /* for ibcm_post_lap_complete */ 4431 } 4432 4433 mutex_exit(&statep->state_mutex); 4434 4435 /* No more failure returns below */ 4436 4437 /* Allocate MAD for LAP */ 4438 IBTF_DPRINTF_L5(cmlog, "ibt_set_alt_path:" 4439 " statep's mad addr = 0x%p", IBCM_OUT_HDRP(statep->lapr_msg)); 4440 4441 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lap_msgp)) 4442 4443 lap_msgp = (ibcm_lap_msg_t *)IBCM_OUT_MSGP(statep->lapr_msg); 4444 4445 lap_msgp->lap_alt_l_port_lid = h2b16(alternate_slid); 4446 lap_msgp->lap_alt_r_port_lid = 4447 h2b16(alt_path->ap_alt_cep_path.cep_adds_vect.av_dlid); 4448 4449 /* Fill in remote port gid */ 4450 lap_msgp->lap_alt_r_port_gid.gid_prefix = 4451 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_dgid.gid_prefix); 4452 lap_msgp->lap_alt_r_port_gid.gid_guid = 4453 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_dgid.gid_guid); 4454 4455 /* Fill in local port gid */ 4456 lap_msgp->lap_alt_l_port_gid.gid_prefix = 4457 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_sgid.gid_prefix); 4458 lap_msgp->lap_alt_l_port_gid.gid_guid = 4459 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_sgid.gid_guid); 4460 4461 alt_grh = alt_path->ap_alt_cep_path.cep_adds_vect.av_send_grh; 4462 4463 /* alternate_flow_label, and alternate srate, alternate traffic class */ 4464 lap_msgp->lap_alt_srate_plus = 4465 alt_path->ap_alt_cep_path.cep_adds_vect.av_srate & 0x3f; 4466 lap_msgp->lap_alt_flow_label_plus = h2b32(((alt_grh == B_TRUE) ? 4467 (alt_path->ap_alt_cep_path.cep_adds_vect.av_flow << 12) : 0) | 4468 alt_path->ap_alt_cep_path.cep_adds_vect.av_tclass); 4469 4470 /* Alternate hop limit, service level */ 4471 lap_msgp->lap_alt_hop_limit = (alt_grh == B_TRUE) ? 4472 alt_path->ap_alt_cep_path.cep_adds_vect.av_hop : 1; 4473 lap_msgp->lap_alt_sl_plus = 4474 alt_path->ap_alt_cep_path.cep_adds_vect.av_srvl << 4 | 4475 ((alt_grh == B_FALSE) ? 0x8 : 0); 4476 4477 lap_msgp->lap_alt_local_acktime_plus = ibt_usec2ib( 4478 (2 * statep->rc_alt_pkt_lt) + 4479 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 4480 4481 lap_msgp->lap_local_comm_id = h2b32(statep->local_comid); 4482 lap_msgp->lap_remote_comm_id = h2b32(statep->remote_comid); 4483 4484 lap_msgp->lap_remote_qpn_eecn_plus = 4485 h2b32((statep->remote_qpn << 8) | 4486 ibt_usec2ib(ibcm_remote_response_time) << 3); 4487 4488 len = min(priv_data_len, IBT_LAP_PRIV_DATA_SZ); 4489 if ((len > 0) && priv_data) { 4490 bcopy(priv_data, lap_msgp->lap_private_data, len); 4491 } 4492 4493 /* only rc_alt_pkt_lt and ap_return_data fields are initialized */ 4494 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*statep)) 4495 4496 statep->rc_alt_pkt_lt = ibt_ib2usec(alt_path->ap_alt_pkt_lt); 4497 4498 /* return_data is filled up in the state machine code */ 4499 statep->ap_return_data = ret_args; 4500 4501 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*statep)) 4502 4503 IBCM_OUT_HDRP(statep->lapr_msg)->AttributeID = 4504 h2b16(IBCM_INCOMING_LAP + IBCM_ATTR_BASE_ID); 4505 4506 IBCM_OUT_HDRP(statep->lapr_msg)->TransactionID = 4507 h2b64(ibcm_generate_tranid(IBCM_INCOMING_LAP, statep->local_comid, 4508 0)); 4509 IBTF_DPRINTF_L3(cmlog, "ibt_set_alt_path: statep %p, tid %llx", 4510 statep, IBCM_OUT_HDRP(statep->lapr_msg)->TransactionID); 4511 4512 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*lap_msgp)) 4513 4514 /* Send LAP */ 4515 ibcm_post_rc_mad(statep, statep->lapr_msg, ibcm_post_lap_complete, 4516 statep); 4517 4518 mutex_enter(&statep->state_mutex); 4519 4520 if (mode == IBT_BLOCKING) { 4521 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: blocking"); 4522 4523 /* wait for APR */ 4524 while (statep->ap_done != B_TRUE) { 4525 cv_wait(&statep->block_client_cv, 4526 &statep->state_mutex); 4527 } 4528 4529 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: done blocking"); 4530 4531 /* 4532 * In the case that ibt_set_alt_path fails, 4533 * change retval to IBT_CM_FAILURE 4534 */ 4535 if (statep->ap_return_data->ap_status != IBT_CM_AP_LOADED) 4536 status = IBT_CM_FAILURE; 4537 4538 } 4539 4540 /* decrement the ref-count before leaving here */ 4541 IBCM_REF_CNT_DECR(statep); 4542 4543 mutex_exit(&statep->state_mutex); 4544 4545 ibcm_lapr_exit(); 4546 4547 /* If this message isn't seen then ibt_set_alt_path failed */ 4548 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: done"); 4549 4550 return (status); 4551 } 4552 4553 4554 #ifdef DEBUG 4555 4556 /* 4557 * ibcm_query_classport_info: 4558 * Query classportinfo 4559 * 4560 * INPUTS: 4561 * channel - Channel that is associated with a statep 4562 * 4563 * RETURN VALUE: NONE 4564 * This function is currently used to generate a valid get method classport 4565 * info, and test CM functionality. There is no ibtl client interface to 4566 * generate a classportinfo. It is possible that CM may use classportinfo 4567 * from other nodes in the future, and most of the code below could be re-used. 4568 */ 4569 void 4570 ibcm_query_classport_info(ibt_channel_hdl_t channel) 4571 { 4572 ibcm_state_data_t *statep; 4573 ibmf_msg_t *msgp; 4574 4575 IBTF_DPRINTF_L3(cmlog, "ibcm_query_classport_info(%p)", channel); 4576 4577 /* validate channel, first */ 4578 if (IBCM_INVALID_CHANNEL(channel)) { 4579 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info: " 4580 "invalid channel (%p)", channel); 4581 return; 4582 } 4583 4584 /* get the statep */ 4585 IBCM_GET_CHAN_PRIVATE(channel, statep); 4586 4587 /* 4588 * This can happen, if the statep is already gone by a DREQ from 4589 * the remote side 4590 */ 4591 if (statep == NULL) { 4592 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info: " 4593 "statep NULL"); 4594 return; 4595 } 4596 4597 mutex_enter(&statep->state_mutex); 4598 IBCM_RELEASE_CHAN_PRIVATE(channel); 4599 IBCM_REF_CNT_INCR(statep); 4600 mutex_exit(&statep->state_mutex); 4601 4602 /* Debug/test code, so don't care about return status */ 4603 (void) ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl, &msgp, 4604 MAD_METHOD_GET); 4605 4606 IBCM_OUT_HDRP(msgp)->TransactionID = h2b64(ibcm_generate_tranid( 4607 MAD_ATTR_ID_CLASSPORTINFO, statep->local_comid, 0)); 4608 IBCM_OUT_HDRP(msgp)->AttributeID = h2b16(MAD_ATTR_ID_CLASSPORTINFO); 4609 4610 (void) ibcm_post_mad(msgp, &statep->stored_reply_addr, NULL, NULL); 4611 4612 IBTF_DPRINTF_L3(cmlog, "ibcm_query_classport_info(%p) " 4613 "Get method MAD posted ", channel); 4614 4615 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl, &msgp); 4616 4617 mutex_enter(&statep->state_mutex); 4618 IBCM_REF_CNT_DECR(statep); 4619 mutex_exit(&statep->state_mutex); 4620 } 4621 4622 static void 4623 ibcm_print_reply_addr(ibt_channel_hdl_t channel, ibcm_mad_addr_t *cm_reply_addr) 4624 { 4625 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: chan 0x%p, SLID %x, " 4626 "DLID %x", channel, cm_reply_addr->rcvd_addr.ia_local_lid, 4627 cm_reply_addr->rcvd_addr.ia_remote_lid); 4628 4629 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: QKEY %x, PKEY %x, " 4630 "RQPN %x SL %x", cm_reply_addr->rcvd_addr.ia_q_key, 4631 cm_reply_addr->rcvd_addr.ia_p_key, 4632 cm_reply_addr->rcvd_addr.ia_remote_qno, 4633 cm_reply_addr->rcvd_addr.ia_service_level); 4634 4635 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM SGID %llX:%llX ", 4636 cm_reply_addr->grh_hdr.ig_sender_gid.gid_prefix, 4637 cm_reply_addr->grh_hdr.ig_sender_gid.gid_guid); 4638 4639 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM DGID %llX:%llX", 4640 cm_reply_addr->grh_hdr.ig_recver_gid.gid_prefix, 4641 cm_reply_addr->grh_hdr.ig_recver_gid.gid_guid); 4642 4643 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM FL %x TC %x HL %x", 4644 cm_reply_addr->grh_hdr.ig_flow_label, 4645 cm_reply_addr->grh_hdr.ig_tclass, 4646 cm_reply_addr->grh_hdr.ig_hop_limit); 4647 } 4648 4649 #endif 4650 4651 /* For MCG List search */ 4652 typedef struct ibcm_mcg_list_s { 4653 struct ibcm_mcg_list_s *ml_next; 4654 ib_gid_t ml_sgid; 4655 ib_gid_t ml_mgid; 4656 ib_pkey_t ml_pkey; 4657 ib_qkey_t ml_qkey; 4658 uint_t ml_refcnt; 4659 uint8_t ml_jstate; 4660 } ibcm_mcg_list_t; 4661 4662 ibcm_mcg_list_t *ibcm_mcglist = NULL; 4663 4664 _NOTE(MUTEX_PROTECTS_DATA(ibcm_mcglist_lock, ibcm_mcg_list_s)) 4665 _NOTE(MUTEX_PROTECTS_DATA(ibcm_mcglist_lock, ibcm_mcglist)) 4666 4667 typedef struct ibcm_join_mcg_tqarg_s { 4668 ib_gid_t rgid; 4669 ibt_mcg_attr_t mcg_attr; 4670 ibt_mcg_info_t *mcg_infop; 4671 ibt_mcg_handler_t func; 4672 void *arg; 4673 } ibcm_join_mcg_tqarg_t; 4674 4675 _NOTE(READ_ONLY_DATA(ibcm_join_mcg_tqarg_s)) 4676 4677 void 4678 ibcm_add_incr_mcg_entry(sa_mcmember_record_t *mcg_req, 4679 sa_mcmember_record_t *mcg_resp) 4680 { 4681 ibcm_mcg_list_t *new = NULL; 4682 ibcm_mcg_list_t *head = NULL; 4683 4684 IBTF_DPRINTF_L3(cmlog, "ibcm_add_incr_mcg_entry: MGID %llX:%llX" 4685 "\n SGID %llX:%llX, JState %X)", mcg_req->MGID.gid_prefix, 4686 mcg_req->MGID.gid_guid, mcg_req->PortGID.gid_prefix, 4687 mcg_req->PortGID.gid_guid, mcg_req->JoinState); 4688 4689 mutex_enter(&ibcm_mcglist_lock); 4690 head = ibcm_mcglist; 4691 4692 while (head != NULL) { 4693 if ((head->ml_mgid.gid_guid == mcg_resp->MGID.gid_guid) && 4694 (head->ml_mgid.gid_prefix == mcg_resp->MGID.gid_prefix) && 4695 (head->ml_sgid.gid_guid == mcg_resp->PortGID.gid_guid)) { 4696 /* Increment the count */ 4697 head->ml_refcnt++; 4698 /* OR the join_state value, we need this during leave */ 4699 head->ml_jstate |= mcg_req->JoinState; 4700 4701 IBTF_DPRINTF_L3(cmlog, "ibcm_add_incr_mcg_entry: Entry " 4702 "FOUND: refcnt %d JState %X", head->ml_refcnt, 4703 head->ml_jstate); 4704 4705 mutex_exit(&ibcm_mcglist_lock); 4706 return; 4707 } 4708 head = head->ml_next; 4709 } 4710 mutex_exit(&ibcm_mcglist_lock); 4711 4712 IBTF_DPRINTF_L3(cmlog, "ibcm_add_incr_mcg_entry: Create NEW Entry "); 4713 4714 /* If we are here, either list is empty or match couldn't be found */ 4715 new = kmem_zalloc(sizeof (ibcm_mcg_list_t), KM_SLEEP); 4716 4717 mutex_enter(&ibcm_mcglist_lock); 4718 /* Initialize the fields */ 4719 new->ml_sgid = mcg_resp->PortGID; 4720 new->ml_mgid = mcg_resp->MGID; 4721 new->ml_qkey = mcg_req->Q_Key; 4722 new->ml_pkey = mcg_req->P_Key; 4723 new->ml_refcnt = 1; /* As this is the first entry */ 4724 new->ml_jstate = mcg_req->JoinState; 4725 new->ml_next = NULL; 4726 4727 new->ml_next = ibcm_mcglist; 4728 ibcm_mcglist = new; 4729 mutex_exit(&ibcm_mcglist_lock); 4730 } 4731 4732 /* 4733 * ibcm_del_decr_mcg_entry 4734 * 4735 * Return value: 4736 * IBCM_SUCCESS Entry found and ref_cnt is now zero. So go-ahead and 4737 * leave the MCG group. The return arg *jstate will have 4738 * a valid join_state value that needed to be used by 4739 * xxx_leave_mcg(). 4740 * IBCM_LOOKUP_EXISTS Entry found and ref_cnt is decremented but is NOT zero. 4741 * So do not leave the MCG group yet. 4742 * IBCM_LOOKUP_FAIL Entry is NOT found. 4743 */ 4744 ibcm_status_t 4745 ibcm_del_decr_mcg_entry(sa_mcmember_record_t *mcg_req, uint8_t *jstate) 4746 { 4747 ibcm_mcg_list_t *head, *prev; 4748 4749 IBTF_DPRINTF_L3(cmlog, "ibcm_del_decr_mcg_entry: MGID %llX:%llX" 4750 "\n SGID %llX:%llX, JState %X)", mcg_req->MGID.gid_prefix, 4751 mcg_req->MGID.gid_guid, mcg_req->PortGID.gid_prefix, 4752 mcg_req->PortGID.gid_guid, mcg_req->JoinState); 4753 4754 *jstate = 0; 4755 4756 mutex_enter(&ibcm_mcglist_lock); 4757 head = ibcm_mcglist; 4758 prev = NULL; 4759 4760 while (head != NULL) { 4761 if ((head->ml_mgid.gid_guid == mcg_req->MGID.gid_guid) && 4762 (head->ml_mgid.gid_prefix == mcg_req->MGID.gid_prefix) && 4763 (head->ml_sgid.gid_guid == mcg_req->PortGID.gid_guid)) { 4764 if (!(head->ml_jstate & mcg_req->JoinState)) { 4765 IBTF_DPRINTF_L2(cmlog, "ibcm_del_decr_mcg_entry" 4766 ": JoinState mismatch %X %X)", 4767 head->ml_jstate, mcg_req->JoinState); 4768 } 4769 /* Decrement the count */ 4770 head->ml_refcnt--; 4771 4772 if (head->ml_refcnt == 0) { 4773 *jstate = head->ml_jstate; 4774 4775 IBTF_DPRINTF_L3(cmlog, "ibcm_del_decr_mcg_entry" 4776 ": refcnt is ZERO, so delete the entry "); 4777 if ((head == ibcm_mcglist) || (prev == NULL)) { 4778 ibcm_mcglist = head->ml_next; 4779 } else if (prev != NULL) { 4780 prev->ml_next = head->ml_next; 4781 } 4782 mutex_exit(&ibcm_mcglist_lock); 4783 4784 kmem_free(head, sizeof (ibcm_mcg_list_t)); 4785 return (IBCM_SUCCESS); 4786 } 4787 mutex_exit(&ibcm_mcglist_lock); 4788 return (IBCM_LOOKUP_EXISTS); 4789 } 4790 prev = head; 4791 head = head->ml_next; 4792 } 4793 mutex_exit(&ibcm_mcglist_lock); 4794 4795 /* 4796 * If we are here, something went wrong, we don't have the entry 4797 * for that MCG being joined. 4798 */ 4799 IBTF_DPRINTF_L2(cmlog, "ibcm_del_decr_mcg_entry: Match NOT " 4800 "Found "); 4801 4802 return (IBCM_LOOKUP_FAIL); 4803 } 4804 4805 4806 /* 4807 * Function: 4808 * ibt_join_mcg 4809 * Input: 4810 * rgid The request GID that defines the HCA port from which a 4811 * contact to SA Access is performed to add the specified 4812 * endport GID ((mcg_attr->mc_pgid) to a multicast group. 4813 * If mcg_attr->mc_pgid is null, then this (rgid) will be 4814 * treated as endport GID that is to be added to the 4815 * multicast group. 4816 * 4817 * mcg_attr A pointer to an ibt_mcg_attr_t structure that defines 4818 * the attributes of the desired multicast group to be 4819 * created or joined. 4820 * 4821 * func NULL or a pointer to a function to call when 4822 * ibt_join_mcg() completes. If 'func' is not NULL then 4823 * ibt_join_mcg() will return as soon as possible after 4824 * initiating the multicast group join/create process. 4825 * 'func' is then called when the process completes. 4826 * 4827 * arg Argument to the 'func'. 4828 * 4829 * Output: 4830 * mcg_info_p A pointer to the ibt_mcg_info_t structure, allocated 4831 * by the caller, where the attributes of the created or 4832 * joined multicast group are copied. 4833 * Returns: 4834 * IBT_SUCCESS 4835 * IBT_INVALID_PARAM 4836 * IBT_MCG_RECORDS_NOT_FOUND 4837 * IBT_INSUFF_RESOURCE 4838 * Description: 4839 * Join a multicast group. The first full member "join" causes the MCG 4840 * to be created. 4841 */ 4842 ibt_status_t 4843 ibt_join_mcg(ib_gid_t rgid, ibt_mcg_attr_t *mcg_attr, 4844 ibt_mcg_info_t *mcg_info_p, ibt_mcg_handler_t func, void *arg) 4845 { 4846 ibcm_join_mcg_tqarg_t *mcg_tq; 4847 int flag = ((func == NULL) ? KM_SLEEP : KM_NOSLEEP); 4848 4849 IBTF_DPRINTF_L3(cmlog, "ibt_join_mcg(%llX:%llX, %p)", rgid.gid_prefix, 4850 rgid.gid_guid, mcg_attr); 4851 4852 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 4853 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Request GID is required"); 4854 return (IBT_INVALID_PARAM); 4855 } 4856 4857 if ((mcg_attr->mc_pkey == IB_PKEY_INVALID_LIMITED) || 4858 (mcg_attr->mc_pkey == IB_PKEY_INVALID_FULL)) { 4859 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Invalid P_Key specified"); 4860 return (IBT_INVALID_PARAM); 4861 } 4862 4863 if (mcg_attr->mc_join_state == 0) { 4864 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: JoinState not specified"); 4865 return (IBT_INVALID_PARAM); 4866 } 4867 4868 if (mcg_info_p == NULL) { 4869 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: mcg_info_p is NULL"); 4870 return (IBT_INVALID_PARAM); 4871 } 4872 4873 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mcg_tq)) 4874 4875 mcg_tq = kmem_alloc(sizeof (ibcm_join_mcg_tqarg_t), flag); 4876 if (mcg_tq == NULL) { 4877 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: " 4878 "Unable to allocate memory for local usage."); 4879 return (IBT_INSUFF_KERNEL_RESOURCE); 4880 } 4881 4882 mcg_tq->rgid = rgid; 4883 bcopy(mcg_attr, &mcg_tq->mcg_attr, sizeof (ibt_mcg_attr_t)); 4884 mcg_tq->mcg_infop = mcg_info_p; 4885 mcg_tq->func = func; 4886 mcg_tq->arg = arg; 4887 4888 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mcg_tq)) 4889 4890 if (func != NULL) { /* Non-Blocking */ 4891 IBTF_DPRINTF_L3(cmlog, "ibt_join_mcg: Non-Blocking Call"); 4892 if (taskq_dispatch(ibcm_taskq, ibcm_process_async_join_mcg, 4893 mcg_tq, TQ_NOSLEEP) == 0) { 4894 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Failed to " 4895 "Dispatch the TaskQ"); 4896 kmem_free(mcg_tq, sizeof (ibcm_join_mcg_tqarg_t)); 4897 return (IBT_INSUFF_KERNEL_RESOURCE); 4898 } else 4899 return (IBT_SUCCESS); 4900 } else { /* Blocking */ 4901 return (ibcm_process_join_mcg(mcg_tq)); 4902 } 4903 } 4904 4905 static void 4906 ibcm_process_async_join_mcg(void *tq_arg) 4907 { 4908 (void) ibcm_process_join_mcg(tq_arg); 4909 } 4910 4911 static ibt_status_t 4912 ibcm_process_join_mcg(void *taskq_arg) 4913 { 4914 sa_mcmember_record_t mcg_req; 4915 sa_mcmember_record_t *mcg_resp; 4916 ibmf_saa_access_args_t access_args; 4917 ibmf_saa_handle_t saa_handle; 4918 uint64_t component_mask = 0; 4919 ibt_status_t retval; 4920 ibtl_cm_hca_port_t hca_port; 4921 uint_t num_records; 4922 size_t length; 4923 ibcm_hca_info_t *hcap; 4924 ibcm_join_mcg_tqarg_t *mcg_arg = (ibcm_join_mcg_tqarg_t *)taskq_arg; 4925 ibt_mcg_info_t *mcg_info_p = mcg_arg->mcg_infop; 4926 4927 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg(%p)", mcg_arg); 4928 4929 retval = ibtl_cm_get_hca_port(mcg_arg->rgid, 0, &hca_port); 4930 if (retval != IBT_SUCCESS) { 4931 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: Failed to get " 4932 "port info from specified RGID: status = %d", retval); 4933 goto ibcm_join_mcg_exit1; 4934 } 4935 4936 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 4937 4938 if ((mcg_arg->mcg_attr.mc_pgid.gid_prefix == 0) || 4939 (mcg_arg->mcg_attr.mc_pgid.gid_guid == 0)) { 4940 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4941 "Request GID is Port GID"); 4942 mcg_req.PortGID = mcg_arg->rgid; 4943 } else { 4944 mcg_req.PortGID = mcg_arg->mcg_attr.mc_pgid; 4945 } 4946 component_mask |= SA_MC_COMPMASK_PORTGID; 4947 4948 mcg_req.Q_Key = mcg_arg->mcg_attr.mc_qkey; 4949 mcg_req.P_Key = mcg_arg->mcg_attr.mc_pkey; 4950 mcg_req.JoinState = mcg_arg->mcg_attr.mc_join_state; 4951 mcg_req.TClass = mcg_arg->mcg_attr.mc_tclass; 4952 mcg_req.FlowLabel = mcg_arg->mcg_attr.mc_flow; 4953 mcg_req.SL = mcg_arg->mcg_attr.mc_sl; 4954 4955 component_mask |= SA_MC_COMPMASK_QKEY | SA_MC_COMPMASK_PKEY | 4956 SA_MC_COMPMASK_JOINSTATE | SA_MC_COMPMASK_TCLASS | 4957 SA_MC_COMPMASK_FLOWLABEL | SA_MC_COMPMASK_SL; 4958 4959 /* If client has specified MGID, use it else SA will assign one. */ 4960 if ((mcg_arg->mcg_attr.mc_mgid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 4961 mcg_req.MGID = mcg_arg->mcg_attr.mc_mgid; 4962 component_mask |= SA_MC_COMPMASK_MGID; 4963 } 4964 4965 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: "); 4966 IBTF_DPRINTF_L3(cmlog, "PGID=%016llX:%016llX, ", 4967 mcg_req.PortGID.gid_prefix, mcg_req.PortGID.gid_guid); 4968 IBTF_DPRINTF_L3(cmlog, "MGID=%016llX:%016llX", 4969 mcg_req.MGID.gid_prefix, mcg_req.MGID.gid_guid); 4970 IBTF_DPRINTF_L3(cmlog, "JoinState = %X", 4971 mcg_arg->mcg_attr.mc_join_state); 4972 IBTF_DPRINTF_L5(cmlog, "QKey %lX, PKey %lX", 4973 mcg_arg->mcg_attr.mc_qkey, mcg_arg->mcg_attr.mc_pkey); 4974 IBTF_DPRINTF_L5(cmlog, "Scope %X, MLID %X", 4975 mcg_arg->mcg_attr.mc_scope, mcg_arg->mcg_attr.mc_mlid); 4976 4977 /* Is MTU specified. */ 4978 if (mcg_arg->mcg_attr.mc_mtu_req.r_mtu) { 4979 mcg_req.MTU = mcg_arg->mcg_attr.mc_mtu_req.r_mtu; 4980 mcg_req.MTUSelector = mcg_arg->mcg_attr.mc_mtu_req.r_selector; 4981 4982 component_mask |= SA_MC_COMPMASK_MTUSELECTOR | 4983 SA_MC_COMPMASK_MTU; 4984 } 4985 4986 /* Is RATE specified. */ 4987 if (mcg_arg->mcg_attr.mc_rate_req.r_srate) { 4988 mcg_req.Rate = mcg_arg->mcg_attr.mc_rate_req.r_srate; 4989 mcg_req.RateSelector = 4990 mcg_arg->mcg_attr.mc_rate_req.r_selector; 4991 4992 component_mask |= SA_MC_COMPMASK_RATESELECTOR | 4993 SA_MC_COMPMASK_RATE; 4994 } 4995 4996 /* Is Packet Life Time specified. */ 4997 if (mcg_arg->mcg_attr.mc_pkt_lt_req.p_pkt_lt) { 4998 mcg_req.Rate = mcg_arg->mcg_attr.mc_pkt_lt_req.p_pkt_lt; 4999 mcg_req.RateSelector = 5000 mcg_arg->mcg_attr.mc_pkt_lt_req.p_selector; 5001 5002 component_mask |= SA_MC_COMPMASK_PKTLTSELECTOR | 5003 SA_MC_COMPMASK_PKTLT; 5004 } 5005 5006 if (mcg_arg->mcg_attr.mc_hop) { 5007 mcg_req.HopLimit = mcg_arg->mcg_attr.mc_hop; 5008 component_mask |= SA_MC_COMPMASK_HOPLIMIT; 5009 } 5010 5011 if (mcg_arg->mcg_attr.mc_scope) { 5012 mcg_req.Scope = mcg_arg->mcg_attr.mc_scope; 5013 component_mask |= SA_MC_COMPMASK_SCOPE; 5014 } 5015 5016 if (mcg_arg->mcg_attr.mc_mlid) { 5017 mcg_req.MLID = mcg_arg->mcg_attr.mc_mlid; 5018 component_mask |= SA_MC_COMPMASK_MLID; 5019 } 5020 5021 /* Get SA Access Handle. */ 5022 hcap = ibcm_find_hca_entry(hca_port.hp_hca_guid); 5023 if (hcap == NULL) { 5024 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: NO HCA found"); 5025 5026 retval = IBT_HCA_BUSY_DETACHING; 5027 goto ibcm_join_mcg_exit1; 5028 } 5029 5030 saa_handle = ibcm_get_saa_handle(hcap, hca_port.hp_port); 5031 if (saa_handle == NULL) { 5032 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: SA Handle NULL"); 5033 5034 retval = IBT_HCA_PORT_NOT_ACTIVE; 5035 goto ibcm_join_mcg_exit; 5036 } 5037 5038 if ((mcg_arg->mcg_attr.mc_pgid.gid_prefix != 0) && 5039 (mcg_arg->mcg_attr.mc_pgid.gid_guid != 0)) { 5040 retval = ibtl_cm_get_hca_port(mcg_arg->mcg_attr.mc_pgid, 0, 5041 &hca_port); 5042 if (retval != IBT_SUCCESS) { 5043 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: Failed " 5044 "to get PortInfo of specified PGID: status = %d", 5045 retval); 5046 goto ibcm_join_mcg_exit1; 5047 } 5048 } 5049 5050 /* Contact SA Access */ 5051 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 5052 access_args.sq_access_type = IBMF_SAA_UPDATE; 5053 access_args.sq_component_mask = component_mask; 5054 access_args.sq_template = &mcg_req; 5055 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 5056 access_args.sq_callback = NULL; 5057 access_args.sq_callback_arg = NULL; 5058 5059 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 5060 (void **)&mcg_resp); 5061 if (retval != IBT_SUCCESS) { 5062 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: " 5063 "SA Access Failed"); 5064 goto ibcm_join_mcg_exit; 5065 } 5066 5067 num_records = length/sizeof (sa_mcmember_record_t); 5068 5069 IBTF_DPRINTF_L4(cmlog, "ibcm_process_join_mcg: " 5070 "Found %d MCMember Records", num_records); 5071 5072 /* Validate the returned number of records. */ 5073 if ((mcg_resp != NULL) && (num_records > 0)) { 5074 /* Update the return values. */ 5075 mcg_info_p->mc_adds_vect.av_dgid = mcg_resp->MGID; 5076 mcg_info_p->mc_adds_vect.av_sgid = mcg_resp->PortGID; 5077 mcg_info_p->mc_adds_vect.av_srate = mcg_resp->Rate; 5078 mcg_info_p->mc_adds_vect.av_srvl = mcg_resp->SL; 5079 mcg_info_p->mc_adds_vect.av_flow = mcg_resp->FlowLabel; 5080 mcg_info_p->mc_adds_vect.av_tclass = mcg_resp->TClass; 5081 mcg_info_p->mc_adds_vect.av_hop = mcg_resp->HopLimit; 5082 mcg_info_p->mc_adds_vect.av_send_grh = B_TRUE; 5083 mcg_info_p->mc_adds_vect.av_dlid = mcg_resp->MLID; 5084 mcg_info_p->mc_mtu = mcg_resp->MTU; 5085 mcg_info_p->mc_qkey = mcg_resp->Q_Key; 5086 5087 retval = ibt_pkey2index_byguid(hca_port.hp_hca_guid, 5088 hca_port.hp_port, mcg_resp->P_Key, &mcg_info_p->mc_pkey_ix); 5089 if (retval != IBT_SUCCESS) { 5090 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 5091 "Pkey2Index Conversion failed<%d>", retval); 5092 mcg_info_p->mc_pkey_ix = 0; 5093 } 5094 5095 mcg_info_p->mc_scope = mcg_resp->Scope; 5096 mcg_info_p->mc_pkt_lt = mcg_resp->PacketLifeTime; 5097 5098 mcg_info_p->mc_adds_vect.av_port_num = hca_port.hp_port; 5099 mcg_info_p->mc_adds_vect.av_sgid_ix = hca_port.hp_sgid_ix; 5100 mcg_info_p->mc_adds_vect.av_src_path = 0; 5101 5102 /* Add or Incr the matching MCG entry. */ 5103 ibcm_add_incr_mcg_entry(&mcg_req, mcg_resp); 5104 /* Deallocate the memory allocated by SA for mcg_resp. */ 5105 kmem_free(mcg_resp, length); 5106 5107 retval = IBT_SUCCESS; 5108 } else { 5109 retval = IBT_MCG_RECORDS_NOT_FOUND; 5110 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 5111 "MCG RECORDS NOT FOUND"); 5112 } 5113 5114 ibcm_join_mcg_exit: 5115 ibcm_dec_hca_acc_cnt(hcap); 5116 5117 ibcm_join_mcg_exit1: 5118 if (mcg_arg->func) 5119 (*(mcg_arg->func))(mcg_arg->arg, retval, mcg_info_p); 5120 5121 kmem_free(mcg_arg, sizeof (ibcm_join_mcg_tqarg_t)); 5122 5123 return (retval); 5124 } 5125 5126 5127 /* 5128 * Function: 5129 * ibt_leave_mcg 5130 * Input: 5131 * rgid The request GID that defines the HCA port upon which 5132 * to send the request to the Subnet Administrator, to 5133 * remove the specified port (port_gid) from the multicast 5134 * group. If 'port_gid' is the Reserved GID (i.e. 5135 * port_gid.gid_prefix = 0 and port_gid.gid_guid = 0), 5136 * then the end-port associated with 'rgid' is removed 5137 * from the multicast group. 5138 * 5139 * mc_gid A multicast group GID as returned from ibt_join_mcg() 5140 * call. This is optional, if not specified (i.e. 5141 * mc_gid.gid_prefix has 0xFF in its upper 8 bits to 5142 * identify this as being a multicast GID), then the 5143 * port is removed from all the multicast groups of 5144 * which it is a member. 5145 * 5146 * port_gid This is optional, if not the Reserved GID (gid_prefix 5147 * and gid_guid not equal to 0), then this specifies the 5148 * endport GID of the multicast group member being deleted 5149 * from the group. If it is the Reserved GID (gid_prefix 5150 * and gid_guid equal to 0) then the member endport GID is 5151 * determined from 'rgid'. 5152 * 5153 * mc_join_state The Join State attribute used when the group was joined 5154 * using ibt_join_mcg(). This Join State component must 5155 * contains at least one bit set to 1 in the same position 5156 * as that used during ibt_join_mcg(). i.e. the logical 5157 * AND of the two JoinState components is not all zeros. 5158 * This Join State component must not have some bits set 5159 * which are not set using ibt_join_mcg(). 5160 * Output: 5161 * None. 5162 * Returns: 5163 * IBT_SUCCESS 5164 * IBT_INVALID_PARAM 5165 * IBT_MC_GROUP_INVALID 5166 * IBT_INSUFF_RESOURCE 5167 * Description: 5168 * The port associated with the port GID shall be removed from the 5169 * multicast group specified by MGID (mc_gid) or from all the multicast 5170 * groups of which it is a member if the MGID (mc_gid) is not specified. 5171 * 5172 * The last full member to leave causes the destruction of the Multicast 5173 * Group. 5174 */ 5175 ibt_status_t 5176 ibt_leave_mcg(ib_gid_t rgid, ib_gid_t mc_gid, ib_gid_t port_gid, 5177 uint8_t mc_join_state) 5178 { 5179 sa_mcmember_record_t mcg_req; 5180 ibmf_saa_access_args_t access_args; 5181 ibmf_saa_handle_t saa_handle; 5182 uint64_t component_mask = 0; 5183 int sa_retval; 5184 ibt_status_t retval; 5185 ibcm_status_t ret; 5186 ibtl_cm_hca_port_t hca_port; 5187 size_t length; 5188 void *results_p; 5189 ibcm_hca_info_t *hcap; 5190 uint8_t jstate = 0; 5191 5192 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg(%llX:%llX, %llX:%llX)", 5193 rgid.gid_prefix, rgid.gid_guid, mc_gid.gid_prefix, mc_gid.gid_guid); 5194 5195 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg(%llX:%llX, 0x%X)", 5196 port_gid.gid_prefix, port_gid.gid_guid, mc_join_state); 5197 5198 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 5199 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: RequestGID is required"); 5200 return (IBT_INVALID_PARAM); 5201 } 5202 5203 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 5204 5205 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: MGID: %llX%llX", 5206 mc_gid.gid_prefix, mc_gid.gid_guid); 5207 5208 /* Validate MGID */ 5209 if ((mc_gid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 5210 mcg_req.MGID = mc_gid; 5211 component_mask |= SA_MC_COMPMASK_MGID; 5212 } else if ((mc_gid.gid_prefix != 0) || (mc_gid.gid_guid != 0)) { 5213 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Invalid MGID specified"); 5214 return (IBT_MC_MGID_INVALID); 5215 } 5216 5217 if ((port_gid.gid_prefix == 0) || (port_gid.gid_guid == 0)) { 5218 mcg_req.PortGID = rgid; 5219 } else { 5220 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Performing PROXY Leave"); 5221 mcg_req.PortGID = port_gid; 5222 } 5223 component_mask |= SA_MC_COMPMASK_PORTGID; 5224 5225 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Port GID <%llX:%llX>", 5226 mcg_req.PortGID.gid_prefix, mcg_req.PortGID.gid_guid); 5227 5228 /* Join State */ 5229 mcg_req.JoinState = mc_join_state; 5230 component_mask |= SA_MC_COMPMASK_JOINSTATE; 5231 5232 ret = ibcm_del_decr_mcg_entry(&mcg_req, &jstate); 5233 if (ret == IBCM_LOOKUP_EXISTS) { 5234 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Multiple JoinMCG record " 5235 " still exists, we shall leave for last leave_mcg call"); 5236 return (IBT_SUCCESS); 5237 } else if (ret == IBCM_LOOKUP_FAIL) { 5238 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: No Record found, " 5239 "continue with leave_mcg call"); 5240 } else if ((ret == IBCM_SUCCESS) && (jstate != 0)) { 5241 /* 5242 * Update with cached "jstate", as this will be OR'ed of 5243 * all ibt_join_mcg() calls for this record. 5244 */ 5245 mcg_req.JoinState = jstate; 5246 } 5247 5248 retval = ibtl_cm_get_hca_port(rgid, 0, &hca_port); 5249 if (retval != IBT_SUCCESS) { 5250 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: Failed to get port info " 5251 "from specified RGID : status = %d", retval); 5252 return (retval); 5253 } 5254 5255 /* Get SA Access Handle. */ 5256 hcap = ibcm_find_hca_entry(hca_port.hp_hca_guid); 5257 if (hcap == NULL) { 5258 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: " 5259 "NO HCA found"); 5260 return (IBT_HCA_BUSY_DETACHING); 5261 } 5262 5263 saa_handle = ibcm_get_saa_handle(hcap, hca_port.hp_port); 5264 if (saa_handle == NULL) { 5265 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: saa_handle is NULL"); 5266 ibcm_dec_hca_acc_cnt(hcap); 5267 return (IBT_HCA_PORT_NOT_ACTIVE); 5268 } 5269 5270 /* Contact SA Access */ 5271 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 5272 access_args.sq_access_type = IBMF_SAA_DELETE; 5273 access_args.sq_component_mask = component_mask; 5274 access_args.sq_template = &mcg_req; 5275 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 5276 access_args.sq_callback = NULL; 5277 access_args.sq_callback_arg = NULL; 5278 5279 ibcm_sa_access_enter(); 5280 5281 sa_retval = ibmf_sa_access(saa_handle, &access_args, 0, &length, 5282 &results_p); 5283 if (sa_retval != IBMF_SUCCESS) { 5284 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: SA access Failed: %d", 5285 sa_retval); 5286 (void) ibcm_ibmf_analyze_error(sa_retval); 5287 retval = IBT_MC_GROUP_INVALID; 5288 } 5289 5290 ibcm_sa_access_exit(); 5291 5292 ibcm_dec_hca_acc_cnt(hcap); 5293 5294 return (retval); 5295 } 5296 5297 5298 /* 5299 * Function: 5300 * ibt_query_mcg 5301 * Input: 5302 * rgid The request GID that defines the HCA port upon which 5303 * to send the request to the Subnet Administrator, to 5304 * retrieve Multicast Records matching attributes as 5305 * specified through 'mcg_attr' argument. 5306 * 5307 * mcg_attr NULL or a pointer to an ibt_mcg_attr_t structure that 5308 * specifies MCG attributes that are to be matched. 5309 * Attributes that are not required can be wild carded 5310 * by specifying as '0'. 5311 * 5312 * mcgs_max_num The maximum number of matching multicast groups to 5313 * return. If zero, then all available matching multicast 5314 * groups are returned. 5315 * Output: 5316 * mcgs_info_p The address of an ibt_mcg_info_t pointer, where 5317 * multicast group information is returned. The actual 5318 * number of entries filled in the array is returned in 5319 * entries_p. 5320 * 5321 * entries_p The number of ibt_mcg_attr_t entries returned. 5322 * Returns: 5323 * IBT_SUCCESS 5324 * IBT_INVALID_PARAM 5325 * IBT_MCG_RECORDS_NOT_FOUND 5326 * Description: 5327 * Request information on multicast groups that match the parameters 5328 * specified in mcg_attr. Information on each multicast group is returned 5329 * to the caller in the form of an array of ibt_mcg_info_t. 5330 * ibt_query_mcg() allocates the memory for this array and returns a 5331 * pointer to the array (mcgs_p) and the number of entries in the array 5332 * (entries_p). This memory should be freed by the client using 5333 * ibt_free_mcg_info(). 5334 */ 5335 ibt_status_t 5336 ibt_query_mcg(ib_gid_t rgid, ibt_mcg_attr_t *mcg_attr, uint_t mcgs_max_num, 5337 ibt_mcg_info_t **mcgs_info_p, uint_t *entries_p) 5338 { 5339 sa_mcmember_record_t mcg_req; 5340 sa_mcmember_record_t *mcg_resp; 5341 ibt_mcg_info_t *mcg_infop; 5342 ibmf_saa_access_args_t access_args; 5343 ibmf_saa_handle_t saa_handle; 5344 uint64_t component_mask = 0; 5345 ibt_status_t retval; 5346 ibtl_cm_hca_port_t hport; 5347 uint_t num_records; 5348 size_t length; 5349 void *results_p; 5350 ib_gid_t port_gid; 5351 ibcm_hca_info_t *hcap; 5352 5353 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg(%p, %d)", mcg_attr, mcgs_max_num); 5354 5355 if ((entries_p == NULL) || (mcgs_info_p == NULL)) { 5356 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: " 5357 "entries_p or mcgs_info_p is NULL"); 5358 return (IBT_INVALID_PARAM); 5359 } 5360 5361 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 5362 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: RequestGID is required"); 5363 return (IBT_INVALID_PARAM); 5364 } 5365 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: Request GID <%llX:%llX>", 5366 rgid.gid_prefix, rgid.gid_guid); 5367 5368 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 5369 port_gid.gid_prefix = port_gid.gid_guid = 0; 5370 5371 if (mcg_attr != NULL) { 5372 port_gid = mcg_attr->mc_pgid; 5373 5374 if ((port_gid.gid_prefix != 0) && (port_gid.gid_guid != 0)) { 5375 mcg_req.PortGID = mcg_attr->mc_pgid; 5376 component_mask |= SA_MC_COMPMASK_PORTGID; 5377 5378 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: PGID %llX:%llX", 5379 port_gid.gid_prefix, port_gid.gid_guid); 5380 } 5381 5382 /* Is Q_Key specified. */ 5383 if (mcg_attr->mc_qkey != 0) { 5384 mcg_req.Q_Key = mcg_attr->mc_qkey; 5385 component_mask |= SA_MC_COMPMASK_QKEY; 5386 } 5387 5388 /* Is P_Key specified. */ 5389 if (mcg_attr->mc_pkey != 0) { 5390 mcg_req.P_Key = mcg_attr->mc_pkey; 5391 component_mask |= SA_MC_COMPMASK_PKEY; 5392 } 5393 5394 /* Is MGID specified. */ 5395 if ((mcg_attr->mc_mgid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 5396 mcg_req.MGID = mcg_attr->mc_mgid; 5397 component_mask |= SA_MC_COMPMASK_MGID; 5398 } 5399 5400 /* Is MTU specified. */ 5401 if (mcg_attr->mc_mtu_req.r_mtu) { 5402 mcg_req.MTU = mcg_attr->mc_mtu_req.r_mtu; 5403 mcg_req.MTUSelector = mcg_attr->mc_mtu_req.r_selector; 5404 5405 component_mask |= SA_MC_COMPMASK_MTUSELECTOR | 5406 SA_MC_COMPMASK_MTU; 5407 } 5408 5409 if (mcg_attr->mc_tclass) { 5410 mcg_req.TClass = mcg_attr->mc_tclass; 5411 component_mask |= SA_MC_COMPMASK_TCLASS; 5412 } 5413 5414 /* Is RATE specified. */ 5415 if (mcg_attr->mc_rate_req.r_srate) { 5416 mcg_req.Rate = mcg_attr->mc_rate_req.r_srate; 5417 mcg_req.RateSelector = mcg_attr->mc_rate_req.r_selector; 5418 5419 component_mask |= SA_MC_COMPMASK_RATESELECTOR | 5420 SA_MC_COMPMASK_RATE; 5421 } 5422 5423 /* Is Packet Life Time specified. */ 5424 if (mcg_attr->mc_pkt_lt_req.p_pkt_lt) { 5425 mcg_req.Rate = mcg_attr->mc_pkt_lt_req.p_pkt_lt; 5426 mcg_req.RateSelector = 5427 mcg_attr->mc_pkt_lt_req.p_selector; 5428 5429 component_mask |= SA_MC_COMPMASK_PKTLTSELECTOR | 5430 SA_MC_COMPMASK_PKTLT; 5431 } 5432 5433 if (mcg_attr->mc_hop) { 5434 mcg_req.HopLimit = mcg_attr->mc_hop; 5435 component_mask |= SA_MC_COMPMASK_HOPLIMIT; 5436 } 5437 5438 if (mcg_attr->mc_flow) { 5439 mcg_req.FlowLabel = mcg_attr->mc_flow; 5440 component_mask |= SA_MC_COMPMASK_FLOWLABEL; 5441 } 5442 5443 if (mcg_attr->mc_sl) { 5444 mcg_req.SL = mcg_attr->mc_sl; 5445 component_mask |= SA_MC_COMPMASK_SL; 5446 } 5447 5448 if (mcg_attr->mc_scope) { 5449 mcg_req.Scope = mcg_attr->mc_scope; 5450 component_mask |= SA_MC_COMPMASK_SCOPE; 5451 } 5452 5453 if (mcg_attr->mc_join_state) { 5454 mcg_req.JoinState = mcg_attr->mc_join_state; 5455 component_mask |= SA_MC_COMPMASK_JOINSTATE; 5456 } 5457 5458 if (mcg_attr->mc_mlid) { 5459 mcg_req.MLID = mcg_attr->mc_mlid; 5460 component_mask |= SA_MC_COMPMASK_MLID; 5461 } 5462 } 5463 5464 retval = ibtl_cm_get_hca_port(rgid, 0, &hport); 5465 if (retval != IBT_SUCCESS) { 5466 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: Failed to get port info " 5467 "from specified RGID : status = %d", retval); 5468 return (retval); 5469 } 5470 5471 /* Get SA Access Handle. */ 5472 hcap = ibcm_find_hca_entry(hport.hp_hca_guid); 5473 if (hcap == NULL) { 5474 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: NO HCA found"); 5475 return (IBT_HCA_BUSY_DETACHING); 5476 } 5477 5478 saa_handle = ibcm_get_saa_handle(hcap, hport.hp_port); 5479 if (saa_handle == NULL) { 5480 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: saa_handle is NULL"); 5481 ibcm_dec_hca_acc_cnt(hcap); 5482 return (IBT_HCA_PORT_NOT_ACTIVE); 5483 } 5484 5485 /* Contact SA Access */ 5486 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 5487 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 5488 access_args.sq_component_mask = component_mask; 5489 access_args.sq_template = &mcg_req; 5490 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 5491 access_args.sq_callback = NULL; 5492 access_args.sq_callback_arg = NULL; 5493 5494 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 5495 &results_p); 5496 if (retval != IBT_SUCCESS) { 5497 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: SA access Failed"); 5498 ibcm_dec_hca_acc_cnt(hcap); 5499 return (retval); 5500 } 5501 5502 num_records = length/sizeof (sa_mcmember_record_t); 5503 5504 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: Found %d MCMember Records", 5505 num_records); 5506 5507 /* Validate the returned number of records. */ 5508 if ((results_p != NULL) && (num_records > 0)) { 5509 uint_t i; 5510 5511 /* 5512 * If mcgs_max_num is zero, then return all records else 5513 * return only requested number of records 5514 */ 5515 if ((mcgs_max_num != 0) && (num_records > mcgs_max_num)) { 5516 /* we are interested in only mcgs_max_num records */ 5517 num_records = mcgs_max_num; 5518 } 5519 5520 /* 5521 * The SGID returned in "mcg_info_p" buffer should be PortGID, 5522 * (mcg_attr->mc_pgid), if 'mcg_attr->mc_pgid' was specified, 5523 * else RequestGID (rgid) should be returned. 5524 */ 5525 if ((port_gid.gid_prefix != 0) && (port_gid.gid_guid != 0)) { 5526 5527 /* Get sgid_ix and port number of 'port_gid' */ 5528 retval = ibtl_cm_get_hca_port(port_gid, 0, &hport); 5529 if (retval != IBT_SUCCESS) { 5530 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: " 5531 "Failed to Get Portinfo for PortGID :" 5532 "status = %d", retval); 5533 return (retval); 5534 } 5535 } else { 5536 /* 5537 * The sgid_ix and port number related to RequestGID 5538 * are already obtained at the beginning. 5539 */ 5540 port_gid = rgid; 5541 } 5542 5543 /* 5544 * Allocate memory for return buffer, to be freed in 5545 * ibt_free_mcg_info(). 5546 */ 5547 mcg_infop = kmem_alloc((num_records * sizeof (ibt_mcg_info_t)), 5548 KM_SLEEP); 5549 5550 *mcgs_info_p = mcg_infop; 5551 *entries_p = num_records; 5552 5553 /* Update the return values. */ 5554 for (i = 0; i < num_records; i++) { 5555 5556 mcg_resp = (sa_mcmember_record_t *)((uchar_t *) 5557 results_p + i * sizeof (sa_mcmember_record_t)); 5558 5559 mcg_infop[i].mc_adds_vect.av_dgid = mcg_resp->MGID; 5560 mcg_infop[i].mc_adds_vect.av_sgid = port_gid; 5561 mcg_infop[i].mc_adds_vect.av_srate = mcg_resp->Rate; 5562 mcg_infop[i].mc_adds_vect.av_srvl = mcg_resp->SL; 5563 mcg_infop[i].mc_adds_vect.av_flow = mcg_resp->FlowLabel; 5564 mcg_infop[i].mc_adds_vect.av_tclass = mcg_resp->TClass; 5565 mcg_infop[i].mc_adds_vect.av_hop = mcg_resp->HopLimit; 5566 mcg_infop[i].mc_adds_vect.av_port_num = hport.hp_port; 5567 mcg_infop[i].mc_adds_vect.av_send_grh = B_TRUE; 5568 mcg_infop[i].mc_adds_vect.av_dlid = mcg_resp->MLID; 5569 mcg_infop[i].mc_adds_vect.av_sgid_ix = hport.hp_sgid_ix; 5570 mcg_infop[i].mc_adds_vect.av_src_path = 0; 5571 mcg_infop[i].mc_mtu = mcg_resp->MTU; 5572 mcg_infop[i].mc_qkey = mcg_resp->Q_Key; 5573 mcg_infop[i].mc_scope = mcg_resp->Scope; 5574 mcg_infop[i].mc_pkt_lt = mcg_resp->PacketLifeTime; 5575 5576 if (ibt_pkey2index_byguid(hport.hp_hca_guid, 5577 hport.hp_port, mcg_resp->P_Key, 5578 &mcg_infop[i].mc_pkey_ix) != IBT_SUCCESS) { 5579 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: " 5580 "Pkey2Index Conversion failed"); 5581 mcg_infop[i].mc_pkey_ix = 0; 5582 } 5583 } 5584 5585 /* 5586 * Deallocate the memory allocated by SA for results_p. 5587 */ 5588 kmem_free(results_p, length); 5589 retval = IBT_SUCCESS; 5590 5591 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: returning %d MCGRecords", 5592 num_records); 5593 5594 } else { 5595 retval = IBT_MCG_RECORDS_NOT_FOUND; 5596 *entries_p = 0; 5597 5598 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: MCG RECORDS NOT FOUND"); 5599 } 5600 5601 ibcm_dec_hca_acc_cnt(hcap); 5602 5603 return (retval); 5604 } 5605 5606 5607 /* 5608 * ibt_free_mcg_info() 5609 * Free the memory allocated by successful ibt_query_mcg() 5610 * 5611 * mcgs_info Pointer returned by ibt_query_mcg(). 5612 * 5613 * entries The number of ibt_mcg_info_t entries to free. 5614 */ 5615 void 5616 ibt_free_mcg_info(ibt_mcg_info_t *mcgs_info, uint_t entries) 5617 { 5618 IBTF_DPRINTF_L3(cmlog, "ibt_free_mcg_info: " 5619 "Free <%d> entries from 0x%p", entries, mcgs_info); 5620 5621 if ((mcgs_info != NULL) && (entries > 0)) 5622 kmem_free(mcgs_info, entries * sizeof (ibt_mcg_info_t)); 5623 else 5624 IBTF_DPRINTF_L2(cmlog, "ibt_free_mcg_info: " 5625 "ERROR: NULL buf pointer or length specified."); 5626 } 5627 5628 5629 /* 5630 * Function: 5631 * ibt_gid_to_node_info() 5632 * Input: 5633 * gid Identifies the IB Node and port for which to obtain 5634 * Node information. 5635 * Output: 5636 * node_info_p A pointer to an ibt_node_info_t structure (allocated 5637 * by the caller) in which to return the node information. 5638 * Returns: 5639 * IBT_SUCCESS 5640 * IBT_INVALID_PARAM 5641 * IBT_NODE_RECORDS_NOT_FOUND 5642 * IBT_NO_HCAS_AVAILABLE 5643 * Description: 5644 * Retrieve Node Information for the specified GID. 5645 */ 5646 ibt_status_t 5647 ibt_gid_to_node_info(ib_gid_t gid, ibt_node_info_t *node_info_p) 5648 { 5649 sa_node_record_t nr_req, *nr_resp; 5650 ibmf_saa_handle_t saa_handle; 5651 ibt_status_t retval; 5652 ibcm_hca_info_t *hcap; 5653 ibtl_cm_hca_port_t hport; 5654 int i, j; 5655 uint_t num_rec; 5656 ib_guid_t *guid_array = NULL; 5657 sa_path_record_t *path; 5658 size_t len; 5659 uint8_t npaths; 5660 uint32_t num_hcas = 0; 5661 ib_lid_t node_lid; 5662 boolean_t local_node = B_FALSE; 5663 void *res_p; 5664 uint8_t num_ports = 0; 5665 5666 5667 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info(%llX:%llX, %p)", 5668 gid.gid_prefix, gid.gid_guid, node_info_p); 5669 5670 if ((gid.gid_prefix == 0) || (gid.gid_guid == 0)) { 5671 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: GID is required"); 5672 return (IBT_INVALID_PARAM); 5673 } 5674 5675 if (node_info_p == NULL) { 5676 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5677 "Return Buf (node_info_p) is NULL."); 5678 return (IBT_INVALID_PARAM); 5679 } 5680 5681 /* 5682 * If 'gid' is on local node, then get node lid (i.e. base lid of the 5683 * associated port) info via ibtl_cm_get_hca_port() call. 5684 */ 5685 bzero(&hport, sizeof (ibtl_cm_hca_port_t)); 5686 if (ibtl_cm_get_hca_port(gid, 0, &hport) == IBT_SUCCESS) { 5687 5688 hcap = ibcm_find_hca_entry(hport.hp_hca_guid); 5689 if (hcap == NULL) { 5690 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5691 "HCA(%llX) info not found", hport.hp_hca_guid); 5692 return (IBT_NO_HCAS_AVAILABLE); 5693 } 5694 num_ports = 1; 5695 num_hcas = 1; 5696 node_lid = hport.hp_base_lid; 5697 local_node = B_TRUE; 5698 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info: Local Node: " 5699 "LID = 0x%X", node_lid); 5700 } else { 5701 /* Get the number of HCAs and their GUIDs */ 5702 num_hcas = ibt_get_hca_list(&guid_array); 5703 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info: ibt_get_hca_list " 5704 "returned %d hcas", num_hcas); 5705 5706 if (num_hcas == 0) { 5707 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5708 "NO HCA's Found on this system"); 5709 return (IBT_NO_HCAS_AVAILABLE); 5710 } 5711 } 5712 5713 for (i = 0; i < num_hcas; i++) { 5714 if (local_node == B_FALSE) { 5715 hcap = ibcm_find_hca_entry(guid_array[i]); 5716 if (hcap == NULL) { 5717 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5718 "HCA(%llX) info not found", guid_array[i]); 5719 retval = IBT_NO_HCAS_AVAILABLE; 5720 continue; 5721 } 5722 num_ports = hcap->hca_num_ports; 5723 } 5724 5725 for (j = 0; j < num_ports; j++) { 5726 uint8_t port = 0; 5727 5728 if (local_node == B_TRUE) 5729 port = hport.hp_port; 5730 else 5731 port = j + 1; 5732 5733 /* Get SA Access Handle. */ 5734 saa_handle = ibcm_get_saa_handle(hcap, port); 5735 if (saa_handle == NULL) { 5736 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5737 "Port %d of HCA (%llX) is NOT ACTIVE", 5738 port, hport.hp_hca_guid); 5739 retval = IBT_NODE_RECORDS_NOT_FOUND; 5740 continue; 5741 } 5742 5743 if (local_node == B_FALSE) { 5744 ib_gid_t sgid; 5745 int sa_ret; 5746 5747 /* 5748 * Check whether 'gid' and this port has same 5749 * subnet prefix. If not, then there is no use 5750 * in searching from this port. 5751 */ 5752 sgid = hcap->hca_port_info[j].port_sgid0; 5753 if (gid.gid_prefix != sgid.gid_prefix) { 5754 IBTF_DPRINTF_L3(cmlog, 5755 "ibt_gid_to_node_info:Sn_Prefix of " 5756 "GID(%llX) and Port's(%llX) differ", 5757 gid.gid_prefix, sgid.gid_prefix); 5758 retval = IBT_NODE_RECORDS_NOT_FOUND; 5759 continue; 5760 } 5761 5762 /* 5763 * First Get Path Records for the specified DGID 5764 * from this port (SGID). From Path Records, 5765 * note down DLID, then use this DLID as Input 5766 * attribute to get NodeRecords from SA Access. 5767 */ 5768 npaths = 1; 5769 path = NULL; 5770 5771 sa_ret = ibmf_saa_gid_to_pathrecords(saa_handle, 5772 sgid, gid, 0, 0, B_TRUE, &npaths, 0, &len, 5773 &path); 5774 if (sa_ret != IBMF_SUCCESS) { 5775 IBTF_DPRINTF_L2(cmlog, 5776 "ibt_gid_to_node_info: " 5777 "ibmf_saa_gid_to_pathrecords() " 5778 "returned error: %d ", sa_ret); 5779 retval = 5780 ibcm_ibmf_analyze_error(sa_ret); 5781 continue; 5782 } else if ((npaths == 0) || (path == NULL)) { 5783 IBTF_DPRINTF_L3(cmlog, 5784 "ibt_gid_to_node_info: failed (%d) " 5785 "to get path records for the DGID " 5786 "0x%llX from SGID 0x%llX", sa_ret, 5787 gid.gid_guid, sgid.gid_guid); 5788 retval = IBT_NODE_RECORDS_NOT_FOUND; 5789 continue; 5790 } 5791 node_lid = path->DLID; /* LID */ 5792 5793 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5794 "Remote Node: LID = 0x%X", node_lid); 5795 5796 /* Free SA_Access memory for path record. */ 5797 kmem_free(path, len); 5798 } 5799 5800 /* Retrieve Node Records from SA Access. */ 5801 bzero(&nr_req, sizeof (sa_node_record_t)); 5802 5803 nr_req.LID = node_lid; /* LID */ 5804 5805 retval = ibcm_get_node_rec(saa_handle, &nr_req, 5806 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len); 5807 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5808 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5809 "failed (%d) to get Node records", retval); 5810 continue; 5811 } else if (retval != IBT_SUCCESS) { 5812 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5813 "failed (%d) to get Node records", retval); 5814 ibcm_dec_hca_acc_cnt(hcap); 5815 goto gid_to_ni_exit; 5816 } 5817 5818 num_rec = len/sizeof (sa_node_record_t); 5819 nr_resp = (sa_node_record_t *)(uchar_t *)res_p; 5820 5821 /* Validate the returned number of records. */ 5822 if ((nr_resp != NULL) && (num_rec > 0)) { 5823 5824 IBCM_DUMP_NODE_REC(nr_resp); 5825 5826 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS( 5827 *node_info_p)) 5828 5829 node_info_p->n_sys_img_guid = 5830 nr_resp->NodeInfo.SystemImageGUID; 5831 node_info_p->n_node_guid = 5832 nr_resp->NodeInfo.NodeGUID; 5833 node_info_p->n_port_guid = 5834 nr_resp->NodeInfo.PortGUID; 5835 node_info_p->n_dev_id = 5836 nr_resp->NodeInfo.DeviceID; 5837 node_info_p->n_revision = 5838 nr_resp->NodeInfo.Revision; 5839 node_info_p->n_vendor_id = 5840 nr_resp->NodeInfo.VendorID; 5841 node_info_p->n_num_ports = 5842 nr_resp->NodeInfo.NumPorts; 5843 node_info_p->n_port_num = 5844 nr_resp->NodeInfo.LocalPortNum; 5845 node_info_p->n_node_type = 5846 nr_resp->NodeInfo.NodeType; 5847 (void) strncpy(node_info_p->n_description, 5848 (char *)&nr_resp->NodeDescription, 64); 5849 5850 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS( 5851 *node_info_p)) 5852 5853 /* 5854 * Deallocate the memory allocated by SA for 5855 * 'nr_resp'. 5856 */ 5857 ibcm_dec_hca_acc_cnt(hcap); 5858 kmem_free(nr_resp, len); 5859 retval = IBT_SUCCESS; 5860 5861 goto gid_to_ni_exit; 5862 } else { 5863 retval = IBT_NODE_RECORDS_NOT_FOUND; 5864 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5865 "Node Records NOT found - PortGUID %016llX", 5866 gid.gid_guid); 5867 } 5868 } 5869 ibcm_dec_hca_acc_cnt(hcap); 5870 5871 if (local_node == B_TRUE) 5872 break; 5873 } 5874 5875 gid_to_ni_exit: 5876 if (guid_array) 5877 ibt_free_hca_list(guid_array, num_hcas); 5878 5879 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: done. Status %d", retval); 5880 5881 return (retval); 5882 } 5883 5884 5885 ibt_status_t 5886 ibcm_get_node_rec(ibmf_saa_handle_t saa_handle, sa_node_record_t *nr_req, 5887 uint64_t component_mask, void *result_p, size_t *len) 5888 { 5889 ibmf_saa_access_args_t args; 5890 size_t length; 5891 ibt_status_t retval; 5892 5893 args.sq_attr_id = SA_NODERECORD_ATTRID; 5894 args.sq_template = nr_req; 5895 args.sq_access_type = IBMF_SAA_RETRIEVE; 5896 args.sq_template_length = sizeof (sa_node_record_t); 5897 args.sq_component_mask = component_mask; 5898 args.sq_callback = NULL; 5899 args.sq_callback_arg = NULL; 5900 5901 retval = ibcm_contact_sa_access(saa_handle, &args, &length, result_p); 5902 if (retval != IBT_SUCCESS) { 5903 IBTF_DPRINTF_L2(cmlog, "ibcm_get_node_rec: SA Call Failed"); 5904 return (retval); 5905 } 5906 5907 *len = length; 5908 5909 /* Validate the returned number of records. */ 5910 if ((result_p != NULL) && (length > 0)) { 5911 IBTF_DPRINTF_L3(cmlog, "ibcm_get_node_rec: Node Records FOUND"); 5912 5913 /* Got it, done!. */ 5914 return (IBT_SUCCESS); 5915 } else { 5916 IBTF_DPRINTF_L2(cmlog, "ibcm_get_node_rec: Node Rec NOT found"); 5917 return (IBT_NODE_RECORDS_NOT_FOUND); 5918 } 5919 } 5920 5921 5922 /* 5923 * Function: 5924 * ibt_get_companion_port_gids() 5925 * Description: 5926 * Get list of GID's available on a companion port(s) of the specified 5927 * GID or list of GIDs available on a specified Node GUID/SystemImage GUID. 5928 */ 5929 ibt_status_t 5930 ibt_get_companion_port_gids(ib_gid_t gid, ib_guid_t hca_guid, 5931 ib_guid_t sysimg_guid, ib_gid_t **gids_p, uint_t *num_gids_p) 5932 { 5933 sa_node_record_t nr_req, *nr_resp; 5934 void *res_p; 5935 ibmf_saa_handle_t saa_handle; 5936 int sa_ret; 5937 ibt_status_t retval = IBT_SUCCESS; 5938 ibcm_hca_info_t *hcap; 5939 ibtl_cm_hca_port_t hport; 5940 int i, j; 5941 uint_t num_rec; 5942 ib_guid_t *guid_array = NULL; 5943 sa_path_record_t *path; 5944 size_t len; 5945 uint8_t npaths; 5946 uint32_t num_hcas = 0; 5947 boolean_t local_node = B_FALSE; 5948 boolean_t local_hca = B_FALSE; 5949 ib_guid_t h_guid = hca_guid; 5950 ib_gid_t *gidp = NULL, *t_gidp = NULL; 5951 int multi_hca_loop = 0; 5952 5953 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids(%llX:%llX, %llX, " 5954 "%llX)", gid.gid_prefix, gid.gid_guid, hca_guid, sysimg_guid); 5955 5956 if (((gid.gid_prefix == 0) || (gid.gid_guid == 0)) && (hca_guid == 0) && 5957 (sysimg_guid == 0)) { 5958 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5959 "Null Input attribute specified."); 5960 return (IBT_INVALID_PARAM); 5961 } 5962 5963 if ((num_gids_p == NULL) || (gids_p == NULL)) { 5964 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5965 "num_gids_p or gids_p is NULL"); 5966 return (IBT_INVALID_PARAM); 5967 } 5968 5969 *num_gids_p = 0; 5970 5971 /* Get the number of HCAs and their GUIDs */ 5972 if ((num_hcas = ibt_get_hca_list(&guid_array)) == 0) { 5973 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5974 "NO HCA's Found on this system"); 5975 return (IBT_NO_HCAS_AVAILABLE); 5976 } 5977 5978 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids: " 5979 "ibt_get_hca_list() returned %d hcas", num_hcas); 5980 5981 /* 5982 * If 'gid' is on local node, then get node lid (i.e. base lid of the 5983 * associated port) info via ibtl_cm_get_hca_port() call. 5984 */ 5985 bzero(&hport, sizeof (ibtl_cm_hca_port_t)); 5986 if ((gid.gid_prefix != 0) && (gid.gid_guid != 0) && 5987 (ibtl_cm_get_hca_port(gid, 0, &hport) == IBT_SUCCESS)) { 5988 5989 if ((hca_guid != 0) && (hca_guid != hport.hp_hca_guid)) { 5990 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5991 "Invalid GID<->HCAGUID combination specified."); 5992 retval = IBT_INVALID_PARAM; 5993 goto get_comp_pgid_exit; 5994 } 5995 h_guid = hport.hp_hca_guid; 5996 local_node = B_TRUE; 5997 5998 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids: " 5999 "Local Node: HCA (0x%llX)", h_guid); 6000 } else if (h_guid) { /* Is specified HCA GUID - local? */ 6001 for (i = 0; i < num_hcas; i++) { 6002 if (h_guid == guid_array[i]) { 6003 local_hca = B_TRUE; 6004 break; 6005 } 6006 } 6007 } else if (sysimg_guid) { /* Is specified SystemImage GUID - local? */ 6008 for (i = 0; i < num_hcas; i++) { 6009 ibt_status_t ret; 6010 ibt_hca_attr_t hca_attr; 6011 6012 ret = ibt_query_hca_byguid(guid_array[i], &hca_attr); 6013 if (ret != IBT_SUCCESS) { 6014 IBTF_DPRINTF_L2(cmlog, 6015 "ibt_get_companion_port_gids: HCA(%llX) " 6016 "info not found", guid_array[i]); 6017 retval = IBT_NO_HCAS_AVAILABLE; 6018 continue; 6019 } 6020 if (hca_attr.hca_si_guid == sysimg_guid) { 6021 if ((hca_guid != 0) && 6022 (hca_guid != hca_attr.hca_node_guid)) { 6023 IBTF_DPRINTF_L2(cmlog, 6024 "ibt_get_companion_port_gids: " 6025 "Invalid SysImg<->HCA GUID " 6026 "combination specified."); 6027 retval = IBT_INVALID_PARAM; 6028 goto get_comp_pgid_exit; 6029 } 6030 local_hca = B_TRUE; 6031 h_guid = hca_attr.hca_node_guid; 6032 break; 6033 } 6034 } 6035 } 6036 6037 if ((local_node == B_TRUE) || (local_hca == B_TRUE)) { 6038 retval = ibtl_cm_get_local_comp_gids(h_guid, gid, gids_p, 6039 num_gids_p); 6040 goto get_comp_pgid_exit; 6041 } 6042 6043 get_comp_for_multihca: 6044 /* We will be here, if request is for remote node */ 6045 for (i = 0; i < num_hcas; i++) { 6046 int multism; 6047 uint_t count = 0; 6048 int multi_sm_loop = 0; 6049 uint_t k = 0, l; 6050 6051 hcap = ibcm_find_hca_entry(guid_array[i]); 6052 if (hcap == NULL) { 6053 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 6054 "HCA(%llX) info not found", guid_array[i]); 6055 retval = IBT_NO_HCAS_AVAILABLE; 6056 continue; 6057 } 6058 6059 /* 1 - MultiSM, 0 - Single SM */ 6060 multism = ibtl_cm_is_multi_sm(guid_array[i]); 6061 6062 for (j = 0; j < hcap->hca_num_ports; j++) { 6063 ib_gid_t sgid; 6064 uint64_t c_mask = 0; 6065 ib_guid_t pg; 6066 uint_t port = j; 6067 6068 get_comp_for_multism: 6069 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 6070 "Port %d, HCA %llX, MultiSM= %d, Loop=%d", 6071 port + 1, h_guid, multism, multi_sm_loop); 6072 6073 /* Get SA Access Handle. */ 6074 saa_handle = ibcm_get_saa_handle(hcap, port + 1); 6075 if (saa_handle == NULL) { 6076 IBTF_DPRINTF_L2(cmlog, 6077 "ibt_get_companion_port_gids: " 6078 "Port (%d) - NOT ACTIVE", port + 1); 6079 retval = IBT_GIDS_NOT_FOUND; 6080 continue; 6081 } 6082 6083 /* 6084 * Check whether 'gid' and this port has same subnet 6085 * prefix. If not, then there is no use in searching 6086 * from this port. 6087 */ 6088 sgid = hcap->hca_port_info[port].port_sgid0; 6089 if ((h_guid == 0) && (gid.gid_prefix != 0) && 6090 (multi_sm_loop == 0) && 6091 (gid.gid_prefix != sgid.gid_prefix)) { 6092 IBTF_DPRINTF_L2(cmlog, 6093 "ibt_get_companion_port_gids: SnPrefix of " 6094 "GID(%llX) and Port SN_Pfx(%llX) differ", 6095 gid.gid_prefix, sgid.gid_prefix); 6096 retval = IBT_GIDS_NOT_FOUND; 6097 continue; 6098 } 6099 6100 /* 6101 * If HCA GUID or System Image GUID is specified, then 6102 * we can achieve our goal sooner!. 6103 */ 6104 if ((h_guid == 0) && (sysimg_guid == 0)) { 6105 /* So only GID info is provided. */ 6106 6107 /* 6108 * First Get Path Records for the specified DGID 6109 * from this port (SGID). From Path Records, 6110 * note down DLID, then use this DLID as Input 6111 * attribute to get NodeRecords. 6112 */ 6113 npaths = 1; 6114 path = NULL; 6115 6116 sa_ret = ibmf_saa_gid_to_pathrecords(saa_handle, 6117 sgid, gid, 0, 0, B_TRUE, &npaths, 0, &len, 6118 &path); 6119 if (sa_ret != IBMF_SUCCESS) { 6120 IBTF_DPRINTF_L2(cmlog, 6121 "ibt_get_companion_port_gids: " 6122 "ibmf_saa_gid_to_pathrecords() " 6123 "returned error: %d ", sa_ret); 6124 retval = 6125 ibcm_ibmf_analyze_error(sa_ret); 6126 ibcm_dec_hca_acc_cnt(hcap); 6127 goto get_comp_pgid_exit; 6128 } else if ((npaths == 0) || (path == NULL)) { 6129 IBTF_DPRINTF_L2(cmlog, 6130 "ibt_get_companion_port_gids: " 6131 "failed (%d) to get path records " 6132 "for the DGID (0x%llX) from SGID " 6133 "(0x%llX)", sa_ret, gid.gid_guid, 6134 sgid.gid_guid); 6135 retval = IBT_GIDS_NOT_FOUND; 6136 continue; 6137 } 6138 6139 bzero(&nr_req, sizeof (sa_node_record_t)); 6140 nr_req.LID = path->DLID; /* LID */ 6141 6142 IBTF_DPRINTF_L3(cmlog, 6143 "ibt_get_companion_port_gids: " 6144 "Remote Node: LID = 0x%X", nr_req.LID); 6145 6146 /* Free SA_Access memory for path record. */ 6147 kmem_free(path, len); 6148 6149 IBTF_DPRINTF_L3(cmlog, 6150 "ibt_get_companion_port_gids: SAA Call: " 6151 "based on LID "); 6152 6153 retval = ibcm_get_node_rec(saa_handle, &nr_req, 6154 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len); 6155 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 6156 IBTF_DPRINTF_L2(cmlog, 6157 "ibt_get_companion_port_gids: " 6158 "failed (%d) to get Node records", 6159 retval); 6160 continue; 6161 } else if (retval != IBT_SUCCESS) { 6162 IBTF_DPRINTF_L2(cmlog, 6163 "ibt_get_companion_port_gids: " 6164 "failed (%d) to get Node records", 6165 retval); 6166 ibcm_dec_hca_acc_cnt(hcap); 6167 goto get_comp_pgid_exit; 6168 } 6169 6170 nr_resp = (sa_node_record_t *)(uchar_t *)res_p; 6171 /* Note down HCA GUID info. */ 6172 h_guid = nr_resp->NodeInfo.NodeGUID; 6173 6174 IBTF_DPRINTF_L3(cmlog, 6175 "ibt_get_companion_port_gids: " 6176 "Remote HCA GUID: 0x%llX", h_guid); 6177 6178 IBCM_DUMP_NODE_REC(nr_resp); 6179 6180 kmem_free(res_p, len); 6181 } 6182 6183 bzero(&nr_req, sizeof (sa_node_record_t)); 6184 if (h_guid != 0) { 6185 nr_req.NodeInfo.NodeGUID = h_guid; 6186 c_mask = SA_NODEINFO_COMPMASK_NODEGUID; 6187 } 6188 6189 if (sysimg_guid != 0) { 6190 nr_req.NodeInfo.SystemImageGUID = sysimg_guid; 6191 c_mask |= SA_NODEINFO_COMPMASK_SYSIMAGEGUID; 6192 } 6193 6194 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 6195 "SAA Call: CMASK= 0x%llX", c_mask); 6196 6197 retval = ibcm_get_node_rec(saa_handle, &nr_req, c_mask, 6198 &res_p, &len); 6199 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 6200 IBTF_DPRINTF_L3(cmlog, 6201 "ibt_get_companion_port_gids: " 6202 "failed (%d) to get Node records", retval); 6203 continue; 6204 } else if (retval != IBT_SUCCESS) { 6205 IBTF_DPRINTF_L2(cmlog, 6206 "ibt_get_companion_port_gids: Error: (%d) " 6207 "while getting Node records", retval); 6208 ibcm_dec_hca_acc_cnt(hcap); 6209 goto get_comp_pgid_exit; 6210 } 6211 6212 num_rec = len/sizeof (sa_node_record_t); 6213 6214 /* We will be here, only if we found some NodeRec */ 6215 if (gid.gid_prefix && gid.gid_guid) { 6216 nr_resp = (sa_node_record_t *)res_p; 6217 for (l = 0; l < num_rec; l++, nr_resp++) { 6218 pg = nr_resp->NodeInfo.PortGUID; 6219 if (gid.gid_guid != pg) 6220 count++; 6221 } 6222 } else { 6223 count = num_rec; 6224 } 6225 6226 if (count != 0) { 6227 if (multi_sm_loop == 1) { 6228 count += k; 6229 t_gidp = kmem_zalloc(count * 6230 sizeof (ib_gid_t), KM_SLEEP); 6231 6232 if ((k != 0) && (gidp != NULL)) { 6233 bcopy(gidp, t_gidp, 6234 k * sizeof (ib_gid_t)); 6235 kmem_free(gidp, 6236 k * sizeof (ib_gid_t)); 6237 } 6238 gidp = t_gidp; 6239 } else { 6240 gidp = kmem_zalloc(count * 6241 sizeof (ib_gid_t), KM_SLEEP); 6242 } 6243 *num_gids_p = count; 6244 *gids_p = gidp; 6245 6246 nr_resp = (sa_node_record_t *)res_p; 6247 for (l = 0; l < num_rec; l++, nr_resp++) { 6248 IBCM_DUMP_NODE_REC(nr_resp); 6249 6250 pg = nr_resp->NodeInfo.PortGUID; 6251 IBTF_DPRINTF_L4(cmlog, 6252 "ibt_get_companion_port_gids: " 6253 "PortGID %llX", pg); 6254 6255 if (pg != gid.gid_guid) { 6256 gidp[k].gid_prefix = 6257 sgid.gid_prefix; 6258 gidp[k].gid_guid = pg; 6259 6260 IBTF_DPRINTF_L3(cmlog, 6261 "ibt_get_companion_pgids: " 6262 "GID[%d] = %llX:%llX", k, 6263 gidp[k].gid_prefix, 6264 gidp[k].gid_guid); 6265 6266 k++; 6267 if (k == count) 6268 break; 6269 } 6270 } 6271 retval = IBT_SUCCESS; /* done!. */ 6272 kmem_free(res_p, len); 6273 ibcm_dec_hca_acc_cnt(hcap); 6274 goto get_comp_pgid_exit; 6275 } else { 6276 IBTF_DPRINTF_L2(cmlog, 6277 "ibt_get_companion_port_gids: " 6278 "Companion PortGIDs not available"); 6279 retval = IBT_GIDS_NOT_FOUND; 6280 } 6281 /* Deallocate the memory for 'res_p'. */ 6282 kmem_free(res_p, len); 6283 6284 /* 6285 * If we are on MultiSM setup, then we need to lookout 6286 * from that subnet port too. 6287 */ 6288 if (multism) { 6289 /* break if already searched both the subnet */ 6290 if (multi_sm_loop == 1) 6291 break; 6292 6293 port = (j == 0) ? 1 : 0; 6294 multi_sm_loop = 1; 6295 goto get_comp_for_multism; 6296 } else { 6297 break; 6298 } 6299 } 6300 ibcm_dec_hca_acc_cnt(hcap); 6301 6302 /* 6303 * We may be on dual HCA with dual SM configured system. And 6304 * the input attr GID was visible from second HCA. So in order 6305 * to get the companion portgid we need to re-look from the 6306 * first HCA ports. 6307 */ 6308 if ((num_hcas > 1) && (i > 0) && (h_guid != 0) && 6309 (multi_hca_loop != 1)) { 6310 multi_hca_loop = 1; 6311 goto get_comp_for_multihca; 6312 } 6313 } 6314 if (*num_gids_p == 0) 6315 retval = IBT_GIDS_NOT_FOUND; 6316 6317 get_comp_pgid_exit: 6318 if (guid_array) 6319 ibt_free_hca_list(guid_array, num_hcas); 6320 6321 if ((retval != IBT_SUCCESS) && (*num_gids_p != 0)) { 6322 retval = IBT_SUCCESS; 6323 } 6324 6325 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: done. Status %d, " 6326 "Found %d GIDs", retval, *num_gids_p); 6327 6328 return (retval); 6329 } 6330 6331 /* RDMA IP CM Support routines */ 6332 ibt_status_t 6333 ibt_get_src_ip(ib_gid_t gid, ib_pkey_t pkey, ibt_ip_addr_t *src_ip) 6334 { 6335 ibcm_arp_ip_t *ipp; 6336 ibcm_arp_ibd_insts_t ibds; 6337 int i; 6338 boolean_t found = B_FALSE; 6339 ibt_status_t retval = IBT_SUCCESS; 6340 6341 IBTF_DPRINTF_L4(cmlog, "ibt_get_src_ip(%llX:%llX, %X, %p)", 6342 gid.gid_prefix, gid.gid_guid, pkey, src_ip); 6343 6344 if (gid.gid_prefix == 0 || gid.gid_guid == 0) { 6345 IBTF_DPRINTF_L3(cmlog, "ibt_get_src_ip: Invalid GID."); 6346 return (IBT_INVALID_PARAM); 6347 } 6348 6349 if (src_ip == NULL) { 6350 IBTF_DPRINTF_L3(cmlog, "ibt_get_src_ip: ERROR: src_ip NULL"); 6351 return (IBT_INVALID_PARAM); 6352 } 6353 6354 bzero(&ibds, sizeof (ibcm_arp_ibd_insts_t)); 6355 ibds.ibcm_arp_ibd_alloc = IBCM_ARP_IBD_INSTANCES; 6356 ibds.ibcm_arp_ibd_cnt = 0; 6357 ibds.ibcm_arp_ip = (ibcm_arp_ip_t *)kmem_zalloc( 6358 ibds.ibcm_arp_ibd_alloc * sizeof (ibcm_arp_ip_t), KM_SLEEP); 6359 6360 retval = ibcm_arp_get_ibds(&ibds, AF_UNSPEC); 6361 if (retval != IBT_SUCCESS) { 6362 IBTF_DPRINTF_L2(cmlog, "ibt_get_src_ip: ibcm_arp_get_ibds " 6363 "failed to get IBD Instances: ret 0x%x", retval); 6364 goto get_src_ip_end; 6365 } 6366 6367 for (i = 0, ipp = ibds.ibcm_arp_ip; i < ibds.ibcm_arp_ibd_cnt; 6368 i++, ipp++) { 6369 if (ipp->ip_port_gid.gid_prefix == gid.gid_prefix && 6370 ipp->ip_port_gid.gid_guid == gid.gid_guid) { 6371 if (pkey) { 6372 if (ipp->ip_pkey == pkey) { 6373 found = B_TRUE; 6374 break; 6375 } else 6376 continue; 6377 } 6378 found = B_TRUE; 6379 break; 6380 } 6381 } 6382 6383 if (found == B_FALSE) { 6384 retval = IBT_SRC_IP_NOT_FOUND; 6385 } else { 6386 src_ip->family = ipp->ip_inet_family; 6387 if (src_ip->family == AF_INET) { 6388 bcopy(&ipp->ip_cm_sin.sin_addr, &src_ip->un.ip4addr, 6389 sizeof (in_addr_t)); 6390 } else if (src_ip->family == AF_INET6) { 6391 bcopy(&ipp->ip_cm_sin6.sin6_addr, &src_ip->un.ip6addr, 6392 sizeof (in6_addr_t)); 6393 } 6394 IBCM_PRINT_IP("ibt_get_src_ip", src_ip); 6395 } 6396 6397 get_src_ip_end: 6398 if (ibds.ibcm_arp_ip) 6399 kmem_free(ibds.ibcm_arp_ip, 6400 ibds.ibcm_arp_ibd_alloc * sizeof (ibcm_arp_ip_t)); 6401 6402 return (retval); 6403 } 6404 6405 ib_svc_id_t 6406 ibt_get_ip_sid(uint8_t protocol_num, in_port_t dst_port) 6407 { 6408 ib_svc_id_t sid; 6409 6410 IBTF_DPRINTF_L4(cmlog, "ibt_get_ip_sid(%X, %lX)", protocol_num, 6411 dst_port); 6412 6413 /* 6414 * If protocol_num is non-zero, then formulate the SID and return it. 6415 * If protocol_num is zero, then we need to assign a locally generated 6416 * IP SID with IB_SID_IPADDR_PREFIX. 6417 */ 6418 if (protocol_num) { 6419 sid = IB_SID_IPADDR_PREFIX | protocol_num << 16 | dst_port; 6420 } else { 6421 sid = ibcm_alloc_ip_sid(); 6422 } 6423 6424 IBTF_DPRINTF_L3(cmlog, "ibt_get_ip_sid: SID: 0x%016llX", sid); 6425 return (sid); 6426 } 6427 6428 ibt_status_t 6429 ibt_release_ip_sid(ib_svc_id_t ip_sid) 6430 { 6431 IBTF_DPRINTF_L4(cmlog, "ibt_release_ip_sid(%llX)", ip_sid); 6432 6433 if (((ip_sid & IB_SID_IPADDR_PREFIX_MASK) != 0) || 6434 (!(ip_sid & IB_SID_IPADDR_PREFIX))) { 6435 IBTF_DPRINTF_L2(cmlog, "ibt_release_ip_sid(0x%016llX): ERROR: " 6436 "Called for Non-RDMA IP SID", ip_sid); 6437 return (IBT_INVALID_PARAM); 6438 } 6439 6440 /* 6441 * If protocol_num in ip_sid are all ZEROs, then this SID is allocated 6442 * by IBTF. If not, then the specified ip_sid is invalid. 6443 */ 6444 if (ip_sid & IB_SID_IPADDR_IPNUM_MASK) { 6445 IBTF_DPRINTF_L2(cmlog, "ibt_release_ip_sid(0x%016llX): ERROR: " 6446 "Called for Non-IBTF assigned RDMA IP SID", ip_sid); 6447 return (IBT_INVALID_PARAM); 6448 } 6449 6450 ibcm_free_ip_sid(ip_sid); 6451 6452 return (IBT_SUCCESS); 6453 } 6454 6455 6456 uint8_t 6457 ibt_get_ip_protocol_num(ib_svc_id_t sid) 6458 { 6459 return ((sid & IB_SID_IPADDR_IPNUM_MASK) >> 16); 6460 } 6461 6462 in_port_t 6463 ibt_get_ip_dst_port(ib_svc_id_t sid) 6464 { 6465 return (sid & IB_SID_IPADDR_PORTNUM_MASK); 6466 } 6467 6468 _NOTE(SCHEME_PROTECTS_DATA("Unshared data", ibt_ip_cm_info_t)) 6469 _NOTE(SCHEME_PROTECTS_DATA("Unshared data", ibcm_ip_pvtdata_t)) 6470 6471 ibt_status_t 6472 ibt_format_ip_private_data(ibt_ip_cm_info_t *ip_cm_info, 6473 ibt_priv_data_len_t priv_data_len, void *priv_data_p) 6474 { 6475 ibcm_ip_pvtdata_t ip_data; 6476 6477 IBTF_DPRINTF_L4(cmlog, "ibt_format_ip_private_data(%p, %d, %p)", 6478 ip_cm_info, priv_data_len, priv_data_p); 6479 6480 if ((ip_cm_info == NULL) || (priv_data_p == NULL) || 6481 (priv_data_len < IBT_IP_HDR_PRIV_DATA_SZ)) { 6482 IBTF_DPRINTF_L2(cmlog, "ibt_format_ip_private_data: ERROR " 6483 "Invalid Inputs."); 6484 return (IBT_INVALID_PARAM); 6485 } 6486 6487 bzero(&ip_data, sizeof (ibcm_ip_pvtdata_t)); 6488 ip_data.ip_srcport = ip_cm_info->src_port; /* Source Port */ 6489 6490 IBCM_PRINT_IP("format_ip_pvt: src", &ip_cm_info->src_addr); 6491 IBCM_PRINT_IP("format_ip_pvt: dst", &ip_cm_info->dst_addr); 6492 /* IPV = 0x4, if IP-Addr are IPv4 format, else 0x6 for IPv6 */ 6493 if (ip_cm_info->src_addr.family == AF_INET) { 6494 ip_data.ip_ipv = IBT_CM_IP_IPV_V4; 6495 ip_data.ip_srcv4 = ip_cm_info->src_addr.un.ip4addr; 6496 ip_data.ip_dstv4 = ip_cm_info->dst_addr.un.ip4addr; 6497 } else if (ip_cm_info->src_addr.family == AF_INET6) { 6498 ip_data.ip_ipv = IBT_CM_IP_IPV_V6; 6499 bcopy(&ip_cm_info->src_addr.un.ip6addr, 6500 &ip_data.ip_srcv6, sizeof (in6_addr_t)); 6501 bcopy(&ip_cm_info->dst_addr.un.ip6addr, 6502 &ip_data.ip_dstv6, sizeof (in6_addr_t)); 6503 } else { 6504 IBTF_DPRINTF_L2(cmlog, "ibt_format_ip_private_data: ERROR " 6505 "IP Addr needs to be either AF_INET or AF_INET6 family."); 6506 return (IBT_INVALID_PARAM); 6507 } 6508 6509 ip_data.ip_MajV = IBT_CM_IP_MAJ_VER; 6510 ip_data.ip_MinV = IBT_CM_IP_MIN_VER; 6511 6512 bcopy(&ip_data, priv_data_p, IBT_IP_HDR_PRIV_DATA_SZ); 6513 6514 return (IBT_SUCCESS); 6515 } 6516 6517 6518 ibt_status_t 6519 ibt_get_ip_data(ibt_priv_data_len_t priv_data_len, void *priv_data, 6520 ibt_ip_cm_info_t *ip_cm_infop) 6521 { 6522 ibcm_ip_pvtdata_t ip_data; 6523 6524 IBTF_DPRINTF_L4(cmlog, "ibt_get_ip_data(%d, %p, %p)", 6525 priv_data_len, priv_data, ip_cm_infop); 6526 6527 if ((ip_cm_infop == NULL) || (priv_data == NULL) || 6528 (priv_data_len < IBT_IP_HDR_PRIV_DATA_SZ)) { 6529 IBTF_DPRINTF_L2(cmlog, "ibt_get_ip_data: ERROR Invalid Inputs"); 6530 return (IBT_INVALID_PARAM); 6531 } 6532 6533 bcopy(priv_data, &ip_data, IBT_IP_HDR_PRIV_DATA_SZ); 6534 ip_cm_infop->src_port = ip_data.ip_srcport; /* Source Port */ 6535 6536 /* IPV = 0x4, if IP Address are IPv4 format, else 0x6 for IPv6 */ 6537 if (ip_data.ip_ipv == IBT_CM_IP_IPV_V4) { 6538 /* Copy IPv4 Addr */ 6539 ip_cm_infop->src_addr.family = ip_cm_infop->dst_addr.family = 6540 AF_INET; 6541 ip_cm_infop->src_addr.un.ip4addr = ip_data.ip_srcv4; 6542 ip_cm_infop->dst_addr.un.ip4addr = ip_data.ip_dstv4; 6543 } else if (ip_data.ip_ipv == IBT_CM_IP_IPV_V6) { 6544 /* Copy IPv6 Addr */ 6545 ip_cm_infop->src_addr.family = ip_cm_infop->dst_addr.family = 6546 AF_INET6; 6547 bcopy(&ip_data.ip_srcv6, &ip_cm_infop->src_addr.un.ip6addr, 6548 sizeof (in6_addr_t)); 6549 bcopy(&ip_data.ip_dstv6, &ip_cm_infop->dst_addr.un.ip6addr, 6550 sizeof (in6_addr_t)); 6551 } else { 6552 IBTF_DPRINTF_L2(cmlog, "ibt_get_ip_data: ERROR: IP Addr needs" 6553 " to be either AF_INET or AF_INET6 family."); 6554 return (IBT_INVALID_PARAM); 6555 } 6556 IBCM_PRINT_IP("ibt_get_ip_data: src", &ip_cm_infop->src_addr); 6557 IBCM_PRINT_IP("ibt_get_ip_data: dst", &ip_cm_infop->dst_addr); 6558 6559 return (IBT_SUCCESS); 6560 } 6561 6562 6563 /* Routines for warlock */ 6564 6565 /* ARGSUSED */ 6566 static void 6567 ibcm_dummy_mcg_handler(void *arg, ibt_status_t retval, ibt_mcg_info_t *minfo) 6568 { 6569 ibcm_join_mcg_tqarg_t dummy_mcg; 6570 6571 dummy_mcg.func = ibcm_dummy_mcg_handler; 6572 6573 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_mcg_handler: " 6574 "dummy_mcg.func %p", dummy_mcg.func); 6575 } 6576 6577 6578 /* ARGSUSED */ 6579 static void 6580 ibcm_dummy_recycle_rc_handler(ibt_status_t retval, void *arg) 6581 { 6582 ibcm_taskq_recycle_arg_t dummy_rc_recycle; 6583 6584 dummy_rc_recycle.func = ibcm_dummy_recycle_rc_handler; 6585 6586 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_recycle_rc_handler: " 6587 "dummy_rc_recycle.func %p", dummy_rc_recycle.func); 6588 } 6589 6590 6591 /* ARGSUSED */ 6592 static ibt_cm_status_t 6593 ibcm_dummy_ud_handler(void *priv, ibt_cm_ud_event_t *event, 6594 ibt_cm_ud_return_args_t *ret_args, 6595 void *priv_data, ibt_priv_data_len_t len) 6596 { 6597 /* 6598 * Let warlock see that ibcm_local_handler_s::actual_cm_handler 6599 * points to this routine. 6600 */ 6601 ibcm_local_handler_t p; 6602 ibcm_ud_state_data_t dummy_ud; 6603 6604 p.actual_cm_handler = ibcm_dummy_ud_handler; 6605 dummy_ud.ud_cm_handler = ibcm_dummy_ud_handler; 6606 6607 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_ud_handler: p.actual_cm_handler %p" 6608 "dummy_ud.ud_cm_handler %p", p.actual_cm_handler, 6609 dummy_ud.ud_cm_handler); 6610 /* 6611 * Call all routines that the client's callback routine could call. 6612 */ 6613 6614 return (IBT_CM_ACCEPT); 6615 } 6616 6617 /* ARGSUSED */ 6618 static ibt_cm_status_t 6619 ibcm_dummy_rc_handler(void *priv, ibt_cm_event_t *event, 6620 ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len) 6621 { 6622 ibcm_state_data_t dummy_rc; 6623 6624 dummy_rc.cm_handler = ibcm_dummy_rc_handler; 6625 6626 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_rc_handler: " 6627 "dummy_ud.ud_cm_handler %p", dummy_rc.cm_handler); 6628 /* 6629 * Call all routines that the client's callback routine could call. 6630 */ 6631 6632 return (IBT_CM_ACCEPT); 6633 } 6634