1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/types.h> 28 #include <sys/socket.h> 29 #include <sys/sockio.h> 30 #include <sys/param.h> 31 #include <sys/stat.h> 32 #include <netinet/in.h> 33 #include <arpa/inet.h> 34 #include <net/if.h> 35 #include <unistd.h> 36 #include <stdlib.h> 37 #include <string.h> 38 #include <errno.h> 39 #include <libdevinfo.h> 40 41 #include "dapl.h" 42 #include "dapl_adapter_util.h" 43 #include "dapl_tavor_ibtf_impl.h" 44 #include "dapl_hca_util.h" 45 #include "dapl_name_service.h" 46 #define IF_NAME "ibd" 47 #define MAX_HCAS 64 48 #define PROP_HCA_GUID "hca-guid" 49 #define PROP_PORT_NUM "port-number" 50 #define PROP_PORT_PKEY "port-pkey" 51 52 #define DEVDAPLT "/dev/daplt" 53 54 /* function prototypes */ 55 static DAT_RETURN dapli_process_tavor_node(di_node_t node, int *hca_idx, 56 int try_blueflame); 57 static DAT_RETURN dapli_process_ibd_node(di_node_t node, DAPL_HCA *hca_ptr, 58 int hca_idx); 59 60 #if defined(IBHOSTS_NAMING) 61 #include <stdio.h> 62 static int dapli_process_fake_ibds(DAPL_HCA **hca_list, int hca_idx); 63 #endif /* IBHOSTS_NAMING */ 64 65 static DAPL_OS_LOCK g_tavor_state_lock; 66 static struct dapls_ib_hca_state g_tavor_state[MAX_HCAS]; 67 DAPL_OS_LOCK g_tavor_uar_lock; 68 69 DAT_RETURN 70 dapli_init_hca( 71 IN DAPL_HCA *hca_ptr) 72 { 73 di_node_t root_node; 74 di_node_t hca_node; 75 di_node_t ibd_node; 76 DAT_RETURN dat_status = DAT_SUCCESS; 77 int hca_idx = 0; 78 int ia_instance; 79 int check_for_bf = 0; 80 81 ia_instance = (int)dapl_os_strtol(hca_ptr->name + strlen(IF_NAME), 82 NULL, 0); 83 84 root_node = di_init("/", DINFOCPYALL); 85 if (root_node == DI_NODE_NIL) { 86 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 87 "init_hca: di_init failed %s\n", strerror(errno)); 88 return (DAT_INTERNAL_ERROR); 89 } 90 91 ibd_node = di_drv_first_node(IF_NAME, root_node); 92 while (ibd_node != DI_NODE_NIL) { 93 /* find the ibd node matching our ianame */ 94 if (di_instance(ibd_node) == ia_instance) { 95 break; 96 } 97 ibd_node = di_drv_next_node(ibd_node); 98 } 99 100 if (ibd_node == DI_NODE_NIL) { 101 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0); 102 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 103 "init_hcas: ibd%d di_node not found\n", ia_instance); 104 goto bail; 105 } 106 107 hca_node = di_parent_node(ibd_node); 108 if ((hca_node != DI_NODE_NIL) && (strncmp(di_driver_name(hca_node), 109 "tavor", strlen("tavor")) == 0)) 110 dapls_init_funcs_tavor(hca_ptr); 111 else if ((hca_node != DI_NODE_NIL) && (strncmp(di_driver_name 112 (hca_node), "arbel", strlen("arbel")) == 0)) 113 dapls_init_funcs_arbel(hca_ptr); 114 else if ((hca_node != DI_NODE_NIL) && (strncmp(di_driver_name 115 (hca_node), "hermon", strlen("hermon")) == 0)) { 116 dapls_init_funcs_hermon(hca_ptr); 117 check_for_bf = 1; 118 } else { 119 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0); 120 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 121 "init_hcas: ibd%d hca_node not found\n", ia_instance); 122 goto bail; 123 } 124 125 dat_status = dapli_process_tavor_node(hca_node, &hca_idx, check_for_bf); 126 if (dat_status != DAT_SUCCESS) { 127 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 128 "init_hcas: ibd%d process_tavor_node failed(0x%x)\n", 129 ia_instance, dat_status); 130 goto bail; 131 } 132 133 #if defined(IBHOSTS_NAMING) 134 if (dapli_process_fake_ibds(hca_ptr, hca_idx) == 0) { 135 /* no entries were found */ 136 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0); 137 } 138 #else 139 dat_status = dapli_process_ibd_node(ibd_node, hca_ptr, hca_idx); 140 #endif 141 if (dat_status != DAT_SUCCESS) { 142 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 143 "init_hcas: ibd%d process_ibd_node failed(0x%x)\n", 144 ia_instance, dat_status); 145 goto bail; 146 } 147 148 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 149 "init_hcas: done ibd%d\n", ia_instance); 150 151 bail: 152 di_fini(root_node); 153 return (dat_status); 154 } 155 156 static DAT_RETURN 157 dapli_process_tavor_node(di_node_t node, int *hca_idx, int try_blueflame) 158 { 159 char *dev_path; 160 char path_buf[MAXPATHLEN]; 161 int i, idx, fd; 162 #ifndef _LP64 163 int tmpfd; 164 #endif 165 size_t pagesize; 166 void *mapaddr; 167 pid_t cur_pid; 168 off64_t uarpg_offset; 169 170 dapl_os_lock(&g_tavor_state_lock); 171 172 for (idx = 0; idx < MAX_HCAS; idx++) { 173 /* 174 * page size == 0 means this entry is not occupied 175 */ 176 if (g_tavor_state[idx].uarpg_size == 0) { 177 break; 178 } 179 } 180 if (idx == MAX_HCAS) { 181 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 182 "process_tavor: all hcas are being used!\n"); 183 dapl_os_unlock(&g_tavor_state_lock); 184 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 185 } 186 dev_path = di_devfs_path(node); 187 188 for (i = 0; i < idx; i++) { 189 if (strcmp(dev_path, g_tavor_state[i].hca_path) == 0) { 190 /* no need for a refcnt */ 191 idx = i; 192 goto done; 193 } 194 } 195 196 /* Add 16 to accomodate the prefix "/devices" and suffix ":devctl" */ 197 if (strlen("/devices") + strlen(dev_path) + strlen(":devctl") + 1 > 198 MAXPATHLEN) { 199 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 200 "process_tavor: devfs path %s is too long\n", 201 dev_path); 202 di_devfs_path_free(dev_path); 203 dapl_os_unlock(&g_tavor_state_lock); 204 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 205 } 206 (void) dapl_os_strcpy(path_buf, "/devices"); 207 (void) dapl_os_strcat(path_buf, dev_path); 208 (void) dapl_os_strcat(path_buf, ":devctl"); 209 (void) dapl_os_strcpy(g_tavor_state[idx].hca_path, dev_path); 210 di_devfs_path_free(dev_path); 211 212 pagesize = (size_t)sysconf(_SC_PAGESIZE); 213 if (pagesize == 0) { 214 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 215 "process_tavor: page_size == 0\n"); 216 dapl_os_unlock(&g_tavor_state_lock); 217 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 218 } 219 cur_pid = getpid(); 220 221 fd = open(path_buf, O_RDWR); 222 if (fd < 0) { 223 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 224 "process_tavor: cannot open %s: %s\n", 225 path_buf, strerror(errno)); 226 dapl_os_unlock(&g_tavor_state_lock); 227 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 228 } 229 #ifndef _LP64 230 /* 231 * libc can't handle fd's greater than 255, in order to 232 * ensure that these values remain available make fd > 255. 233 * Note: not needed for LP64 234 */ 235 tmpfd = fcntl(fd, F_DUPFD, 256); 236 if (tmpfd < 0) { 237 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 238 "process_tavor: cannot F_DUPFD: %s\n", strerror(errno)); 239 } else { 240 (void) close(fd); 241 fd = tmpfd; 242 } 243 #endif /* _LP64 */ 244 245 if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) { 246 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 247 "process_tavor: cannot F_SETFD: %s\n", strerror(errno)); 248 (void) close(fd); 249 dapl_os_unlock(&g_tavor_state_lock); 250 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 251 } 252 253 uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) | 254 MLNX_UMAP_UARPG_RSRC) * pagesize; 255 256 mapaddr = mmap64((void *)0, pagesize, PROT_READ | PROT_WRITE, 257 MAP_SHARED, fd, uarpg_offset); 258 if (mapaddr == MAP_FAILED) { 259 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 260 "process_tavor: mmap failed %s\n", strerror(errno)); 261 (void) close(fd); 262 dapl_os_unlock(&g_tavor_state_lock); 263 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 264 } 265 266 g_tavor_state[idx].hca_fd = fd; 267 g_tavor_state[idx].uarpg_baseaddr = mapaddr; 268 g_tavor_state[idx].uarpg_size = pagesize; 269 270 if (try_blueflame == 0) 271 goto done; 272 273 /* Try to do the Hermon Blueflame page mapping */ 274 uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) | 275 MLNX_UMAP_BLUEFLAMEPG_RSRC) * pagesize; 276 277 mapaddr = mmap64((void *)0, pagesize, PROT_READ | PROT_WRITE, 278 MAP_SHARED, fd, uarpg_offset); 279 if (mapaddr == MAP_FAILED) { 280 /* This is not considered to be fatal. Charge on! */ 281 dapl_dbg_log(DAPL_DBG_TYPE_WARN, 282 "process_tavor: mmap of blueflame page failed %s\n", 283 strerror(errno)); 284 } else { 285 g_tavor_state[idx].bf_pg_baseaddr = mapaddr; 286 g_tavor_state[idx].bf_toggle = 0; 287 } 288 done: 289 dapl_os_unlock(&g_tavor_state_lock); 290 291 *hca_idx = idx; 292 293 return (DAT_SUCCESS); 294 } 295 296 static DAT_RETURN 297 dapli_process_ibd_node(di_node_t node, DAPL_HCA *hca_ptr, int hca_idx) 298 { 299 di_prop_t prop; 300 ib_guid_t hca_guid = 0; 301 struct lifreq lifreq; 302 uint32_t port_num = 0; 303 uint32_t partition_key = 0; 304 int instance, sfd, retval, af; 305 int tmp; 306 int digits; 307 char *drv_name; 308 char addr_buf[64]; 309 310 prop = di_prop_next(node, DI_PROP_NIL); 311 while (prop != DI_PROP_NIL) { 312 char *prop_name; 313 uchar_t *bytep; 314 int *intp, count; 315 316 prop_name = di_prop_name(prop); 317 count = 0; 318 319 if (strcmp(prop_name, PROP_HCA_GUID) == 0) { 320 count = di_prop_bytes(prop, &bytep); 321 dapl_os_assert(count == sizeof (ib_guid_t)); 322 (void) dapl_os_memcpy((void *)&hca_guid, (void *)bytep, 323 sizeof (ib_guid_t)); 324 } else if (strcmp(prop_name, PROP_PORT_NUM) == 0) { 325 count = di_prop_ints(prop, &intp); 326 dapl_os_assert(count == 1); 327 port_num = (uint32_t)intp[0]; 328 } else if (strcmp(prop_name, PROP_PORT_PKEY) == 0) { 329 count = di_prop_ints(prop, &intp); 330 dapl_os_assert(count == 1); 331 partition_key = (uint32_t)intp[0]; 332 } 333 prop = di_prop_next(node, prop); 334 } 335 if (hca_guid == 0 || port_num == 0 || partition_key == 0) { 336 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 337 "process_ibd: invalid properties: guid 0x%016llx, " 338 "port %d, pkey 0x%08x\n", hca_guid, port_num, 339 partition_key); 340 return (DAT_ERROR(DAT_INVALID_PARAMETER, 0)); 341 } 342 343 /* 344 * if an interface has both v4 and v6 addresses plumbed, 345 * we'll take the v4 address. 346 */ 347 af = AF_INET; 348 again: 349 sfd = socket(af, SOCK_DGRAM, 0); 350 if (sfd < 0) { 351 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 352 "process_ibd: socket failed: %s\n", strerror(errno)); 353 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 354 } 355 instance = di_instance(node); 356 drv_name = di_driver_name(node); 357 358 /* calculate the number of digits in instance */ 359 tmp = instance; 360 digits = 0; 361 do { 362 tmp = tmp / 10; 363 digits++; 364 } while (tmp > 0); 365 /* check if name will fit in lifr_name */ 366 if (dapl_os_strlen(drv_name) + digits + 1 > LIFNAMSIZ) { 367 (void) close(sfd); 368 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 369 "process_ibd: if name overflow %s:%d\n", 370 drv_name, instance); 371 return (DAT_ERROR(DAT_INVALID_PARAMETER, 0)); 372 } 373 374 (void) dapl_os_strcpy(lifreq.lifr_name, drv_name); 375 (void) sprintf(&lifreq.lifr_name[dapl_os_strlen(drv_name)], "%d", 376 instance); 377 retval = ioctl(sfd, SIOCGLIFADDR, (caddr_t)&lifreq); 378 if (retval < 0) { 379 (void) close(sfd); 380 if (af == AF_INET6) { 381 /* 382 * the interface is not plumbed. 383 */ 384 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 385 "process_ibd: %s: ip address not found\n", 386 lifreq.lifr_name); 387 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 388 } else { 389 /* 390 * we've failed to find a v4 address. now 391 * let's try v6. 392 */ 393 af = AF_INET6; 394 goto again; 395 } 396 } 397 (void) close(sfd); 398 399 hca_ptr->hca_ibd_inst = instance; 400 hca_ptr->tavor_idx = hca_idx; 401 hca_ptr->node_GUID = hca_guid; 402 hca_ptr->port_num = port_num; 403 hca_ptr->partition_key = partition_key; 404 (void) dapl_os_memcpy((void *)&hca_ptr->hca_address, 405 (void *)&lifreq.lifr_addr, sizeof (hca_ptr->hca_address)); 406 hca_ptr->max_inline_send = dapls_tavor_max_inline(); 407 408 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 409 "process_ibd: interface %s, hca guid 0x%016llx, port %d, " 410 "pkey 0x%08x, ip addr %s\n", lifreq.lifr_name, hca_guid, 411 port_num, partition_key, dapls_inet_ntop( 412 (struct sockaddr *)&hca_ptr->hca_address, addr_buf, 64)); 413 return (DAT_SUCCESS); 414 } 415 416 void 417 dapls_ib_state_init(void) 418 { 419 int i; 420 421 (void) dapl_os_lock_init(&g_tavor_state_lock); 422 (void) dapl_os_lock_init(&g_tavor_uar_lock); 423 (void) dapl_os_lock_init(&dapls_ib_dbp_lock); 424 425 for (i = 0; i < MAX_HCAS; i++) { 426 g_tavor_state[i].hca_fd = 0; 427 g_tavor_state[i].uarpg_baseaddr = NULL; 428 g_tavor_state[i].uarpg_size = 0; 429 g_tavor_state[i].bf_pg_baseaddr = NULL; 430 } 431 } 432 433 void 434 dapls_ib_state_fini(void) 435 { 436 int i, count = 0; 437 438 /* 439 * Uinitialize the per hca instance state 440 */ 441 dapl_os_lock(&g_tavor_state_lock); 442 for (i = 0; i < MAX_HCAS; i++) { 443 if (g_tavor_state[i].uarpg_size == 0) { 444 dapl_os_assert(g_tavor_state[i].uarpg_baseaddr == 445 NULL); 446 continue; 447 } 448 if (munmap(g_tavor_state[i].uarpg_baseaddr, 449 g_tavor_state[i].uarpg_size) < 0) { 450 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 451 "ib_state_fini: " 452 "munmap(0x%p, 0x%llx) failed(%d)\n", 453 g_tavor_state[i].uarpg_baseaddr, 454 g_tavor_state[i].uarpg_size, errno); 455 } 456 if ((g_tavor_state[i].bf_pg_baseaddr != NULL) && 457 (munmap(g_tavor_state[i].bf_pg_baseaddr, 458 g_tavor_state[i].uarpg_size) < 0)) { 459 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 460 "ib_state_fini: " 461 "munmap(0x%p, 0x%llx) of blueflame failed(%d)\n", 462 g_tavor_state[i].bf_pg_baseaddr, 463 g_tavor_state[i].uarpg_size, errno); 464 } 465 466 (void) close(g_tavor_state[i].hca_fd); 467 count++; 468 } 469 dapl_os_unlock(&g_tavor_state_lock); 470 471 dapl_os_lock_destroy(&g_tavor_uar_lock); 472 dapl_os_lock_destroy(&g_tavor_state_lock); 473 dapl_os_lock_destroy(&dapls_ib_dbp_lock); 474 475 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 476 "ib_state_fini: cleaned %d hcas\n", count); 477 } 478 479 /* 480 * dapls_ib_open_hca 481 * 482 * Open HCA 483 * 484 * Input: 485 * *hca_ptr pointer to hca device 486 * *ib_hca_handle_p pointer to provide HCA handle 487 * 488 * Output: 489 * none 490 * 491 * Return: 492 * DAT_SUCCESS 493 * DAT_INSUFFICIENT_RESOURCES 494 * 495 */ 496 DAT_RETURN 497 dapls_ib_open_hca( 498 IN DAPL_HCA *hca_ptr, 499 OUT ib_hca_handle_t *ib_hca_handle_p) 500 { 501 dapl_ia_create_t args; 502 DAT_RETURN dat_status; 503 struct dapls_ib_hca_handle *hca_p; 504 int fd; 505 #ifndef _LP64 506 int tmpfd; 507 #endif 508 int retval; 509 struct sockaddr *s; 510 struct sockaddr_in6 *v6addr; 511 struct sockaddr_in *v4addr; 512 dapl_ia_addr_t *sap; 513 514 dat_status = dapli_init_hca(hca_ptr); 515 if (dat_status != DAT_SUCCESS) { 516 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 517 "dapls_ib_open_hca: init_hca failed %d\n", dat_status); 518 return (dat_status); 519 } 520 521 fd = open(DEVDAPLT, O_RDONLY); 522 if (fd < 0) { 523 return (DAT_INSUFFICIENT_RESOURCES); 524 } 525 526 #ifndef _LP64 527 /* 528 * libc can't handle fd's greater than 255, in order to 529 * ensure that these values remain available make fd > 255. 530 * Note: not needed for LP64 531 */ 532 tmpfd = fcntl(fd, F_DUPFD, 256); 533 if (tmpfd < 0) { 534 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 535 "dapls_ib_open_hca: cannot F_DUPFD: %s\n", 536 strerror(errno)); 537 } else { 538 (void) close(fd); 539 fd = tmpfd; 540 } 541 #endif /* _LP64 */ 542 543 if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) { 544 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 545 "dapls_ib_open_hca: cannot F_SETFD: %s\n", strerror(errno)); 546 (void) close(fd); 547 return (DAT_INTERNAL_ERROR); 548 } 549 550 hca_p = (struct dapls_ib_hca_handle *)dapl_os_alloc( 551 sizeof (struct dapls_ib_hca_handle)); 552 if (hca_p == NULL) { 553 (void) close(fd); 554 return (DAT_INSUFFICIENT_RESOURCES); 555 } 556 557 args.ia_guid = hca_ptr->node_GUID; 558 args.ia_port = hca_ptr->port_num; 559 args.ia_pkey = hca_ptr->partition_key; 560 args.ia_version = DAPL_IF_VERSION; 561 (void) dapl_os_memzero((void *)args.ia_sadata, DAPL_ATS_NBYTES); 562 563 /* pass down local ip address to be stored in SA */ 564 s = (struct sockaddr *)&hca_ptr->hca_address; 565 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 566 sap = (dapl_ia_addr_t *)args.ia_sadata; 567 switch (s->sa_family) { 568 case AF_INET: 569 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 570 v4addr = (struct sockaddr_in *)s; 571 sap->iad_v4 = v4addr->sin_addr; 572 break; 573 case AF_INET6: 574 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 575 v6addr = (struct sockaddr_in6 *)s; 576 sap->iad_v6 = v6addr->sin6_addr; 577 break; 578 default: 579 break; /* fall through */ 580 } 581 582 retval = ioctl(fd, DAPL_IA_CREATE, &args); 583 if (retval != 0) { 584 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 585 "open_hca: ia_create failed, fd %d, " 586 "guid 0x%016llx, port %d, pkey 0x%x, version %d\n", 587 fd, args.ia_guid, args.ia_port, args.ia_pkey, 588 args.ia_version); 589 590 dapl_os_free(hca_p, sizeof (*hca_p)); 591 (void) close(fd); 592 return (dapls_convert_error(errno, retval)); 593 } 594 595 hca_p->ia_fd = fd; 596 hca_p->ia_rnum = args.ia_resnum; 597 hca_p->hca_fd = g_tavor_state[hca_ptr->tavor_idx].hca_fd; 598 hca_p->ia_uar = g_tavor_state[hca_ptr->tavor_idx].uarpg_baseaddr; 599 hca_p->ia_bf = g_tavor_state[hca_ptr->tavor_idx].bf_pg_baseaddr; 600 hca_p->ia_bf_toggle = &g_tavor_state[hca_ptr->tavor_idx].bf_toggle; 601 *ib_hca_handle_p = hca_p; 602 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 603 "open_hca: ia_created, hca_p 0x%p, fd %d, " 604 "rnum %d, guid 0x%016llx, port %d, pkey 0x%x\n", 605 hca_p, hca_p->ia_fd, hca_p->ia_rnum, hca_ptr->node_GUID, 606 hca_ptr->port_num, hca_ptr->partition_key); 607 608 return (DAT_SUCCESS); 609 } 610 611 /* 612 * dapls_ib_close_hca 613 * 614 * Open HCA 615 * 616 * Input: 617 * ib_hca_handle provide HCA handle 618 * 619 * Output: 620 * none 621 * 622 * Return: 623 * DAT_SUCCESS 624 * DAT_INSUFFICIENT_RESOURCES 625 * 626 */ 627 DAT_RETURN 628 dapls_ib_close_hca( 629 IN ib_hca_handle_t ib_hca_handle) 630 { 631 if (ib_hca_handle == NULL) { 632 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 633 "close_hca: ib_hca_handle == NULL\n"); 634 return (DAT_SUCCESS); 635 } 636 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 637 "close_hca: closing hca 0x%p, fd %d, rnum %d\n", 638 ib_hca_handle, ib_hca_handle->ia_fd, ib_hca_handle->ia_rnum); 639 640 (void) close(ib_hca_handle->ia_fd); 641 dapl_os_free((void *)ib_hca_handle, 642 sizeof (struct dapls_ib_hca_handle)); 643 return (DAT_SUCCESS); 644 } 645 646 #if defined(IBHOSTS_NAMING) 647 #define LINE_LEN 256 648 static int 649 dapli_process_fake_ibds(DAPL_HCA *hca_ptr, int hca_idx) 650 { 651 char line_buf[LINE_LEN]; 652 char host_buf[LINE_LEN]; 653 char localhost[LINE_LEN]; 654 ib_guid_t prefix; 655 ib_guid_t guid; 656 FILE *fp; 657 int count = 0; 658 DAPL_HCA *hca_ptr; 659 660 fp = fopen("/etc/dapl/ibhosts", "r"); 661 if (fp == NULL) { 662 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 663 "fake_ibds: ibhosts not found!\n"); 664 return (0); 665 } 666 if (gethostname(localhost, LINE_LEN) != 0) { 667 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 668 "fake_ibds: hostname not found!\n"); 669 return (0); 670 } 671 while (!feof(fp)) { 672 (void) fgets(line_buf, LINE_LEN, fp); 673 sscanf(line_buf, "%s %llx %llx", host_buf, &prefix, &guid); 674 (void) sprintf(line_buf, "%s-ib%d", localhost, count + 1); 675 if (strncmp(line_buf, host_buf, strlen(line_buf)) == 0) { 676 guid &= 0xfffffffffffffff0; 677 hca_ptr->hca_ibd_inst = count + 1; 678 hca_ptr->tavor_idx = hca_idx; 679 hca_ptr->node_GUID = guid; 680 hca_ptr->port_num = count + 1; 681 hca_ptr->partition_key = 0x0000ffff; 682 count++; 683 } 684 if (count >= 2) break; 685 } 686 (void) fclose(fp); 687 return (count); 688 } 689 690 #endif /* IBHOSTS_NAMING */ 691