1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/socket.h> 28 #include <sys/sockio.h> 29 #include <sys/param.h> 30 #include <sys/stat.h> 31 #include <netinet/in.h> 32 #include <arpa/inet.h> 33 #include <net/if.h> 34 #include <unistd.h> 35 #include <stdlib.h> 36 #include <string.h> 37 #include <errno.h> 38 #include <strings.h> 39 #include <fcntl.h> 40 #include <libdladm.h> 41 #include <libdlib.h> 42 #include <libdllink.h> 43 #include <sys/ib/ibnex/ibnex_devctl.h> 44 45 #include "dapl.h" 46 #include "dapl_adapter_util.h" 47 #include "dapl_tavor_ibtf_impl.h" 48 #include "dapl_hca_util.h" 49 #include "dapl_name_service.h" 50 #define MAX_HCAS 64 51 #define PROP_HCA_GUID "hca-guid" 52 #define PROP_PORT_NUM "port-number" 53 #define PROP_PORT_PKEY "port-pkey" 54 55 #define DEVDAPLT "/dev/daplt" 56 57 /* function prototypes */ 58 static DAT_RETURN dapli_process_tavor_node(char *dev_path, int *hca_idx, 59 int try_blueflame); 60 static DAT_RETURN dapli_process_ia(dladm_ib_attr_t *ib_attr, DAPL_HCA *hca_ptr, 61 int hca_idx); 62 63 #if defined(IBHOSTS_NAMING) 64 #include <stdio.h> 65 static int dapli_process_fake_ibds(DAPL_HCA **hca_list, int hca_idx); 66 #endif /* IBHOSTS_NAMING */ 67 68 static DAPL_OS_LOCK g_tavor_state_lock; 69 static struct dapls_ib_hca_state g_tavor_state[MAX_HCAS]; 70 DAPL_OS_LOCK g_tavor_uar_lock; 71 72 DAT_RETURN 73 dapli_init_hca( 74 IN DAPL_HCA *hca_ptr) 75 { 76 DAT_RETURN dat_status = DAT_SUCCESS; 77 int hca_idx = 0; 78 int check_for_bf = 0; 79 datalink_class_t class; 80 datalink_id_t linkid; 81 dladm_ib_attr_t ib_attr; 82 ibnex_ctl_query_hca_t query_hca; 83 int ibnex_fd = -1; 84 dladm_handle_t dlh; 85 char hca_device_path[MAXPATHLEN]; 86 87 if (dladm_open(&dlh) != DLADM_STATUS_OK) { 88 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 89 "init_hca: dladm_open failed\n"); 90 return (DAT_INTERNAL_ERROR); 91 } 92 93 if ((ibnex_fd = open(IBNEX_DEVCTL_DEV, O_RDONLY)) < 0) { 94 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0); 95 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 96 "init_hca: could not open ib nexus (%s)\n", 97 strerror(errno)); 98 goto bail; 99 } 100 101 if ((dladm_name2info(dlh, hca_ptr->name, &linkid, NULL, &class, 102 NULL) != DLADM_STATUS_OK) || 103 (class != DATALINK_CLASS_PART) || 104 (dladm_part_info(dlh, linkid, &ib_attr, 105 DLADM_OPT_ACTIVE) != DLADM_STATUS_OK)) { 106 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0); 107 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 108 "init_hca: %s not found - couldn't get partition info\n", 109 hca_ptr->name); 110 goto bail; 111 } 112 113 bzero(&query_hca, sizeof (query_hca)); 114 query_hca.hca_guid = ib_attr.dia_hca_guid; 115 query_hca.hca_device_path = hca_device_path; 116 query_hca.hca_device_path_alloc_sz = sizeof (hca_device_path); 117 if (ioctl(ibnex_fd, IBNEX_CTL_QUERY_HCA, &query_hca) == -1) { 118 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0); 119 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 120 "init_hca: %s not found; query_hca failed\n", 121 hca_ptr->name); 122 goto bail; 123 } 124 125 if (strcmp(query_hca.hca_info.hca_driver_name, "tavor") == 0) 126 dapls_init_funcs_tavor(hca_ptr); 127 else if (strcmp(query_hca.hca_info.hca_driver_name, "arbel") == 0) 128 dapls_init_funcs_arbel(hca_ptr); 129 else if (strcmp(query_hca.hca_info.hca_driver_name, "hermon") == 0) { 130 dapls_init_funcs_hermon(hca_ptr); 131 check_for_bf = 1; 132 } else { 133 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0); 134 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 135 "init_hca: %s not found\n", hca_ptr->name); 136 goto bail; 137 } 138 139 dat_status = dapli_process_tavor_node(hca_device_path, &hca_idx, 140 check_for_bf); 141 if (dat_status != DAT_SUCCESS) { 142 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 143 "init_hcas: %s process_tavor_node failed(0x%x)\n", 144 hca_ptr->name, dat_status); 145 goto bail; 146 } 147 148 #if defined(IBHOSTS_NAMING) 149 if (dapli_process_fake_ibds(hca_ptr, hca_idx) == 0) { 150 /* no entries were found */ 151 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0); 152 } 153 #else 154 dat_status = dapli_process_ia(&ib_attr, hca_ptr, hca_idx); 155 #endif 156 if (dat_status != DAT_SUCCESS) { 157 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 158 "init_hcas: %s process_ia failed(0x%x)\n", 159 hca_ptr->name, dat_status); 160 goto bail; 161 } 162 163 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 164 "init_hcas: done %s\n", hca_ptr->name); 165 166 bail: 167 if (ibnex_fd != -1) 168 (void) close(ibnex_fd); 169 dladm_close(dlh); 170 return (dat_status); 171 } 172 173 static DAT_RETURN 174 dapli_process_tavor_node(char *dev_path, int *hca_idx, int try_blueflame) 175 { 176 char path_buf[MAXPATHLEN]; 177 int i, idx, fd; 178 #ifndef _LP64 179 int tmpfd; 180 #endif 181 size_t pagesize; 182 void *mapaddr; 183 pid_t cur_pid; 184 off64_t uarpg_offset; 185 186 dapl_os_lock(&g_tavor_state_lock); 187 188 for (idx = 0; idx < MAX_HCAS; idx++) { 189 /* 190 * page size == 0 means this entry is not occupied 191 */ 192 if (g_tavor_state[idx].uarpg_size == 0) { 193 break; 194 } 195 } 196 if (idx == MAX_HCAS) { 197 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 198 "process_tavor: all hcas are being used!\n"); 199 dapl_os_unlock(&g_tavor_state_lock); 200 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 201 } 202 203 for (i = 0; i < idx; i++) { 204 if (strcmp(dev_path, g_tavor_state[i].hca_path) == 0) { 205 /* no need for a refcnt */ 206 idx = i; 207 goto done; 208 } 209 } 210 211 /* Add 16 to accomodate the prefix "/devices" and suffix ":devctl" */ 212 if (strlen("/devices") + strlen(dev_path) + strlen(":devctl") + 1 > 213 MAXPATHLEN) { 214 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 215 "process_tavor: devfs path %s is too long\n", 216 dev_path); 217 dapl_os_unlock(&g_tavor_state_lock); 218 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 219 } 220 (void) dapl_os_strcpy(path_buf, "/devices"); 221 (void) dapl_os_strcat(path_buf, dev_path); 222 (void) dapl_os_strcat(path_buf, ":devctl"); 223 (void) dapl_os_strcpy(g_tavor_state[idx].hca_path, dev_path); 224 225 pagesize = (size_t)sysconf(_SC_PAGESIZE); 226 if (pagesize == 0) { 227 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 228 "process_tavor: page_size == 0\n"); 229 dapl_os_unlock(&g_tavor_state_lock); 230 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 231 } 232 cur_pid = getpid(); 233 234 fd = open(path_buf, O_RDWR); 235 if (fd < 0) { 236 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 237 "process_tavor: cannot open %s: %s\n", 238 path_buf, strerror(errno)); 239 dapl_os_unlock(&g_tavor_state_lock); 240 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 241 } 242 #ifndef _LP64 243 /* 244 * libc can't handle fd's greater than 255, in order to 245 * ensure that these values remain available make fd > 255. 246 * Note: not needed for LP64 247 */ 248 tmpfd = fcntl(fd, F_DUPFD, 256); 249 if (tmpfd < 0) { 250 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 251 "process_tavor: cannot F_DUPFD: %s\n", strerror(errno)); 252 } else { 253 (void) close(fd); 254 fd = tmpfd; 255 } 256 #endif /* _LP64 */ 257 258 if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) { 259 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 260 "process_tavor: cannot F_SETFD: %s\n", strerror(errno)); 261 (void) close(fd); 262 dapl_os_unlock(&g_tavor_state_lock); 263 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 264 } 265 266 uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) | 267 MLNX_UMAP_UARPG_RSRC) * pagesize; 268 269 mapaddr = mmap64((void *)0, pagesize, PROT_READ | PROT_WRITE, 270 MAP_SHARED, fd, uarpg_offset); 271 if (mapaddr == MAP_FAILED) { 272 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 273 "process_tavor: mmap failed %s\n", strerror(errno)); 274 (void) close(fd); 275 dapl_os_unlock(&g_tavor_state_lock); 276 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 277 } 278 279 g_tavor_state[idx].hca_fd = fd; 280 g_tavor_state[idx].uarpg_baseaddr = mapaddr; 281 g_tavor_state[idx].uarpg_size = pagesize; 282 283 if (try_blueflame == 0) 284 goto done; 285 286 /* Try to do the Hermon Blueflame page mapping */ 287 uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) | 288 MLNX_UMAP_BLUEFLAMEPG_RSRC) * pagesize; 289 290 mapaddr = mmap64((void *)0, pagesize, PROT_READ | PROT_WRITE, 291 MAP_SHARED, fd, uarpg_offset); 292 if (mapaddr == MAP_FAILED) { 293 /* This is not considered to be fatal. Charge on! */ 294 dapl_dbg_log(DAPL_DBG_TYPE_WARN, 295 "process_tavor: mmap of blueflame page failed %s\n", 296 strerror(errno)); 297 } else { 298 g_tavor_state[idx].bf_pg_baseaddr = mapaddr; 299 g_tavor_state[idx].bf_toggle = 0; 300 } 301 done: 302 dapl_os_unlock(&g_tavor_state_lock); 303 304 *hca_idx = idx; 305 306 return (DAT_SUCCESS); 307 } 308 309 static DAT_RETURN 310 dapli_process_ia(dladm_ib_attr_t *ib_attr, DAPL_HCA *hca_ptr, int hca_idx) 311 { 312 struct lifreq lifreq; 313 int sfd, retval, af; 314 char addr_buf[64]; 315 316 if (ib_attr->dia_hca_guid == 0 || ib_attr->dia_portnum == 0 || 317 ib_attr->dia_pkey == 0) { 318 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 319 "process_ia: invalid properties: guid 0x%016llx, " 320 "port %d, pkey 0x%08x\n", ib_attr->dia_hca_guid, 321 ib_attr->dia_portnum, (uint_t)ib_attr->dia_pkey); 322 return (DAT_ERROR(DAT_INVALID_PARAMETER, 0)); 323 } 324 325 /* 326 * if an interface has both v4 and v6 addresses plumbed, 327 * we'll take the v4 address. 328 */ 329 af = AF_INET; 330 again: 331 sfd = socket(af, SOCK_DGRAM, 0); 332 if (sfd < 0) { 333 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 334 "process_ia: socket failed: %s\n", strerror(errno)); 335 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 336 } 337 338 /* check if name will fit in lifr_name */ 339 if (dapl_os_strlen(hca_ptr->name) >= LIFNAMSIZ) { 340 (void) close(sfd); 341 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 342 "process_ia: if name overflow %s\n", 343 hca_ptr->name); 344 return (DAT_ERROR(DAT_INVALID_PARAMETER, 0)); 345 } 346 347 (void) dapl_os_strcpy(lifreq.lifr_name, hca_ptr->name); 348 retval = ioctl(sfd, SIOCGLIFADDR, (caddr_t)&lifreq); 349 if (retval < 0) { 350 (void) close(sfd); 351 if (af == AF_INET6) { 352 /* 353 * the interface is not plumbed. 354 */ 355 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 356 "process_ia: %s: ip address not found\n", 357 lifreq.lifr_name); 358 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 359 } else { 360 /* 361 * we've failed to find a v4 address. now 362 * let's try v6. 363 */ 364 af = AF_INET6; 365 goto again; 366 } 367 } 368 (void) close(sfd); 369 370 hca_ptr->tavor_idx = hca_idx; 371 hca_ptr->node_GUID = ib_attr->dia_hca_guid; 372 hca_ptr->port_num = ib_attr->dia_portnum; 373 hca_ptr->partition_key = ib_attr->dia_pkey; 374 (void) dapl_os_memcpy((void *)&hca_ptr->hca_address, 375 (void *)&lifreq.lifr_addr, sizeof (hca_ptr->hca_address)); 376 hca_ptr->max_inline_send = dapls_tavor_max_inline(); 377 378 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 379 "process_ia: interface %s, hca guid 0x%016llx, port %d, " 380 "pkey 0x%08x, ip addr %s\n", lifreq.lifr_name, hca_ptr->node_GUID, 381 hca_ptr->port_num, hca_ptr->partition_key, dapls_inet_ntop( 382 (struct sockaddr *)&hca_ptr->hca_address, addr_buf, 64)); 383 return (DAT_SUCCESS); 384 } 385 386 void 387 dapls_ib_state_init(void) 388 { 389 int i; 390 391 (void) dapl_os_lock_init(&g_tavor_state_lock); 392 (void) dapl_os_lock_init(&g_tavor_uar_lock); 393 (void) dapl_os_lock_init(&dapls_ib_dbp_lock); 394 395 for (i = 0; i < MAX_HCAS; i++) { 396 g_tavor_state[i].hca_fd = 0; 397 g_tavor_state[i].uarpg_baseaddr = NULL; 398 g_tavor_state[i].uarpg_size = 0; 399 g_tavor_state[i].bf_pg_baseaddr = NULL; 400 } 401 } 402 403 void 404 dapls_ib_state_fini(void) 405 { 406 int i, count = 0; 407 408 /* 409 * Uinitialize the per hca instance state 410 */ 411 dapl_os_lock(&g_tavor_state_lock); 412 for (i = 0; i < MAX_HCAS; i++) { 413 if (g_tavor_state[i].uarpg_size == 0) { 414 dapl_os_assert(g_tavor_state[i].uarpg_baseaddr == 415 NULL); 416 continue; 417 } 418 if (munmap(g_tavor_state[i].uarpg_baseaddr, 419 g_tavor_state[i].uarpg_size) < 0) { 420 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 421 "ib_state_fini: " 422 "munmap(0x%p, 0x%llx) failed(%d)\n", 423 g_tavor_state[i].uarpg_baseaddr, 424 g_tavor_state[i].uarpg_size, errno); 425 } 426 if ((g_tavor_state[i].bf_pg_baseaddr != NULL) && 427 (munmap(g_tavor_state[i].bf_pg_baseaddr, 428 g_tavor_state[i].uarpg_size) < 0)) { 429 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 430 "ib_state_fini: " 431 "munmap(0x%p, 0x%llx) of blueflame failed(%d)\n", 432 g_tavor_state[i].bf_pg_baseaddr, 433 g_tavor_state[i].uarpg_size, errno); 434 } 435 436 (void) close(g_tavor_state[i].hca_fd); 437 count++; 438 } 439 dapl_os_unlock(&g_tavor_state_lock); 440 441 dapl_os_lock_destroy(&g_tavor_uar_lock); 442 dapl_os_lock_destroy(&g_tavor_state_lock); 443 dapl_os_lock_destroy(&dapls_ib_dbp_lock); 444 445 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 446 "ib_state_fini: cleaned %d hcas\n", count); 447 } 448 449 /* 450 * dapls_ib_open_hca 451 * 452 * Open HCA 453 * 454 * Input: 455 * *hca_ptr pointer to hca device 456 * *ib_hca_handle_p pointer to provide HCA handle 457 * 458 * Output: 459 * none 460 * 461 * Return: 462 * DAT_SUCCESS 463 * DAT_INSUFFICIENT_RESOURCES 464 * 465 */ 466 DAT_RETURN 467 dapls_ib_open_hca( 468 IN DAPL_HCA *hca_ptr, 469 OUT ib_hca_handle_t *ib_hca_handle_p) 470 { 471 dapl_ia_create_t args; 472 DAT_RETURN dat_status; 473 struct dapls_ib_hca_handle *hca_p; 474 int fd; 475 #ifndef _LP64 476 int tmpfd; 477 #endif 478 int retval; 479 struct sockaddr *s; 480 struct sockaddr_in6 *v6addr; 481 struct sockaddr_in *v4addr; 482 dapl_ia_addr_t *sap; 483 484 dat_status = dapli_init_hca(hca_ptr); 485 if (dat_status != DAT_SUCCESS) { 486 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 487 "dapls_ib_open_hca: init_hca failed %d\n", dat_status); 488 return (dat_status); 489 } 490 491 fd = open(DEVDAPLT, O_RDONLY); 492 if (fd < 0) { 493 return (DAT_INSUFFICIENT_RESOURCES); 494 } 495 496 #ifndef _LP64 497 /* 498 * libc can't handle fd's greater than 255, in order to 499 * ensure that these values remain available make fd > 255. 500 * Note: not needed for LP64 501 */ 502 tmpfd = fcntl(fd, F_DUPFD, 256); 503 if (tmpfd < 0) { 504 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 505 "dapls_ib_open_hca: cannot F_DUPFD: %s\n", 506 strerror(errno)); 507 } else { 508 (void) close(fd); 509 fd = tmpfd; 510 } 511 #endif /* _LP64 */ 512 513 if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) { 514 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 515 "dapls_ib_open_hca: cannot F_SETFD: %s\n", strerror(errno)); 516 (void) close(fd); 517 return (DAT_INTERNAL_ERROR); 518 } 519 520 hca_p = (struct dapls_ib_hca_handle *)dapl_os_alloc( 521 sizeof (struct dapls_ib_hca_handle)); 522 if (hca_p == NULL) { 523 (void) close(fd); 524 return (DAT_INSUFFICIENT_RESOURCES); 525 } 526 527 args.ia_guid = hca_ptr->node_GUID; 528 args.ia_port = hca_ptr->port_num; 529 args.ia_pkey = hca_ptr->partition_key; 530 args.ia_version = DAPL_IF_VERSION; 531 (void) dapl_os_memzero((void *)args.ia_sadata, DAPL_ATS_NBYTES); 532 533 /* pass down local ip address to be stored in SA */ 534 s = (struct sockaddr *)&hca_ptr->hca_address; 535 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 536 sap = (dapl_ia_addr_t *)args.ia_sadata; 537 switch (s->sa_family) { 538 case AF_INET: 539 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 540 v4addr = (struct sockaddr_in *)s; 541 sap->iad_v4 = v4addr->sin_addr; 542 break; 543 case AF_INET6: 544 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 545 v6addr = (struct sockaddr_in6 *)s; 546 sap->iad_v6 = v6addr->sin6_addr; 547 break; 548 default: 549 break; /* fall through */ 550 } 551 552 retval = ioctl(fd, DAPL_IA_CREATE, &args); 553 if (retval != 0) { 554 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 555 "open_hca: ia_create failed, fd %d, " 556 "guid 0x%016llx, port %d, pkey 0x%x, version %d\n", 557 fd, args.ia_guid, args.ia_port, args.ia_pkey, 558 args.ia_version); 559 560 dapl_os_free(hca_p, sizeof (*hca_p)); 561 (void) close(fd); 562 return (dapls_convert_error(errno, retval)); 563 } 564 565 hca_p->ia_fd = fd; 566 hca_p->ia_rnum = args.ia_resnum; 567 hca_p->hca_fd = g_tavor_state[hca_ptr->tavor_idx].hca_fd; 568 hca_p->ia_uar = g_tavor_state[hca_ptr->tavor_idx].uarpg_baseaddr; 569 hca_p->ia_bf = g_tavor_state[hca_ptr->tavor_idx].bf_pg_baseaddr; 570 hca_p->ia_bf_toggle = &g_tavor_state[hca_ptr->tavor_idx].bf_toggle; 571 *ib_hca_handle_p = hca_p; 572 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 573 "open_hca: ia_created, hca_p 0x%p, fd %d, " 574 "rnum %d, guid 0x%016llx, port %d, pkey 0x%x\n", 575 hca_p, hca_p->ia_fd, hca_p->ia_rnum, hca_ptr->node_GUID, 576 hca_ptr->port_num, hca_ptr->partition_key); 577 578 return (DAT_SUCCESS); 579 } 580 581 /* 582 * dapls_ib_close_hca 583 * 584 * Open HCA 585 * 586 * Input: 587 * ib_hca_handle provide HCA handle 588 * 589 * Output: 590 * none 591 * 592 * Return: 593 * DAT_SUCCESS 594 * DAT_INSUFFICIENT_RESOURCES 595 * 596 */ 597 DAT_RETURN 598 dapls_ib_close_hca( 599 IN ib_hca_handle_t ib_hca_handle) 600 { 601 if (ib_hca_handle == NULL) { 602 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 603 "close_hca: ib_hca_handle == NULL\n"); 604 return (DAT_SUCCESS); 605 } 606 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 607 "close_hca: closing hca 0x%p, fd %d, rnum %d\n", 608 ib_hca_handle, ib_hca_handle->ia_fd, ib_hca_handle->ia_rnum); 609 610 (void) close(ib_hca_handle->ia_fd); 611 dapl_os_free((void *)ib_hca_handle, 612 sizeof (struct dapls_ib_hca_handle)); 613 return (DAT_SUCCESS); 614 } 615 616 #if defined(IBHOSTS_NAMING) 617 #define LINE_LEN 256 618 static int 619 dapli_process_fake_ibds(DAPL_HCA *hca_ptr, int hca_idx) 620 { 621 char line_buf[LINE_LEN]; 622 char host_buf[LINE_LEN]; 623 char localhost[LINE_LEN]; 624 ib_guid_t prefix; 625 ib_guid_t guid; 626 FILE *fp; 627 int count = 0; 628 DAPL_HCA *hca_ptr; 629 630 fp = fopen("/etc/dapl/ibhosts", "r"); 631 if (fp == NULL) { 632 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 633 "fake_ibds: ibhosts not found!\n"); 634 return (0); 635 } 636 if (gethostname(localhost, LINE_LEN) != 0) { 637 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 638 "fake_ibds: hostname not found!\n"); 639 return (0); 640 } 641 while (!feof(fp)) { 642 (void) fgets(line_buf, LINE_LEN, fp); 643 sscanf(line_buf, "%s %llx %llx", host_buf, &prefix, &guid); 644 (void) sprintf(line_buf, "%s-ib%d", localhost, count + 1); 645 if (strncmp(line_buf, host_buf, strlen(line_buf)) == 0) { 646 guid &= 0xfffffffffffffff0; 647 hca_ptr->tavor_idx = hca_idx; 648 hca_ptr->node_GUID = guid; 649 hca_ptr->port_num = count + 1; 650 hca_ptr->partition_key = 0x0000ffff; 651 count++; 652 } 653 if (count >= 2) break; 654 } 655 (void) fclose(fp); 656 return (count); 657 } 658 659 #endif /* IBHOSTS_NAMING */ 660