1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 /* 27 * sol_uverbs_event.c 28 * 29 * OFED User Verbs Kernel Async Event funtions 30 * 31 */ 32 #include <sys/file.h> 33 #include <sys/fcntl.h> 34 #include <sys/vfs.h> 35 #include <sys/errno.h> 36 #include <sys/cred.h> 37 #include <sys/uio.h> 38 #include <sys/semaphore.h> 39 #include <sys/ddi.h> 40 #include <sys/sunddi.h> 41 42 #include <sys/ib/ibtl/ibvti.h> 43 #include <sys/ib/clients/of/ofa_solaris.h> 44 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h> 45 #include <sys/ib/clients/of/ofed_kernel.h> 46 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h> 47 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_event.h> 48 49 extern char *sol_uverbs_dbg_str; 50 51 static void 52 uverbs_async_event_common(uverbs_uctxt_uobj_t *, uint64_t, uint32_t, 53 llist_head_t *, uint32_t *); 54 55 /* 56 * Function: 57 * sol_uverbs_event_file_close 58 * Input: 59 * ufile - Pointer to the event ufile 60 * 61 * Output: 62 * None 63 * Returns: 64 * Zero on success, else error code. 65 * Description: 66 * Called when all kernel references to the event file have been 67 * removed and the kernel (asynchronous) or user library (completion) 68 * have closed the file. 69 */ 70 void 71 sol_uverbs_event_file_close(uverbs_ufile_uobj_t *ufile) 72 { 73 if (!ufile) { 74 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 75 "UFILE CLOSE: Ufile NULL\n"); 76 return; 77 } 78 79 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 80 "UFILE CLOSE: Is async? %s", 81 ufile->is_async ? "yes" : "no"); 82 83 /* 84 * Remove the user file from the user object table and 85 * releases appropriate references. The object resources 86 * are freed when it is no longer referenced. 87 * 88 * If sol_ofs_uobj_remove() returns NULL then the obj was already 89 * removed. 90 */ 91 rw_enter(&(ufile->uobj.uo_lock), RW_WRITER); 92 if (sol_ofs_uobj_remove(&uverbs_ufile_uo_tbl, &ufile->uobj)) { 93 rw_exit(&(ufile->uobj.uo_lock)); 94 sol_ofs_uobj_deref(&ufile->uobj, uverbs_release_event_file); 95 } else { 96 rw_exit(&(ufile->uobj.uo_lock)); 97 } 98 } 99 100 /* 101 * Function: 102 * sol_uverbs_event_file_read 103 * Input: 104 * ufile - The user file pointer of the event channel. 105 * uiop - The user I/O pointer in which to place the event. 106 * cred - Pointer to the callers credentials. 107 * Output: 108 * uiop - Upon success the caller's buffer has been updated with 109 * the event details. 110 * Returns: 111 * Zero on success, else error code. 112 * EAGAIN - No event available and caller is non- 113 * blocking. 114 * ERESTART- A signal was received. 115 * EINVAL - Caller parameter/user buffer invalid. 116 * EFAULT - Failure copying data to user. 117 * Description: 118 * Perfrom a blocking read to retrieve an event (asynchronous or 119 * completion) for the user file specified. If an event is available it 120 * is immediately returned. If an event is not available, then the 121 * caller will block unless the open flags indicate it is a non- 122 * blocking open. If the caller does block it does so interruptable 123 * and therefore will return upon an event or receipt of a signal. 124 */ 125 /* ARGSUSED */ 126 int 127 sol_uverbs_event_file_read(uverbs_ufile_uobj_t *ufile, struct uio *uiop, 128 cred_t *cred) 129 { 130 int rc = 0; 131 uverbs_event_t *evt; 132 llist_head_t *entry; 133 int eventsz; 134 int ioflag; 135 136 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read(%p), " 137 "ufile = %p, is_async =%d, uio_resid=%d", 138 ufile, ufile->is_async, uiop->uio_resid); 139 140 ioflag = uiop->uio_fmode & (FNONBLOCK | FNDELAY); 141 142 mutex_enter(&ufile->lock); 143 144 /* 145 * If Event list not empty and CQ event notification is disabled 146 * by sol_ucma, do not return events. Either return EAGAIN (if 147 * flag is O_NONBLOCK, or wait using cv_wait_sig(). 148 */ 149 if (ufile->ufile_notify_enabled == SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE && 150 llist_empty(&ufile->event_list) != 0) { 151 if (ioflag) { 152 mutex_exit(&ufile->lock); 153 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 154 "event_file_read - notify disabled, no block"); 155 return (EAGAIN); 156 } 157 158 if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) { 159 mutex_exit(&ufile->lock); 160 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 161 "event_file_read - sig_wakeup"); 162 return (ERESTART); 163 } 164 } 165 166 while (llist_empty(&ufile->event_list)) { 167 if (ioflag) { 168 mutex_exit(&ufile->lock); 169 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 170 "event_file_read - no events, no block"); 171 return (EAGAIN); 172 } 173 174 if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) { 175 mutex_exit(&ufile->lock); 176 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 177 "event_file_read - sig_wakeup"); 178 return (ERESTART); 179 } 180 } 181 182 entry = ufile->event_list.nxt; 183 evt = entry->ptr; 184 185 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read: " 186 "Event entry found: entry:%p, event:%p, evt_list %p", 187 entry, evt, &evt->ev_list); 188 189 if (ufile->is_async) { 190 eventsz = sizeof (struct ib_uverbs_async_event_desc); 191 } else { 192 eventsz = sizeof (struct ib_uverbs_comp_event_desc); 193 } 194 195 if (eventsz > uiop->uio_resid) { 196 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 197 "event_file_read - Event too big"); 198 rc = EINVAL; 199 evt = NULL; 200 } else { 201 llist_del(ufile->event_list.nxt); 202 if (evt->ev_counter) { 203 ++(*evt->ev_counter); 204 llist_del(&evt->ev_obj_list); 205 } 206 } 207 208 mutex_exit(&ufile->lock); 209 210 if (evt && (uiomove(evt, eventsz, UIO_READ, uiop) != 0)) { 211 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 212 "EVENT FILE READ: Error writing ev"); 213 rc = EFAULT; 214 } 215 216 if (evt) { 217 kmem_free(evt, sizeof (*evt)); 218 } 219 220 return (rc); 221 } 222 223 /* 224 * Function: 225 * sol_uverbs_event_file_poll 226 * Input: 227 * ufile - user file for desired completion channel event file 228 * events - The events that may occur. 229 * anyyet - A flag that is non-zero if any files in the set 230 * of descriptors has an event waiting. 231 * ct - Pointer to the callers context. 232 * Output: 233 * reventssp - A pointer updated to return a bitmask of events specified. 234 * phpp - A pointer to a pollhead pointer, updated to reflect the 235 * the event file's pollhead used for synchronization. 236 * Returns: 237 * Zero on success, else error code. 238 * EINVAL - Vnode does not point to valid event file. 239 * Description: 240 * Support for event channel polling interface, allows use of completion 241 * channel in asynchronous type environment. If events may be read 242 * without blocking indicate a POLLIN | POLLRDNORM event; otherwise if 243 * no other descriptors in the set have data waiting, set the pollhead 244 * pointer to our the associated completion event file's pollhead. 245 */ 246 int 247 sol_uverbs_event_file_poll(uverbs_ufile_uobj_t *ufile, short events, 248 int anyyet, short *reventsp, pollhead_t **phpp) 249 { 250 short revent = 0; 251 252 #ifdef DEBUG 253 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll(%p, %x)", 254 ufile, events); 255 #endif 256 257 if (!ufile) { 258 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "event_file_poll ", 259 "ufile %p", ufile); 260 return (EINVAL); 261 } 262 263 #ifdef DEBUG 264 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll " 265 "ufile = %p, is_async =%d", ufile, ufile->is_async); 266 #endif 267 268 mutex_enter(&ufile->lock); 269 270 /* 271 * If poll request and event is ready. 272 */ 273 if ((events & (POLLIN | POLLRDNORM)) && 274 !llist_empty(&ufile->event_list)) { 275 276 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll " 277 "Event entry available"); 278 279 revent |= POLLIN | POLLRDNORM; 280 } 281 282 /* 283 * If we didn't get an event 284 */ 285 if (revent == 0 && !anyyet) { 286 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll " 287 "Event entry NOT available"); 288 289 *phpp = &ufile->poll_head; 290 } 291 292 mutex_exit(&ufile->lock); 293 *reventsp = revent; 294 return (0); 295 } 296 297 /* 298 * Function: 299 * uverbs_alloc_event_file 300 * Input: 301 * uctxt - The Solaris User Verbs user context associated with the 302 * event channel. 303 * is_async - Indicates the file is for asynchronous events if non-zero; 304 * other wise it is for completion events. 305 * Output: 306 * None. 307 * Returns: 308 * New user verb event file object or NULL on error. 309 * Description: 310 * Allocate an asynchronous or completion event file 311 */ 312 uverbs_ufile_uobj_t * 313 uverbs_alloc_event_file(uverbs_uctxt_uobj_t *uctxt, int is_async) 314 { 315 uverbs_ufile_uobj_t *ufile; 316 317 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "alloc_event_file(%p, %x)", 318 uctxt, is_async); 319 320 ufile = kmem_zalloc(sizeof (*ufile), KM_NOSLEEP); 321 if (!ufile) { 322 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 323 "alloc_event_file: mem alloc fail"); 324 return (NULL); 325 } 326 ufile->ufile_notify_enabled = SOL_UVERBS2UCMA_CQ_NOTIFY_ENABLE; 327 sol_ofs_uobj_init(&ufile->uobj, 0, SOL_UVERBS_UFILE_UOBJ_TYPE); 328 rw_enter(&ufile->uobj.uo_lock, RW_WRITER); 329 330 if (sol_ofs_uobj_add(&uverbs_ufile_uo_tbl, &ufile->uobj) != 0) { 331 /* 332 * The initialization routine set's the initial reference, 333 * we dereference the object here to clean it up. 334 */ 335 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 336 "ALLOC UFILE: Object add failed"); 337 rw_exit(&ufile->uobj.uo_lock); 338 ufile->uobj.uo_uobj_sz = sizeof (uverbs_ufile_uobj_t); 339 sol_ofs_uobj_deref(&ufile->uobj, sol_ofs_uobj_free); 340 return (NULL); 341 } 342 343 ufile->is_async = is_async ? 1 : 0; 344 llist_head_init(&ufile->event_list, NULL); 345 346 mutex_init(&ufile->lock, NULL, MUTEX_DRIVER, NULL); 347 cv_init(&ufile->poll_wait, NULL, CV_DRIVER, NULL); 348 349 ufile->uctxt = uctxt; 350 ufile->uobj.uo_live = 1; 351 rw_exit(&ufile->uobj.uo_lock); 352 return (ufile); 353 } 354 355 /* 356 * Function: 357 * uverbs_release_event_file 358 * Input: 359 * ufile - Pointer to the ufile user object that is being freed. 360 * Output: 361 * None. 362 * Returns: 363 * None. 364 * Description: 365 * Release/destroy event file resources before freeing memory. This 366 * routine should only be used if the event file is successfully 367 * created. 368 */ 369 void 370 uverbs_release_event_file(sol_ofs_uobj_t *uobj) 371 { 372 uverbs_ufile_uobj_t *ufile = (uverbs_ufile_uobj_t *)uobj; 373 uverbs_event_t *evt; 374 llist_head_t *entry; 375 llist_head_t *list_tmp; 376 377 if (!ufile) { 378 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 379 "UFILE RELEASE: Ufile NULL\n"); 380 return; 381 } 382 383 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 384 "UFILE RELEASE: Event file=%p, is async = %s", 385 ufile, ufile->is_async ? "yes" : "no"); 386 387 /* 388 * Release any events still queued to the event file. 389 */ 390 mutex_enter(&ufile->lock); 391 392 entry = ufile->event_list.nxt; 393 list_tmp = entry->nxt; 394 while (entry != &ufile->event_list) { 395 ASSERT(entry); 396 evt = (uverbs_event_t *)entry->ptr; 397 398 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 399 "UFILE RELEASE: Deleting event %p on event file %p", 400 evt, ufile); 401 402 llist_del(&evt->ev_list); 403 kmem_free(evt, sizeof (*evt)); 404 entry = list_tmp; 405 list_tmp = entry->nxt; 406 } 407 408 mutex_exit(&ufile->lock); 409 410 cv_destroy(&ufile->poll_wait); 411 mutex_destroy(&ufile->lock); 412 sol_ofs_uobj_free(uobj); 413 } 414 415 /* 416 * Function: 417 * uverbs_ibt_to_ofa_event_code 418 * Input: 419 * code - The OFA event code. 420 * Output: 421 * The OFED event code. 422 * Returns: 423 * Returns the OFA equivalent of an IBT Asynchronous Event code, -1 if 424 * a valid translation does not exist. 425 * Description: 426 * Map an IBT asynchronous event code to an OFED event code. 427 */ 428 enum ib_event_type 429 uverbs_ibt_to_ofa_event_code(ibt_async_code_t code) 430 { 431 enum ib_event_type ofa_code; 432 433 switch (code) { 434 case IBT_EVENT_PATH_MIGRATED: 435 ofa_code = IB_EVENT_PATH_MIG; 436 break; 437 438 case IBT_EVENT_SQD: 439 ofa_code = IB_EVENT_SQ_DRAINED; 440 break; 441 442 case IBT_EVENT_COM_EST: 443 ofa_code = IB_EVENT_COMM_EST; 444 break; 445 446 case IBT_ERROR_CATASTROPHIC_CHAN: 447 case IBT_ERROR_LOCAL_CATASTROPHIC: 448 ofa_code = IB_EVENT_QP_FATAL; 449 break; 450 451 case IBT_ERROR_INVALID_REQUEST_CHAN: 452 ofa_code = IB_EVENT_QP_REQ_ERR; 453 break; 454 455 case IBT_ERROR_ACCESS_VIOLATION_CHAN: 456 ofa_code = IB_EVENT_QP_ACCESS_ERR; 457 break; 458 459 case IBT_ERROR_PATH_MIGRATE_REQ: 460 ofa_code = IB_EVENT_PATH_MIG_ERR; 461 break; 462 463 case IBT_ERROR_CQ: 464 ofa_code = IB_EVENT_CQ_ERR; 465 break; 466 467 case IBT_EVENT_PORT_UP: 468 ofa_code = IB_EVENT_PORT_ACTIVE; 469 break; 470 471 case IBT_ERROR_PORT_DOWN: 472 ofa_code = IB_EVENT_PORT_ERR; 473 break; 474 475 case IBT_HCA_ATTACH_EVENT: 476 ofa_code = IB_EVENT_CLIENT_REREGISTER; 477 break; 478 479 case IBT_EVENT_LIMIT_REACHED_SRQ: 480 ofa_code = IB_EVENT_SRQ_LIMIT_REACHED; 481 break; 482 483 case IBT_ERROR_CATASTROPHIC_SRQ: 484 ofa_code = IB_EVENT_SRQ_ERR; 485 break; 486 487 case IBT_EVENT_EMPTY_CHAN: 488 ofa_code = IB_EVENT_QP_LAST_WQE_REACHED; 489 break; 490 491 /* 492 * No mapping exists. 493 */ 494 case IBT_ASYNC_OPAQUE1: 495 case IBT_ASYNC_OPAQUE2: 496 case IBT_ASYNC_OPAQUE3: 497 case IBT_ASYNC_OPAQUE4: 498 case IBT_HCA_DETACH_EVENT: 499 default: 500 ofa_code = -1; 501 } 502 503 return (ofa_code); 504 } 505 506 /* 507 * Function: 508 * uverbs_async_qp_event_handler 509 * Input: 510 * clnt_private - The IBT attach client handle. 511 * hca_hdl - The IBT hca handle associated with the notification. 512 * code - The OFED event identifier. 513 * event - The IBT event. 514 * Output: 515 * None 516 * Returns: 517 * None 518 * Description: 519 * Handle QP affiliated asynchronous event noficiation. 520 */ 521 /* ARGSUSED */ 522 void 523 uverbs_async_qp_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 524 enum ib_event_type code, ibt_async_event_t *event) 525 { 526 uverbs_uqp_uobj_t *uqp; 527 528 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 529 "async_qp_event_handler()"); 530 531 if (event->ev_chan_hdl == NULL) { 532 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 533 "async_qp_event_handler: event handle NULL"); 534 return; 535 } 536 uqp = ibt_get_qp_private(event->ev_chan_hdl); 537 ASSERT(uqp); 538 if (uqp->uqp_free_state == SOL_UVERBS2UCMA_FREE_PENDING) { 539 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 540 "async_qp_event_handler: User QP context has been freed"); 541 return; 542 } 543 if (uqp->qp != event->ev_chan_hdl) { 544 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 545 "async_qp_event_handler: QP handle mismatch"); 546 return; 547 } 548 549 uverbs_async_event_common(uqp->uctxt, uqp->uobj.uo_user_handle, 550 code, &uqp->async_list, &uqp->async_events_reported); 551 552 } 553 554 /* 555 * Function: 556 * uverbs_async_cq_event_handler 557 * Input: 558 * clnt_private - The IBT attach client handle. 559 * hca_hdl - The IBT hca handle associated with the notification. 560 * code - The OFED event identifier. 561 * event - The IBT event. 562 * Output: 563 * None 564 * Returns: 565 * None 566 * Description: 567 * Handle a CQ affiliated asynchronous event notification. 568 */ 569 /* ARGSUSED */ 570 void 571 uverbs_async_cq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 572 enum ib_event_type code, ibt_async_event_t *event) 573 { 574 uverbs_ucq_uobj_t *ucq; 575 576 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 577 "ASYNC CQ EVENT HANDLER:"); 578 579 if (event->ev_cq_hdl == NULL) { 580 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 581 "ASYNC CQ EVENT HANDLER: event handle is NULL"); 582 return; 583 } 584 585 ucq = ibt_get_cq_private(event->ev_cq_hdl); 586 if (ucq->cq != event->ev_cq_hdl) { 587 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 588 "ASYNC CQ EVENT HANDLER: CQ handle mismatch"); 589 return; 590 } 591 592 uverbs_async_event_common(ucq->uctxt, ucq->uobj.uo_user_handle, 593 code, &ucq->async_list, &ucq->async_events_reported); 594 } 595 596 /* 597 * Function: 598 * uverbs_async_srq_event_handler 599 * Input: 600 * clnt_private - The IBT attach client handle. 601 * hca_hdl - The IBT hca handle associated with the notification. 602 * code - The OFED event identifier. 603 * event - The IBT event. 604 * Output: 605 * None 606 * Returns: 607 * None 608 * Description: 609 * Handle a shared receive queue asynchronous event notification. 610 */ 611 /* ARGSUSED */ 612 void 613 uverbs_async_srq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 614 enum ib_event_type code, ibt_async_event_t *event) 615 { 616 uverbs_usrq_uobj_t *usrq; 617 618 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 619 "ASYNC SRQ EVENT HANDLER:"); 620 621 if (event->ev_srq_hdl == NULL) { 622 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 623 "ASYNC SRQ EVENT HANDLER: event handle is NULL"); 624 return; 625 } 626 627 usrq = ibt_get_srq_private(event->ev_srq_hdl); 628 if (usrq->srq != event->ev_srq_hdl) { 629 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 630 "ASYNC SRQ EVENT HANDLER: SRQ handle mismatch"); 631 return; 632 } 633 634 uverbs_async_event_common(usrq->uctxt, usrq->uobj.uo_user_handle, 635 code, &usrq->async_list, &usrq->async_events_reported); 636 } 637 638 /* 639 * Function: 640 * uverbs_async_unaff_event_handler 641 * Input: 642 * clnt_private - The IBT attach client handle. 643 * hca_hdl - The IBT hca handle associated with the notification. 644 * code - The OFED event identifier. 645 * event - The IBT event. 646 * Output: 647 * None 648 * Returns: 649 * None 650 * Description: 651 * Handle an unaffiliated asynchronous event notification. 652 */ 653 /* ARGSUSED */ 654 void 655 uverbs_async_unaff_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 656 enum ib_event_type code, ibt_async_event_t *event) 657 { 658 sol_ofs_uobj_table_t *uo_tbl = &uverbs_uctxt_uo_tbl; 659 sol_ofs_uobj_blk_t *blk; 660 uverbs_uctxt_uobj_t *uctxt; 661 int i, j; 662 663 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 664 "ASYNC UNAFF EVENT HANDLER:"); 665 666 /* 667 * Unaffiliated events are returned at the IBT client level. We must 668 * return the event to all user context allocated for the specific 669 * HCA device specified. 670 */ 671 rw_enter(&uo_tbl->uobj_tbl_lock, RW_READER); 672 673 for (i = 0; i < uo_tbl->uobj_tbl_used_blks; i++) { 674 blk = uo_tbl->uobj_tbl_uo_root[i]; 675 if (blk == NULL) { 676 continue; 677 } 678 for (j = 0; j < SOL_OFS_UO_BLKSZ; j++) { 679 uctxt = (uverbs_uctxt_uobj_t *)blk->ofs_uoblk_blks[j]; 680 if (uctxt == NULL) { 681 continue; 682 } 683 /* 684 * OK, check to see if this user context belongs 685 * to the idicated hca. 686 */ 687 if (uctxt->hca->hdl == hca_hdl && uctxt->async_evfile) { 688 uverbs_async_event_common(uctxt, 689 event->ev_port, code, NULL, NULL); 690 } 691 } 692 } 693 rw_exit(&uo_tbl->uobj_tbl_lock); 694 } 695 696 /* 697 * Function: 698 * uverbs_async_event_handler 699 * Input: 700 * clnt_private - The IBT attach client handle. 701 * hca_hdl - The IBT hca handle associated with the notification. 702 * code - The OFED event identifier. 703 * event - The IBT event. 704 * Output: 705 * None 706 * Returns: 707 * None 708 * Description: 709 * Main IBT asynchronous event handler registered at ibt_attach. 710 * Convert to OFA event type and forward to the appropriate 711 * asynchronous handler. 712 */ 713 void 714 uverbs_async_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 715 ibt_async_code_t code, ibt_async_event_t *event) 716 { 717 enum ib_event_type ofa_type; 718 sol_uverbs_ib_event_handler_t *handler; 719 llist_head_t *entry; 720 sol_uverbs_hca_t *hca; 721 722 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 723 "ASYNNC EVENT HANDLER: entry, code=%d", code); 724 725 ofa_type = uverbs_ibt_to_ofa_event_code(code); 726 727 if ((int)ofa_type < 0) { 728 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 729 "ASYNC EVENT HANDLER:Event %d did not map to OFA " 730 "Event", code); 731 return; 732 } 733 734 switch (ofa_type) { 735 case IB_EVENT_QP_FATAL: 736 case IB_EVENT_QP_REQ_ERR: 737 case IB_EVENT_QP_ACCESS_ERR: 738 case IB_EVENT_QP_LAST_WQE_REACHED: 739 case IB_EVENT_SQ_DRAINED: 740 case IB_EVENT_PATH_MIG: 741 /* 742 * These events are related with a QP 743 */ 744 uverbs_async_qp_event_handler(clnt_private, hca_hdl, 745 ofa_type, event); 746 break; 747 748 case IB_EVENT_CQ_ERR: 749 /* 750 * These events are related with a CQ 751 */ 752 uverbs_async_cq_event_handler(clnt_private, hca_hdl, 753 ofa_type, event); 754 break; 755 756 case IB_EVENT_SRQ_ERR: 757 case IB_EVENT_SRQ_LIMIT_REACHED: 758 /* 759 * These events are related with a SRQ 760 */ 761 uverbs_async_srq_event_handler(clnt_private, hca_hdl, 762 ofa_type, event); 763 break; 764 765 766 case IB_EVENT_PORT_ERR: 767 case IB_EVENT_PORT_ACTIVE: 768 case IB_EVENT_LID_CHANGE: 769 case IB_EVENT_PKEY_CHANGE: 770 case IB_EVENT_SM_CHANGE: 771 case IB_EVENT_CLIENT_REREGISTER: 772 case IB_EVENT_DEVICE_FATAL: 773 case IB_EVENT_PATH_MIG_ERR: 774 /* 775 * Unaffiliated asynchronous notifications. 776 */ 777 uverbs_async_unaff_event_handler(clnt_private, hca_hdl, 778 ofa_type, event); 779 break; 780 781 default: 782 break; 783 } 784 785 /* 786 * Give other kernel agents a notification. 787 */ 788 hca = sol_uverbs_ibt_hdl_to_hca(hca_hdl); 789 if (hca) { 790 mutex_enter(&hca->event_handler_lock); 791 list_for_each(entry, &hca->event_handler_list) { 792 handler = (sol_uverbs_ib_event_handler_t *)entry->ptr; 793 794 ASSERT(handler != NULL); 795 handler->handler(handler, hca_hdl, code, event); 796 } 797 mutex_exit(&hca->event_handler_lock); 798 } 799 } 800 801 /* 802 * Function: 803 * uverbs_async_event_common 804 * Input: 805 * uctxt - Pointer to the user context associated with the 806 * affiliated event. 807 * element - The users handle to the associated object. 808 * event - The event type. 809 * uobj_list - The list to enqueue the asynchronous event. 810 * counter - The counter to track the event delivery. 811 * Output: 812 * None 813 * Returns: 814 * None 815 * Description: 816 * Create an asyncronous event and enqueue it on the specified list. 817 * Then wake callers that may be blocked on the list. 818 */ 819 static void 820 uverbs_async_event_common(uverbs_uctxt_uobj_t *uctxt, uint64_t element, 821 uint32_t event, llist_head_t *obj_list, uint32_t *counter) 822 { 823 uverbs_ufile_uobj_t *ufile = uctxt->async_evfile; 824 uverbs_event_t *entry; 825 826 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common(%p, " 827 "%llx, %llx, %p, %p)", uctxt, element, event, obj_list, counter); 828 829 if (!ufile) { 830 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common " 831 "ufile %p", ufile); 832 return; 833 } 834 835 mutex_enter(&ufile->lock); 836 entry = kmem_zalloc(sizeof (*entry), KM_NOSLEEP); 837 if (!entry) { 838 mutex_exit(&ufile->lock); 839 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common " 840 "kmem_zalloc failed"); 841 return; 842 } 843 844 entry->ev_desc.async.element = element; 845 entry->ev_desc.async.event_type = event; 846 entry->ev_counter = counter; 847 848 llist_head_init(&entry->ev_list, entry); 849 llist_head_init(&entry->ev_obj_list, entry); 850 851 llist_add_tail(&entry->ev_list, &ufile->event_list); 852 853 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common " 854 "adding ASYNC entry-ev_list=%p, entry %p", 855 &entry->ev_list, entry); 856 857 if (obj_list) { 858 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common " 859 "adding ASYNC entry-ev_obj_list=%p, entry=%p", 860 &entry->ev_obj_list, entry); 861 llist_add_tail(&entry->ev_obj_list, obj_list); 862 } 863 864 mutex_exit(&ufile->lock); 865 cv_signal(&ufile->poll_wait); 866 pollwakeup(&ufile->poll_head, POLLIN | POLLRDNORM); 867 } 868 869 /* 870 * Function: 871 * uverbs_release_ucq_channel 872 * Input: 873 * uctxt - A pointer to the callers user context. 874 * ufile - A pointer to the event file associated with a CQ. 875 * ucq - A pointer to the user CQ object. 876 * Output: 877 * None 878 * Returns: 879 * None 880 * Description: 881 * Release any completion and asynchronous events that may 882 * be queued to the specified completion channel/UCQ but not 883 * yet reaped. 884 */ 885 void 886 uverbs_release_ucq_channel(uverbs_uctxt_uobj_t *uctxt, 887 uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t *ucq) 888 { 889 uverbs_event_t *evt; 890 llist_head_t *entry; 891 llist_head_t *list_tmp; 892 893 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 894 "RELEASE UCQ CHANNEL: uctxt=%p, ufile=%p, ucq=%p", 895 uctxt, ufile, ucq); 896 897 /* 898 * Release completion events that have been queued on the CQ completion 899 * eventlist. 900 */ 901 if (ufile) { 902 rw_enter(&ufile->uobj.uo_lock, RW_WRITER); 903 ufile->ufile_cq_cnt--; 904 if (ufile->ufile_cq_cnt) { 905 rw_exit(&ufile->uobj.uo_lock); 906 uverbs_release_ucq_uevents(ufile, ucq); 907 return; 908 } 909 rw_exit(&ufile->uobj.uo_lock); 910 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 911 "release_ucq_chan : comp_list %p, prv %p, nxt %p", 912 &ucq->comp_list, ucq->comp_list.prv, 913 ucq->comp_list.nxt); 914 mutex_enter(&ufile->lock); 915 916 entry = ucq->comp_list.nxt; 917 list_tmp = entry->nxt; 918 while (entry != &ucq->comp_list) { 919 ASSERT(entry); 920 evt = (uverbs_event_t *)entry->ptr; 921 922 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 923 "RELEASE UCQ CHANNEL:Deleting event " 924 "on CQ comp list: %p", evt); 925 llist_del(&evt->ev_list); 926 llist_del(&evt->ev_obj_list); 927 kmem_free(evt, sizeof (*evt)); 928 entry = list_tmp; 929 list_tmp = entry->nxt; 930 } 931 mutex_exit(&ufile->lock); 932 933 uverbs_release_ucq_uevents(ufile, ucq); 934 } 935 936 } 937 938 /* 939 * Function: 940 * uverbs_release_ucq_uevents 941 * Input: 942 * ufile - A pointer to the asynchronous event file associated with a QP. 943 * ucq - A pointer to the user CQ object. 944 * Output: 945 * None 946 * Returns: 947 * None 948 * Description: 949 * Free any user asynchronous events that have been queued for the 950 * user CQ object specified. 951 */ 952 void 953 uverbs_release_ucq_uevents(uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t *ucq) 954 { 955 uverbs_event_t *evt; 956 llist_head_t *entry; 957 llist_head_t *list_tmp; 958 959 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 960 "RELEASE UCQ ASYNC EVENTS: ufile=%p, ucq=%p", ufile, ucq); 961 962 if (ufile) { 963 mutex_enter(&ufile->lock); 964 965 entry = ucq->async_list.nxt; 966 list_tmp = entry->nxt; 967 while (entry != &ucq->async_list) { 968 ASSERT(entry); 969 evt = (uverbs_event_t *)entry->ptr; 970 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 971 "RELEASE UCQ EVENTS: Deleting event " 972 "on CQ async list: %p", evt); 973 llist_del(&evt->ev_list); 974 llist_del(&evt->ev_obj_list); 975 kmem_free(evt, sizeof (*evt)); 976 entry = list_tmp; 977 list_tmp = entry->nxt; 978 } 979 mutex_exit(&ufile->lock); 980 } 981 } 982 983 /* 984 * Function: 985 * uverbs_release_uqp_uevents 986 * Input: 987 * ufile - A pointer to the asynchronous event file associated with a QP. 988 * uqp - A pointer to the user QP object. 989 * Output: 990 * None 991 * Returns: 992 * None 993 * Description: 994 * Free any user asynchronous events that have been queued for the 995 * user QP object specified. 996 */ 997 void 998 uverbs_release_uqp_uevents(uverbs_ufile_uobj_t *ufile, uverbs_uqp_uobj_t *uqp) 999 { 1000 uverbs_event_t *evt; 1001 llist_head_t *entry; 1002 llist_head_t *list_tmp; 1003 1004 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 1005 "RELEASE UQP EVENTS: ufile=%p, uqp=%p", ufile, uqp); 1006 1007 if (ufile) { 1008 mutex_enter(&ufile->lock); 1009 entry = uqp->async_list.nxt; 1010 list_tmp = entry->nxt; 1011 while (entry != &uqp->async_list) { 1012 ASSERT(entry); 1013 evt = (uverbs_event_t *)entry->ptr; 1014 ASSERT(evt); 1015 1016 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 1017 "RELEASE UQP EVENTS: Deleting event " 1018 "ON qp async list: %p", evt); 1019 llist_del(&evt->ev_list); 1020 llist_del(&evt->ev_obj_list); 1021 kmem_free(evt, sizeof (*evt)); 1022 entry = list_tmp; 1023 list_tmp = entry->nxt; 1024 } 1025 mutex_exit(&ufile->lock); 1026 } 1027 } 1028 1029 /* 1030 * Function: 1031 * uverbs_release_usrq_uevents 1032 * Input: 1033 * ufile - A pointer to the asynchronous event file associated with a 1034 * SRQ. 1035 * uqp - A pointer to the user SRQ object. 1036 * Output: 1037 * None 1038 * Returns: 1039 * None 1040 * Description: 1041 * Free any user asynchronous events that have been queued for the 1042 * user SRQ object specified. 1043 */ 1044 void 1045 uverbs_release_usrq_uevents(uverbs_ufile_uobj_t *ufile, 1046 uverbs_usrq_uobj_t *usrq) 1047 { 1048 uverbs_event_t *evt; 1049 llist_head_t *entry; 1050 llist_head_t *list_tmp; 1051 1052 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 1053 "RELEASE USRQ EVENTS: ufile=%p, usrq=%p", ufile, usrq); 1054 1055 if (ufile) { 1056 mutex_enter(&ufile->lock); 1057 1058 entry = usrq->async_list.nxt; 1059 list_tmp = entry->nxt; 1060 while (entry != &usrq->async_list) { 1061 ASSERT(entry); 1062 evt = (uverbs_event_t *)entry->ptr; 1063 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 1064 "RELEASE SRQ EVENTS: Deleting event " 1065 "on SRQ async list: %p", evt); 1066 llist_del(&evt->ev_list); 1067 llist_del(&evt->ev_obj_list); 1068 kmem_free(evt, sizeof (*evt)); 1069 entry = list_tmp; 1070 list_tmp = entry->nxt; 1071 } 1072 mutex_exit(&ufile->lock); 1073 } 1074 } 1075