1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Emulex. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Source file containing the implementation of the Transmit 29 * Path 30 */ 31 32 #include <oce_impl.h> 33 34 static void oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed); 35 static int oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, 36 mblk_t *mp); 37 static int oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, 38 uint32_t pkt_len); 39 static void oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd); 40 static int oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq, 41 size_t size, int flags); 42 static oce_wq_bdesc_t *oce_wqb_alloc(struct oce_wq *wq); 43 static void oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd); 44 45 static void oce_wqmd_free(struct oce_wq *wq, oce_wqe_desc_t *wqed); 46 static void oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd); 47 static inline oce_wq_mdesc_t *oce_wqm_alloc(struct oce_wq *wq); 48 static int oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq); 49 static void oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd); 50 static void oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed); 51 static void oce_remove_vtag(mblk_t *mp); 52 static void oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag); 53 54 static ddi_dma_attr_t tx_map_dma_attr = { 55 DMA_ATTR_V0, /* version number */ 56 0x0000000000000000ull, /* low address */ 57 0xFFFFFFFFFFFFFFFFull, /* high address */ 58 0x0000000000010000ull, /* dma counter max */ 59 OCE_TXMAP_ALIGN, /* alignment */ 60 0x1, /* burst sizes */ 61 0x00000001, /* minimum transfer size */ 62 0x00000000FFFFFFFFull, /* maximum transfer size */ 63 0xFFFFFFFFFFFFFFFFull, /* maximum segment size */ 64 OCE_MAX_TXDMA_COOKIES, /* scatter/gather list length */ 65 0x00000001, /* granularity */ 66 0 /* DMA flags */ 67 }; 68 69 /* 70 * WQ map handle destructor 71 * 72 * wq - Pointer to WQ structure 73 * wqmd - pointer to WQE mapping handle descriptor 74 * 75 * return none 76 */ 77 78 static void 79 oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd) 80 { 81 _NOTE(ARGUNUSED(wq)); 82 /* Free the DMA handle */ 83 if (wqmd->dma_handle != NULL) 84 (void) ddi_dma_free_handle(&(wqmd->dma_handle)); 85 wqmd->dma_handle = NULL; 86 } /* oce_wqm_dtor */ 87 88 /* 89 * WQ map handles contructor 90 * 91 * wqmd - pointer to WQE mapping handle descriptor 92 * wq - Pointer to WQ structure 93 * 94 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 95 */ 96 static int 97 oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq) 98 { 99 struct oce_dev *dev; 100 int ret; 101 102 dev = wq->parent; 103 /* Allocate DMA handle */ 104 ret = ddi_dma_alloc_handle(dev->dip, &tx_map_dma_attr, 105 KM_SLEEP, NULL, &wqmd->dma_handle); 106 107 return (ret); 108 } /* oce_wqm_ctor */ 109 110 /* 111 * function to create WQ mapping handles cache 112 * 113 * wq - pointer to WQ structure 114 * 115 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 116 */ 117 int 118 oce_wqm_cache_create(struct oce_wq *wq) 119 { 120 struct oce_dev *dev = wq->parent; 121 int size; 122 int cnt; 123 int ret; 124 125 size = wq->cfg.nhdl * sizeof (oce_wq_mdesc_t); 126 wq->wq_mdesc_array = kmem_zalloc(size, KM_SLEEP); 127 128 /* Create the free buffer list */ 129 OCE_LIST_CREATE(&wq->wq_mdesc_list, DDI_INTR_PRI(dev->intr_pri)); 130 131 for (cnt = 0; cnt < wq->cfg.nhdl; cnt++) { 132 ret = oce_wqm_ctor(&wq->wq_mdesc_array[cnt], wq); 133 if (ret != DDI_SUCCESS) { 134 goto wqm_fail; 135 } 136 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list, 137 &wq->wq_mdesc_array[cnt]); 138 } 139 return (DDI_SUCCESS); 140 141 wqm_fail: 142 oce_wqm_cache_destroy(wq); 143 return (DDI_FAILURE); 144 } 145 146 /* 147 * function to destroy WQ mapping handles cache 148 * 149 * wq - pointer to WQ structure 150 * 151 * return none 152 */ 153 void 154 oce_wqm_cache_destroy(struct oce_wq *wq) 155 { 156 oce_wq_mdesc_t *wqmd; 157 158 while ((wqmd = OCE_LIST_REM_HEAD(&wq->wq_mdesc_list)) != NULL) { 159 oce_wqm_dtor(wq, wqmd); 160 } 161 162 kmem_free(wq->wq_mdesc_array, 163 wq->cfg.nhdl * sizeof (oce_wq_mdesc_t)); 164 165 OCE_LIST_DESTROY(&wq->wq_mdesc_list); 166 } 167 168 /* 169 * function to create WQ buffer cache 170 * 171 * wq - pointer to WQ structure 172 * buf_size - size of the buffer 173 * 174 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 175 */ 176 int 177 oce_wqb_cache_create(struct oce_wq *wq, size_t buf_size) 178 { 179 struct oce_dev *dev = wq->parent; 180 int size; 181 int cnt; 182 int ret; 183 184 size = wq->cfg.nbufs * sizeof (oce_wq_bdesc_t); 185 wq->wq_bdesc_array = kmem_zalloc(size, KM_SLEEP); 186 187 /* Create the free buffer list */ 188 OCE_LIST_CREATE(&wq->wq_buf_list, DDI_INTR_PRI(dev->intr_pri)); 189 190 for (cnt = 0; cnt < wq->cfg.nbufs; cnt++) { 191 ret = oce_wqb_ctor(&wq->wq_bdesc_array[cnt], 192 wq, buf_size, DDI_DMA_STREAMING); 193 if (ret != DDI_SUCCESS) { 194 goto wqb_fail; 195 } 196 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list, 197 &wq->wq_bdesc_array[cnt]); 198 } 199 return (DDI_SUCCESS); 200 201 wqb_fail: 202 oce_wqb_cache_destroy(wq); 203 return (DDI_FAILURE); 204 } 205 206 /* 207 * function to destroy WQ buffer cache 208 * 209 * wq - pointer to WQ structure 210 * 211 * return none 212 */ 213 void 214 oce_wqb_cache_destroy(struct oce_wq *wq) 215 { 216 oce_wq_bdesc_t *wqbd; 217 while ((wqbd = OCE_LIST_REM_HEAD(&wq->wq_buf_list)) != NULL) { 218 oce_wqb_dtor(wq, wqbd); 219 } 220 kmem_free(wq->wq_bdesc_array, 221 wq->cfg.nbufs * sizeof (oce_wq_bdesc_t)); 222 OCE_LIST_DESTROY(&wq->wq_buf_list); 223 } 224 225 /* 226 * WQ buffer constructor 227 * 228 * wqbd - pointer to WQ buffer descriptor 229 * wq - pointer to WQ structure 230 * size - size of the buffer 231 * flags - KM_SLEEP or KM_NOSLEEP 232 * 233 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 234 */ 235 static int 236 oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq, size_t size, int flags) 237 { 238 struct oce_dev *dev; 239 dev = wq->parent; 240 wqbd->wqb = oce_alloc_dma_buffer(dev, size, flags); 241 if (wqbd->wqb == NULL) { 242 return (DDI_FAILURE); 243 } 244 wqbd->frag_addr.dw.addr_lo = ADDR_LO(wqbd->wqb->addr); 245 wqbd->frag_addr.dw.addr_hi = ADDR_HI(wqbd->wqb->addr); 246 return (DDI_SUCCESS); 247 } 248 249 /* 250 * WQ buffer destructor 251 * 252 * wq - pointer to WQ structure 253 * wqbd - pointer to WQ buffer descriptor 254 * 255 * return none 256 */ 257 static void 258 oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd) 259 { 260 oce_free_dma_buffer(wq->parent, wqbd->wqb); 261 } 262 263 /* 264 * function to alloc WQE buffer descriptor 265 * 266 * wq - pointer to WQ structure 267 * 268 * return pointer to WQE buffer descriptor 269 */ 270 static inline oce_wq_bdesc_t * 271 oce_wqb_alloc(struct oce_wq *wq) 272 { 273 oce_wq_bdesc_t *wqbd; 274 wqbd = OCE_LIST_REM_HEAD(&wq->wq_buf_list); 275 return (wqbd); 276 } 277 278 /* 279 * function to free WQE buffer descriptor 280 * 281 * wq - pointer to WQ structure 282 * wqbd - pointer to WQ buffer descriptor 283 * 284 * return none 285 */ 286 static inline void 287 oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd) 288 { 289 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list, wqbd); 290 } /* oce_wqb_free */ 291 292 /* 293 * function to allocate WQE mapping descriptor 294 * 295 * wq - pointer to WQ structure 296 * 297 * return pointer to WQE mapping descriptor 298 */ 299 static inline oce_wq_mdesc_t * 300 oce_wqm_alloc(struct oce_wq *wq) 301 { 302 oce_wq_mdesc_t *wqmd; 303 wqmd = OCE_LIST_REM_HEAD(&wq->wq_mdesc_list); 304 return (wqmd); 305 306 } /* oce_wqm_alloc */ 307 308 /* 309 * function to insert WQE mapping descriptor to the list 310 * 311 * wq - pointer to WQ structure 312 * wqmd - Pointer to WQ mapping descriptor 313 * 314 * return none 315 */ 316 static inline void 317 oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd) 318 { 319 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list, wqmd); 320 } 321 322 /* 323 * function to free WQE mapping descriptor 324 * 325 * wq - pointer to WQ structure 326 * wqmd - Pointer to WQ mapping descriptor 327 * 328 * return none 329 */ 330 static void 331 oce_wqmd_free(struct oce_wq *wq, oce_wqe_desc_t *wqed) 332 { 333 int ndesc; 334 oce_wq_mdesc_t *wqmd; 335 336 if (wqed == NULL) { 337 return; 338 } 339 for (ndesc = 0; ndesc < wqed->nhdl; ndesc++) { 340 wqmd = wqed->hdesc[ndesc].hdl; 341 (void) ddi_dma_unbind_handle(wqmd->dma_handle); 342 oce_wqm_free(wq, wqmd); 343 } 344 } 345 346 /* 347 * WQED kmem_cache constructor 348 * 349 * buf - pointer to WQE descriptor 350 * 351 * return DDI_SUCCESS 352 */ 353 int 354 oce_wqe_desc_ctor(void *buf, void *arg, int kmflags) 355 { 356 oce_wqe_desc_t *wqed = (oce_wqe_desc_t *)buf; 357 358 _NOTE(ARGUNUSED(arg)); 359 _NOTE(ARGUNUSED(kmflags)); 360 361 bzero(wqed, sizeof (oce_wqe_desc_t)); 362 return (DDI_SUCCESS); 363 } 364 365 /* 366 * WQED kmem_cache destructor 367 * 368 * buf - pointer to WQE descriptor 369 * 370 * return none 371 */ 372 void 373 oce_wqe_desc_dtor(void *buf, void *arg) 374 { 375 _NOTE(ARGUNUSED(buf)); 376 _NOTE(ARGUNUSED(arg)); 377 } 378 379 /* 380 * function to choose a WQ given a mblk depending on priority, flowID etc. 381 * 382 * dev - software handle to device 383 * pkt - the mblk to send 384 * 385 * return pointer to the WQ selected 386 */ 387 struct oce_wq * 388 oce_get_wq(struct oce_dev *dev, mblk_t *pkt) 389 { 390 _NOTE(ARGUNUSED(pkt)); 391 /* for the time being hardcode */ 392 return (dev->wq[0]); 393 } /* oce_get_wq */ 394 395 /* 396 * function to populate the single WQE 397 * 398 * wq - pointer to wq 399 * wqed - pointer to WQ entry descriptor 400 * 401 * return none 402 */ 403 #pragma inline(oce_fill_ring_descs) 404 static void 405 oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed) 406 { 407 408 struct oce_nic_frag_wqe *wqe; 409 int i; 410 /* Copy the precreate WQE descs to the ring desc */ 411 for (i = 0; i < wqed->wqe_cnt; i++) { 412 wqe = RING_GET_PRODUCER_ITEM_VA(wq->ring, 413 struct oce_nic_frag_wqe); 414 415 bcopy(&wqed->frag[i], wqe, NIC_WQE_SIZE); 416 RING_PUT(wq->ring, 1); 417 } 418 } /* oce_fill_ring_descs */ 419 420 /* 421 * function to copy the packet to preallocated Tx buffer 422 * 423 * wq - pointer to WQ 424 * wqed - Pointer to WQE descriptor 425 * mp - Pointer to packet chain 426 * pktlen - Size of the packet 427 * 428 * return 0=>success, error code otherwise 429 */ 430 static int 431 oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, 432 uint32_t pkt_len) 433 { 434 oce_wq_bdesc_t *wqbd; 435 caddr_t buf_va; 436 struct oce_dev *dev = wq->parent; 437 438 wqbd = oce_wqb_alloc(wq); 439 if (wqbd == NULL) { 440 atomic_inc_32(&dev->tx_noxmtbuf); 441 oce_log(dev, CE_WARN, MOD_TX, "%s", 442 "wqb pool empty"); 443 return (ENOMEM); 444 } 445 446 /* create a fragment wqe for the packet */ 447 wqed->frag[1].u0.s.frag_pa_hi = wqbd->frag_addr.dw.addr_hi; 448 wqed->frag[1].u0.s.frag_pa_lo = wqbd->frag_addr.dw.addr_lo; 449 buf_va = DBUF_VA(wqbd->wqb); 450 451 /* copy pkt into buffer */ 452 for (; mp != NULL; mp = mp->b_cont) { 453 bcopy(mp->b_rptr, buf_va, MBLKL(mp)); 454 buf_va += MBLKL(mp); 455 } 456 457 wqed->frag[1].u0.s.frag_len = pkt_len; 458 wqed->hdesc[0].hdl = (void *)(wqbd); 459 460 (void) ddi_dma_sync(DBUF_DHDL(wqbd->wqb), 0, pkt_len, 461 DDI_DMA_SYNC_FORDEV); 462 463 if (oce_fm_check_dma_handle(dev, DBUF_DHDL(wqbd->wqb))) { 464 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED); 465 /* Free the buffer */ 466 oce_wqb_free(wq, wqbd); 467 return (EIO); 468 } 469 wqed->frag_cnt = 2; 470 wqed->nhdl = 1; 471 wqed->type = COPY_WQE; 472 return (0); 473 } /* oce_bcopy_wqe */ 474 475 /* 476 * function to copy the packet or dma map on the fly depending on size 477 * 478 * wq - pointer to WQ 479 * wqed - Pointer to WQE descriptor 480 * mp - Pointer to packet chain 481 * 482 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 483 */ 484 static int 485 oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp) 486 { 487 ddi_dma_cookie_t cookie; 488 oce_wq_mdesc_t *wqmd; 489 int32_t nfrag = 1; 490 uint32_t ncookies; 491 int ret; 492 uint32_t len; 493 struct oce_dev *dev = wq->parent; 494 495 wqed->nhdl = 0; 496 wqed->mp = mp; 497 498 for (; mp != NULL; mp = mp->b_cont) { 499 len = MBLKL(mp); 500 if (len == 0) { 501 oce_log(dev, CE_NOTE, MOD_TX, "%s", 502 "Zero len MBLK "); 503 continue; 504 } 505 506 wqmd = oce_wqm_alloc(wq); 507 if (wqmd == NULL) { 508 oce_log(dev, CE_WARN, MOD_TX, "%s", 509 "wqm pool empty"); 510 ret = ENOMEM; 511 goto map_fail; 512 } 513 514 ret = ddi_dma_addr_bind_handle(wqmd->dma_handle, 515 (struct as *)0, (caddr_t)mp->b_rptr, 516 len, DDI_DMA_WRITE | DDI_DMA_STREAMING, 517 DDI_DMA_DONTWAIT, NULL, &cookie, &ncookies); 518 if (ret != DDI_DMA_MAPPED) { 519 oce_log(dev, CE_WARN, MOD_TX, "%s", 520 "Failed to Map SGL"); 521 /* free the last one */ 522 oce_wqm_free(wq, wqmd); 523 goto map_fail; 524 } 525 526 do { 527 wqed->frag[nfrag].u0.s.frag_pa_hi = 528 ADDR_HI(cookie.dmac_laddress); 529 wqed->frag[nfrag].u0.s.frag_pa_lo = 530 ADDR_LO(cookie.dmac_laddress); 531 wqed->frag[nfrag].u0.s.frag_len = 532 (uint32_t)cookie.dmac_size; 533 nfrag++; 534 if (--ncookies > 0) 535 ddi_dma_nextcookie(wqmd->dma_handle, 536 &cookie); 537 else break; 538 } while (ncookies > 0); 539 540 wqed->hdesc[wqed->nhdl].hdl = (void *)wqmd; 541 wqed->nhdl++; 542 } 543 wqed->frag_cnt = nfrag; 544 wqed->type = MAPPED_WQE; 545 return (0); 546 547 map_fail: 548 wqed->mp = NULL; 549 oce_wqmd_free(wq, wqed); 550 return (ret); 551 } /* oce_map_wqe */ 552 553 /* 554 * function to drain a TxCQ and process its CQEs 555 * 556 * dev - software handle to the device 557 * cq - pointer to the cq to drain 558 * 559 * return the number of CQEs processed 560 */ 561 uint16_t 562 oce_drain_wq_cq(void *arg) 563 { 564 struct oce_nic_tx_cqe *cqe; 565 uint16_t num_cqe = 0; 566 struct oce_dev *dev; 567 struct oce_wq *wq; 568 struct oce_cq *cq; 569 oce_wqe_desc_t *wqed; 570 int wqe_freed = 0; 571 boolean_t is_update = B_FALSE; 572 573 wq = (struct oce_wq *)arg; 574 cq = wq->cq; 575 dev = wq->parent; 576 577 /* do while we do not reach a cqe that is not valid */ 578 mutex_enter(&cq->lock); 579 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 580 while (WQ_CQE_VALID(cqe)) { 581 582 DW_SWAP(u32ptr(cqe), sizeof (struct oce_nic_tx_cqe)); 583 584 /* update stats */ 585 if (cqe->u0.s.status != 0) { 586 atomic_inc_32(&dev->tx_errors); 587 } 588 589 /* complete the WQEs */ 590 wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list); 591 592 wqe_freed = wqed->wqe_cnt; 593 oce_free_wqed(wq, wqed); 594 RING_GET(wq->ring, wqe_freed); 595 atomic_add_32(&wq->wq_free, wqe_freed); 596 /* clear the valid bit and progress cqe */ 597 WQ_CQE_INVALIDATE(cqe); 598 RING_GET(cq->ring, 1); 599 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, 600 struct oce_nic_tx_cqe); 601 num_cqe++; 602 } /* for all valid CQE */ 603 604 if (wq->resched && num_cqe) { 605 wq->resched = B_FALSE; 606 is_update = B_TRUE; 607 } 608 609 mutex_exit(&cq->lock); 610 611 oce_arm_cq(dev, cq->cq_id, num_cqe, B_TRUE); 612 613 /* check if we need to restart Tx */ 614 mutex_enter(&wq->lock); 615 if (wq->resched && num_cqe) { 616 wq->resched = B_FALSE; 617 is_update = B_TRUE; 618 } 619 mutex_exit(&wq->lock); 620 621 if (is_update) 622 mac_tx_update(dev->mac_handle); 623 624 return (num_cqe); 625 } /* oce_process_wq_cqe */ 626 627 /* 628 * function to insert vtag to packet 629 * 630 * mp - mblk pointer 631 * vlan_tag - tag to be inserted 632 * 633 * return none 634 */ 635 static inline void 636 oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag) 637 { 638 struct ether_vlan_header *evh; 639 (void) memmove(mp->b_rptr - VLAN_TAGSZ, 640 mp->b_rptr, 2 * ETHERADDRL); 641 mp->b_rptr -= VLAN_TAGSZ; 642 evh = (struct ether_vlan_header *)(void *)mp->b_rptr; 643 evh->ether_tpid = htons(VLAN_TPID); 644 evh->ether_tci = htons(vlan_tag); 645 } 646 647 /* 648 * function to strip vtag from packet 649 * 650 * mp - mblk pointer 651 * 652 * return none 653 */ 654 655 static inline void 656 oce_remove_vtag(mblk_t *mp) 657 { 658 (void) memmove(mp->b_rptr + VLAN_TAGSZ, mp->b_rptr, 659 ETHERADDRL * 2); 660 mp->b_rptr += VLAN_TAGSZ; 661 } 662 663 /* 664 * function to xmit Single packet over the wire 665 * 666 * wq - pointer to WQ 667 * mp - Pointer to packet chain 668 * 669 * return pointer to the packet 670 */ 671 mblk_t * 672 oce_send_packet(struct oce_wq *wq, mblk_t *mp) 673 { 674 675 struct oce_nic_hdr_wqe *wqeh; 676 struct oce_dev *dev; 677 struct ether_header *eh; 678 struct ether_vlan_header *evh; 679 int32_t num_wqes; 680 uint16_t etype; 681 uint32_t ip_offset; 682 uint32_t csum_flags; 683 boolean_t use_copy = B_FALSE; 684 boolean_t tagged = B_FALSE; 685 uint16_t vlan_tag; 686 uint32_t reg_value = 0; 687 oce_wqe_desc_t *wqed = NULL; 688 mblk_t *nmp = NULL; 689 mblk_t *tmp = NULL; 690 uint32_t pkt_len = 0; 691 int num_mblks = 0; 692 int ret = 0; 693 694 /* retrieve the adap priv struct ptr */ 695 dev = wq->parent; 696 697 /* check if we should copy */ 698 for (tmp = mp; tmp != NULL; tmp = tmp->b_cont) { 699 pkt_len += MBLKL(tmp); 700 num_mblks++; 701 } 702 /* get the offload flags */ 703 hcksum_retrieve(mp, NULL, NULL, NULL, NULL, NULL, 704 NULL, &csum_flags); 705 706 /* Limit should be always less than Tx Buffer Size */ 707 if (pkt_len < dev->bcopy_limit) { 708 use_copy = B_TRUE; 709 } else { 710 /* restrict the mapped segment to wat we support */ 711 if (num_mblks > OCE_MAX_TX_HDL) { 712 nmp = msgpullup(mp, -1); 713 if (nmp == NULL) { 714 atomic_inc_32(&wq->pkt_drops); 715 freemsg(mp); 716 return (NULL); 717 } 718 /* Reset it to new collapsed mp */ 719 freemsg(mp); 720 mp = nmp; 721 } 722 } 723 724 /* Get the packet descriptor for Tx */ 725 wqed = kmem_cache_alloc(wq->wqed_cache, KM_NOSLEEP); 726 if (wqed == NULL) { 727 atomic_inc_32(&wq->pkt_drops); 728 freemsg(mp); 729 return (NULL); 730 } 731 eh = (struct ether_header *)(void *)mp->b_rptr; 732 if (ntohs(eh->ether_type) == VLAN_TPID) { 733 evh = (struct ether_vlan_header *)(void *)mp->b_rptr; 734 tagged = B_TRUE; 735 etype = ntohs(evh->ether_type); 736 ip_offset = sizeof (struct ether_vlan_header); 737 pkt_len -= VLAN_TAGSZ; 738 vlan_tag = ntohs(evh->ether_tci); 739 oce_remove_vtag(mp); 740 } else { 741 etype = ntohs(eh->ether_type); 742 ip_offset = sizeof (struct ether_header); 743 } 744 bzero(wqed, sizeof (oce_wqe_desc_t)); 745 746 /* Save the WQ pointer */ 747 wqed->wq = wq; 748 if (use_copy == B_TRUE) { 749 ret = oce_bcopy_wqe(wq, wqed, mp, pkt_len); 750 } else { 751 ret = oce_map_wqe(wq, wqed, mp); 752 } 753 754 /* 755 * Any failure other than insufficient Q entries 756 * drop the packet 757 */ 758 if (ret != 0) { 759 kmem_cache_free(wq->wqed_cache, wqed); 760 /* drop the packet */ 761 atomic_inc_32(&wq->pkt_drops); 762 freemsg(mp); 763 return (NULL); 764 } 765 766 /* increment pending wqed to be scheduled */ 767 wqeh = (struct oce_nic_hdr_wqe *)&wqed->frag[0]; 768 769 /* fill rest of wqe header fields based on packet */ 770 if (DB_CKSUMFLAGS(mp) & HW_LSO) { 771 wqeh->u0.s.lso = B_TRUE; 772 wqeh->u0.s.lso_mss = DB_LSOMSS(mp); 773 } 774 775 if (csum_flags & HCK_FULLCKSUM) { 776 uint8_t *proto; 777 if (etype == ETHERTYPE_IP) { 778 proto = (uint8_t *)(void *) 779 (mp->b_rptr + ip_offset); 780 if (proto[9] == 6) 781 /* IPPROTO_TCP */ 782 wqeh->u0.s.tcpcs = B_TRUE; 783 else if (proto[9] == 17) 784 /* IPPROTO_UDP */ 785 wqeh->u0.s.udpcs = B_TRUE; 786 } 787 } 788 789 if (csum_flags & HCK_IPV4_HDRCKSUM) 790 wqeh->u0.s.ipcs = B_TRUE; 791 if (tagged) { 792 wqeh->u0.s.vlan = B_TRUE; 793 wqeh->u0.s.vlan_tag = vlan_tag; 794 } 795 796 wqeh->u0.s.complete = B_TRUE; 797 wqeh->u0.s.event = B_TRUE; 798 wqeh->u0.s.crc = B_TRUE; 799 wqeh->u0.s.total_length = pkt_len; 800 801 num_wqes = wqed->frag_cnt; 802 803 /* h/w expects even no. of WQEs */ 804 if (num_wqes & 0x1) 805 num_wqes++; 806 wqed->wqe_cnt = (uint16_t)num_wqes; 807 wqeh->u0.s.num_wqe = num_wqes; 808 DW_SWAP(u32ptr(&wqed->frag[0]), (wqed->wqe_cnt * NIC_WQE_SIZE)); 809 810 mutex_enter(&wq->lock); 811 if (num_wqes > wq->wq_free) { 812 atomic_inc_32(&wq->tx_deferd); 813 wq->resched = B_TRUE; 814 mutex_exit(&wq->lock); 815 goto wqe_fail; 816 } 817 atomic_add_32(&wq->wq_free, -num_wqes); 818 wqed->wq_start_idx = wq->ring->pidx; 819 820 /* fill the wq for adapter */ 821 oce_fill_ring_descs(wq, wqed); 822 823 /* Add the packet desc to list to be retrieved during cmpl */ 824 OCE_LIST_INSERT_TAIL(&wq->wqe_desc_list, wqed); 825 826 /* ring tx doorbell */ 827 reg_value = (num_wqes << 16) | wq->wq_id; 828 /* Ring the door bell */ 829 OCE_DB_WRITE32(dev, PD_TXULP_DB, reg_value); 830 mutex_exit(&wq->lock); 831 832 /* free mp if copied or packet chain collapsed */ 833 if (use_copy == B_TRUE) { 834 freemsg(mp); 835 } 836 return (NULL); 837 838 wqe_fail: 839 840 if (tagged) { 841 oce_insert_vtag(mp, vlan_tag); 842 } 843 844 /* set it to null in case map_wqe has set it */ 845 wqed->mp = NULL; 846 oce_free_wqed(wq, wqed); 847 return (mp); 848 } /* oce_send_packet */ 849 850 /* 851 * function to free the WQE descriptor 852 * 853 * wq - pointer to WQ 854 * wqed - Pointer to WQE descriptor 855 * 856 * return none 857 */ 858 #pragma inline(oce_free_wqed) 859 static void 860 oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed) 861 { 862 if (wqed == NULL) { 863 return; 864 } 865 866 if (wqed->type == COPY_WQE) { 867 oce_wqb_free(wq, wqed->hdesc[0].hdl); 868 } else if (wqed->type == MAPPED_WQE) { 869 oce_wqmd_free(wq, wqed); 870 } else ASSERT(0); 871 872 if (wqed->mp) 873 freemsg(wqed->mp); 874 kmem_cache_free(wq->wqed_cache, wqed); 875 } /* oce_free_wqed */ 876 877 /* 878 * function to start the WQ 879 * 880 * wq - pointer to WQ 881 * 882 * return DDI_SUCCESS 883 */ 884 885 int 886 oce_start_wq(struct oce_wq *wq) 887 { 888 oce_arm_cq(wq->parent, wq->cq->cq_id, 0, B_TRUE); 889 return (DDI_SUCCESS); 890 } /* oce_start_wq */ 891 892 /* 893 * function to stop the WQ 894 * 895 * wq - pointer to WQ 896 * 897 * return none 898 */ 899 void 900 oce_stop_wq(struct oce_wq *wq) 901 { 902 oce_wqe_desc_t *wqed; 903 904 /* Max time for already posted TX to complete */ 905 drv_usecwait(150 * 1000); /* 150 mSec */ 906 907 /* Wait for already posted Tx to complete */ 908 while ((OCE_LIST_EMPTY(&wq->wqe_desc_list) == B_FALSE) || 909 (OCE_LIST_SIZE(&wq->wq_buf_list) != wq->cfg.nbufs) || 910 (OCE_LIST_SIZE(&wq->wq_mdesc_list) != wq->cfg.nhdl)) { 911 (void) oce_drain_wq_cq(wq); 912 } 913 914 /* Free the remaining descriptors */ 915 while ((wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list)) != NULL) { 916 atomic_add_32(&wq->wq_free, wqed->wqe_cnt); 917 oce_free_wqed(wq, wqed); 918 } 919 oce_drain_eq(wq->cq->eq); 920 } /* oce_stop_wq */ 921 922 /* 923 * function to set the tx mapping handle fma attr 924 * 925 * fm_caps - capability flags 926 * 927 * return none 928 */ 929 930 void 931 oce_set_tx_map_dma_fma_flags(int fm_caps) 932 { 933 if (fm_caps == DDI_FM_NOT_CAPABLE) { 934 return; 935 } 936 937 if (DDI_FM_DMA_ERR_CAP(fm_caps)) { 938 tx_map_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 939 } else { 940 tx_map_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 941 } 942 } /* oce_set_tx_map_dma_fma_flags */ 943