1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2019-2020 Marvell International Ltd. All rights reserved */ 3 4 #include <linux/bitfield.h> 5 #include <linux/dmapool.h> 6 #include <linux/etherdevice.h> 7 #include <linux/if_vlan.h> 8 #include <linux/of_address.h> 9 #include <linux/of_device.h> 10 #include <linux/of.h> 11 #include <linux/platform_device.h> 12 13 #include "prestera_dsa.h" 14 #include "prestera.h" 15 #include "prestera_hw.h" 16 #include "prestera_rxtx.h" 17 #include "prestera_devlink.h" 18 19 #define PRESTERA_SDMA_WAIT_MUL 10 20 21 struct prestera_sdma_desc { 22 __le32 word1; 23 __le32 word2; 24 __le32 buff; 25 __le32 next; 26 } __packed __aligned(16); 27 28 #define PRESTERA_SDMA_BUFF_SIZE_MAX 1544 29 30 #define PRESTERA_SDMA_RX_DESC_PKT_LEN(desc) \ 31 ((le32_to_cpu((desc)->word2) >> 16) & GENMASK(13, 0)) 32 33 #define PRESTERA_SDMA_RX_DESC_OWNER(desc) \ 34 ((le32_to_cpu((desc)->word1) & BIT(31)) >> 31) 35 36 #define PRESTERA_SDMA_RX_DESC_IS_RCVD(desc) \ 37 (PRESTERA_SDMA_RX_DESC_OWNER(desc) == PRESTERA_SDMA_RX_DESC_CPU_OWN) 38 39 #define PRESTERA_SDMA_RX_DESC_CPU_OWN 0 40 #define PRESTERA_SDMA_RX_DESC_DMA_OWN 1 41 42 #define PRESTERA_SDMA_RX_QUEUE_NUM 8 43 44 #define PRESTERA_SDMA_RX_DESC_PER_Q 1000 45 46 #define PRESTERA_SDMA_TX_DESC_PER_Q 1000 47 #define PRESTERA_SDMA_TX_MAX_BURST 64 48 49 #define PRESTERA_SDMA_TX_DESC_OWNER(desc) \ 50 ((le32_to_cpu((desc)->word1) & BIT(31)) >> 31) 51 52 #define PRESTERA_SDMA_TX_DESC_CPU_OWN 0 53 #define PRESTERA_SDMA_TX_DESC_DMA_OWN 1U 54 55 #define PRESTERA_SDMA_TX_DESC_IS_SENT(desc) \ 56 (PRESTERA_SDMA_TX_DESC_OWNER(desc) == PRESTERA_SDMA_TX_DESC_CPU_OWN) 57 58 #define PRESTERA_SDMA_TX_DESC_LAST BIT(20) 59 #define PRESTERA_SDMA_TX_DESC_FIRST BIT(21) 60 #define PRESTERA_SDMA_TX_DESC_CALC_CRC BIT(12) 61 62 #define PRESTERA_SDMA_TX_DESC_SINGLE \ 63 (PRESTERA_SDMA_TX_DESC_FIRST | PRESTERA_SDMA_TX_DESC_LAST) 64 65 #define PRESTERA_SDMA_TX_DESC_INIT \ 66 (PRESTERA_SDMA_TX_DESC_SINGLE | PRESTERA_SDMA_TX_DESC_CALC_CRC) 67 68 #define PRESTERA_SDMA_RX_INTR_MASK_REG 0x2814 69 #define PRESTERA_SDMA_RX_QUEUE_STATUS_REG 0x2680 70 #define PRESTERA_SDMA_RX_QUEUE_DESC_REG(n) (0x260C + (n) * 16) 71 72 #define PRESTERA_SDMA_TX_QUEUE_DESC_REG 0x26C0 73 #define PRESTERA_SDMA_TX_QUEUE_START_REG 0x2868 74 75 struct prestera_sdma_buf { 76 struct prestera_sdma_desc *desc; 77 dma_addr_t desc_dma; 78 struct sk_buff *skb; 79 dma_addr_t buf_dma; 80 bool is_used; 81 }; 82 83 struct prestera_rx_ring { 84 struct prestera_sdma_buf *bufs; 85 int next_rx; 86 }; 87 88 struct prestera_tx_ring { 89 struct prestera_sdma_buf *bufs; 90 int next_tx; 91 int max_burst; 92 int burst; 93 }; 94 95 struct prestera_sdma { 96 struct prestera_rx_ring rx_ring[PRESTERA_SDMA_RX_QUEUE_NUM]; 97 struct prestera_tx_ring tx_ring; 98 struct prestera_switch *sw; 99 struct dma_pool *desc_pool; 100 struct work_struct tx_work; 101 struct napi_struct rx_napi; 102 struct net_device napi_dev; 103 u32 map_addr; 104 u64 dma_mask; 105 /* protect SDMA with concurrrent access from multiple CPUs */ 106 spinlock_t tx_lock; 107 }; 108 109 struct prestera_rxtx { 110 struct prestera_sdma sdma; 111 }; 112 113 static int prestera_sdma_buf_init(struct prestera_sdma *sdma, 114 struct prestera_sdma_buf *buf) 115 { 116 struct prestera_sdma_desc *desc; 117 dma_addr_t dma; 118 119 desc = dma_pool_alloc(sdma->desc_pool, GFP_DMA | GFP_KERNEL, &dma); 120 if (!desc) 121 return -ENOMEM; 122 123 buf->buf_dma = DMA_MAPPING_ERROR; 124 buf->desc_dma = dma; 125 buf->desc = desc; 126 buf->skb = NULL; 127 128 return 0; 129 } 130 131 static u32 prestera_sdma_map(struct prestera_sdma *sdma, dma_addr_t pa) 132 { 133 return sdma->map_addr + pa; 134 } 135 136 static void prestera_sdma_rx_desc_init(struct prestera_sdma *sdma, 137 struct prestera_sdma_desc *desc, 138 dma_addr_t buf) 139 { 140 u32 word = le32_to_cpu(desc->word2); 141 142 u32p_replace_bits(&word, PRESTERA_SDMA_BUFF_SIZE_MAX, GENMASK(15, 0)); 143 desc->word2 = cpu_to_le32(word); 144 145 desc->buff = cpu_to_le32(prestera_sdma_map(sdma, buf)); 146 147 /* make sure buffer is set before reset the descriptor */ 148 wmb(); 149 150 desc->word1 = cpu_to_le32(0xA0000000); 151 } 152 153 static void prestera_sdma_rx_desc_set_next(struct prestera_sdma *sdma, 154 struct prestera_sdma_desc *desc, 155 dma_addr_t next) 156 { 157 desc->next = cpu_to_le32(prestera_sdma_map(sdma, next)); 158 } 159 160 static int prestera_sdma_rx_skb_alloc(struct prestera_sdma *sdma, 161 struct prestera_sdma_buf *buf) 162 { 163 struct device *dev = sdma->sw->dev->dev; 164 struct sk_buff *skb; 165 dma_addr_t dma; 166 167 skb = alloc_skb(PRESTERA_SDMA_BUFF_SIZE_MAX, GFP_DMA | GFP_ATOMIC); 168 if (!skb) 169 return -ENOMEM; 170 171 dma = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE); 172 if (dma_mapping_error(dev, dma)) 173 goto err_dma_map; 174 175 if (buf->skb) 176 dma_unmap_single(dev, buf->buf_dma, buf->skb->len, 177 DMA_FROM_DEVICE); 178 179 buf->buf_dma = dma; 180 buf->skb = skb; 181 182 return 0; 183 184 err_dma_map: 185 kfree_skb(skb); 186 187 return -ENOMEM; 188 } 189 190 static struct sk_buff *prestera_sdma_rx_skb_get(struct prestera_sdma *sdma, 191 struct prestera_sdma_buf *buf) 192 { 193 dma_addr_t buf_dma = buf->buf_dma; 194 struct sk_buff *skb = buf->skb; 195 u32 len = skb->len; 196 int err; 197 198 err = prestera_sdma_rx_skb_alloc(sdma, buf); 199 if (err) { 200 buf->buf_dma = buf_dma; 201 buf->skb = skb; 202 203 skb = alloc_skb(skb->len, GFP_ATOMIC); 204 if (skb) { 205 skb_put(skb, len); 206 skb_copy_from_linear_data(buf->skb, skb->data, len); 207 } 208 } 209 210 prestera_sdma_rx_desc_init(sdma, buf->desc, buf->buf_dma); 211 212 return skb; 213 } 214 215 static int prestera_rxtx_process_skb(struct prestera_sdma *sdma, 216 struct sk_buff *skb) 217 { 218 struct prestera_port *port; 219 struct prestera_dsa dsa; 220 u32 hw_port, dev_id; 221 u8 cpu_code; 222 int err; 223 224 skb_pull(skb, ETH_HLEN); 225 226 /* ethertype field is part of the dsa header */ 227 err = prestera_dsa_parse(&dsa, skb->data - ETH_TLEN); 228 if (err) 229 return err; 230 231 dev_id = dsa.hw_dev_num; 232 hw_port = dsa.port_num; 233 234 port = prestera_port_find_by_hwid(sdma->sw, dev_id, hw_port); 235 if (unlikely(!port)) { 236 dev_warn_ratelimited(prestera_dev(sdma->sw), "received pkt for non-existent port(%u, %u)\n", 237 dev_id, hw_port); 238 return -ENOENT; 239 } 240 241 if (unlikely(!pskb_may_pull(skb, PRESTERA_DSA_HLEN))) 242 return -EINVAL; 243 244 /* remove DSA tag and update checksum */ 245 skb_pull_rcsum(skb, PRESTERA_DSA_HLEN); 246 247 memmove(skb->data - ETH_HLEN, skb->data - ETH_HLEN - PRESTERA_DSA_HLEN, 248 ETH_ALEN * 2); 249 250 skb_push(skb, ETH_HLEN); 251 252 skb->protocol = eth_type_trans(skb, port->dev); 253 254 if (dsa.vlan.is_tagged) { 255 u16 tci = dsa.vlan.vid & VLAN_VID_MASK; 256 257 tci |= dsa.vlan.vpt << VLAN_PRIO_SHIFT; 258 if (dsa.vlan.cfi_bit) 259 tci |= VLAN_CFI_MASK; 260 261 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tci); 262 } 263 264 cpu_code = dsa.cpu_code; 265 prestera_devlink_trap_report(port, skb, cpu_code); 266 267 return 0; 268 } 269 270 static int prestera_sdma_next_rx_buf_idx(int buf_idx) 271 { 272 return (buf_idx + 1) % PRESTERA_SDMA_RX_DESC_PER_Q; 273 } 274 275 static int prestera_sdma_rx_poll(struct napi_struct *napi, int budget) 276 { 277 int qnum = PRESTERA_SDMA_RX_QUEUE_NUM; 278 unsigned int rxq_done_map = 0; 279 struct prestera_sdma *sdma; 280 struct list_head rx_list; 281 unsigned int qmask; 282 int pkts_done = 0; 283 int q; 284 285 qnum = PRESTERA_SDMA_RX_QUEUE_NUM; 286 qmask = GENMASK(qnum - 1, 0); 287 288 INIT_LIST_HEAD(&rx_list); 289 290 sdma = container_of(napi, struct prestera_sdma, rx_napi); 291 292 while (pkts_done < budget && rxq_done_map != qmask) { 293 for (q = 0; q < qnum && pkts_done < budget; q++) { 294 struct prestera_rx_ring *ring = &sdma->rx_ring[q]; 295 struct prestera_sdma_desc *desc; 296 struct prestera_sdma_buf *buf; 297 int buf_idx = ring->next_rx; 298 struct sk_buff *skb; 299 300 buf = &ring->bufs[buf_idx]; 301 desc = buf->desc; 302 303 if (PRESTERA_SDMA_RX_DESC_IS_RCVD(desc)) { 304 rxq_done_map &= ~BIT(q); 305 } else { 306 rxq_done_map |= BIT(q); 307 continue; 308 } 309 310 pkts_done++; 311 312 __skb_trim(buf->skb, PRESTERA_SDMA_RX_DESC_PKT_LEN(desc)); 313 314 skb = prestera_sdma_rx_skb_get(sdma, buf); 315 if (!skb) 316 goto rx_next_buf; 317 318 if (unlikely(prestera_rxtx_process_skb(sdma, skb))) 319 goto rx_next_buf; 320 321 list_add_tail(&skb->list, &rx_list); 322 rx_next_buf: 323 ring->next_rx = prestera_sdma_next_rx_buf_idx(buf_idx); 324 } 325 } 326 327 if (pkts_done < budget && napi_complete_done(napi, pkts_done)) 328 prestera_write(sdma->sw, PRESTERA_SDMA_RX_INTR_MASK_REG, 329 GENMASK(9, 2)); 330 331 netif_receive_skb_list(&rx_list); 332 333 return pkts_done; 334 } 335 336 static void prestera_sdma_rx_fini(struct prestera_sdma *sdma) 337 { 338 int qnum = PRESTERA_SDMA_RX_QUEUE_NUM; 339 int q, b; 340 341 /* disable all rx queues */ 342 prestera_write(sdma->sw, PRESTERA_SDMA_RX_QUEUE_STATUS_REG, 343 GENMASK(15, 8)); 344 345 for (q = 0; q < qnum; q++) { 346 struct prestera_rx_ring *ring = &sdma->rx_ring[q]; 347 348 if (!ring->bufs) 349 break; 350 351 for (b = 0; b < PRESTERA_SDMA_RX_DESC_PER_Q; b++) { 352 struct prestera_sdma_buf *buf = &ring->bufs[b]; 353 354 if (buf->desc_dma) 355 dma_pool_free(sdma->desc_pool, buf->desc, 356 buf->desc_dma); 357 358 if (!buf->skb) 359 continue; 360 361 if (buf->buf_dma != DMA_MAPPING_ERROR) 362 dma_unmap_single(sdma->sw->dev->dev, 363 buf->buf_dma, buf->skb->len, 364 DMA_FROM_DEVICE); 365 kfree_skb(buf->skb); 366 } 367 } 368 } 369 370 static int prestera_sdma_rx_init(struct prestera_sdma *sdma) 371 { 372 int bnum = PRESTERA_SDMA_RX_DESC_PER_Q; 373 int qnum = PRESTERA_SDMA_RX_QUEUE_NUM; 374 int err; 375 int q; 376 377 /* disable all rx queues */ 378 prestera_write(sdma->sw, PRESTERA_SDMA_RX_QUEUE_STATUS_REG, 379 GENMASK(15, 8)); 380 381 for (q = 0; q < qnum; q++) { 382 struct prestera_sdma_buf *head, *tail, *next, *prev; 383 struct prestera_rx_ring *ring = &sdma->rx_ring[q]; 384 385 ring->bufs = kmalloc_array(bnum, sizeof(*head), GFP_KERNEL); 386 if (!ring->bufs) 387 return -ENOMEM; 388 389 ring->next_rx = 0; 390 391 tail = &ring->bufs[bnum - 1]; 392 head = &ring->bufs[0]; 393 next = head; 394 prev = next; 395 396 do { 397 err = prestera_sdma_buf_init(sdma, next); 398 if (err) 399 return err; 400 401 err = prestera_sdma_rx_skb_alloc(sdma, next); 402 if (err) 403 return err; 404 405 prestera_sdma_rx_desc_init(sdma, next->desc, 406 next->buf_dma); 407 408 prestera_sdma_rx_desc_set_next(sdma, prev->desc, 409 next->desc_dma); 410 411 prev = next; 412 next++; 413 } while (prev != tail); 414 415 /* join tail with head to make a circular list */ 416 prestera_sdma_rx_desc_set_next(sdma, tail->desc, head->desc_dma); 417 418 prestera_write(sdma->sw, PRESTERA_SDMA_RX_QUEUE_DESC_REG(q), 419 prestera_sdma_map(sdma, head->desc_dma)); 420 } 421 422 /* make sure all rx descs are filled before enabling all rx queues */ 423 wmb(); 424 425 prestera_write(sdma->sw, PRESTERA_SDMA_RX_QUEUE_STATUS_REG, 426 GENMASK(7, 0)); 427 428 return 0; 429 } 430 431 static void prestera_sdma_tx_desc_init(struct prestera_sdma *sdma, 432 struct prestera_sdma_desc *desc) 433 { 434 desc->word1 = cpu_to_le32(PRESTERA_SDMA_TX_DESC_INIT); 435 desc->word2 = 0; 436 } 437 438 static void prestera_sdma_tx_desc_set_next(struct prestera_sdma *sdma, 439 struct prestera_sdma_desc *desc, 440 dma_addr_t next) 441 { 442 desc->next = cpu_to_le32(prestera_sdma_map(sdma, next)); 443 } 444 445 static void prestera_sdma_tx_desc_set_buf(struct prestera_sdma *sdma, 446 struct prestera_sdma_desc *desc, 447 dma_addr_t buf, size_t len) 448 { 449 u32 word = le32_to_cpu(desc->word2); 450 451 u32p_replace_bits(&word, len + ETH_FCS_LEN, GENMASK(30, 16)); 452 453 desc->buff = cpu_to_le32(prestera_sdma_map(sdma, buf)); 454 desc->word2 = cpu_to_le32(word); 455 } 456 457 static void prestera_sdma_tx_desc_xmit(struct prestera_sdma_desc *desc) 458 { 459 u32 word = le32_to_cpu(desc->word1); 460 461 word |= PRESTERA_SDMA_TX_DESC_DMA_OWN << 31; 462 463 /* make sure everything is written before enable xmit */ 464 wmb(); 465 466 desc->word1 = cpu_to_le32(word); 467 } 468 469 static int prestera_sdma_tx_buf_map(struct prestera_sdma *sdma, 470 struct prestera_sdma_buf *buf, 471 struct sk_buff *skb) 472 { 473 struct device *dma_dev = sdma->sw->dev->dev; 474 dma_addr_t dma; 475 476 dma = dma_map_single(dma_dev, skb->data, skb->len, DMA_TO_DEVICE); 477 if (dma_mapping_error(dma_dev, dma)) 478 return -ENOMEM; 479 480 buf->buf_dma = dma; 481 buf->skb = skb; 482 483 return 0; 484 } 485 486 static void prestera_sdma_tx_buf_unmap(struct prestera_sdma *sdma, 487 struct prestera_sdma_buf *buf) 488 { 489 struct device *dma_dev = sdma->sw->dev->dev; 490 491 dma_unmap_single(dma_dev, buf->buf_dma, buf->skb->len, DMA_TO_DEVICE); 492 } 493 494 static void prestera_sdma_tx_recycle_work_fn(struct work_struct *work) 495 { 496 int bnum = PRESTERA_SDMA_TX_DESC_PER_Q; 497 struct prestera_tx_ring *tx_ring; 498 struct prestera_sdma *sdma; 499 int b; 500 501 sdma = container_of(work, struct prestera_sdma, tx_work); 502 503 tx_ring = &sdma->tx_ring; 504 505 for (b = 0; b < bnum; b++) { 506 struct prestera_sdma_buf *buf = &tx_ring->bufs[b]; 507 508 if (!buf->is_used) 509 continue; 510 511 if (!PRESTERA_SDMA_TX_DESC_IS_SENT(buf->desc)) 512 continue; 513 514 prestera_sdma_tx_buf_unmap(sdma, buf); 515 dev_consume_skb_any(buf->skb); 516 buf->skb = NULL; 517 518 /* make sure everything is cleaned up */ 519 wmb(); 520 521 buf->is_used = false; 522 } 523 } 524 525 static int prestera_sdma_tx_init(struct prestera_sdma *sdma) 526 { 527 struct prestera_sdma_buf *head, *tail, *next, *prev; 528 struct prestera_tx_ring *tx_ring = &sdma->tx_ring; 529 int bnum = PRESTERA_SDMA_TX_DESC_PER_Q; 530 int err; 531 532 INIT_WORK(&sdma->tx_work, prestera_sdma_tx_recycle_work_fn); 533 spin_lock_init(&sdma->tx_lock); 534 535 tx_ring->bufs = kmalloc_array(bnum, sizeof(*head), GFP_KERNEL); 536 if (!tx_ring->bufs) 537 return -ENOMEM; 538 539 tail = &tx_ring->bufs[bnum - 1]; 540 head = &tx_ring->bufs[0]; 541 next = head; 542 prev = next; 543 544 tx_ring->max_burst = PRESTERA_SDMA_TX_MAX_BURST; 545 tx_ring->burst = tx_ring->max_burst; 546 tx_ring->next_tx = 0; 547 548 do { 549 err = prestera_sdma_buf_init(sdma, next); 550 if (err) 551 return err; 552 553 next->is_used = false; 554 555 prestera_sdma_tx_desc_init(sdma, next->desc); 556 557 prestera_sdma_tx_desc_set_next(sdma, prev->desc, 558 next->desc_dma); 559 560 prev = next; 561 next++; 562 } while (prev != tail); 563 564 /* join tail with head to make a circular list */ 565 prestera_sdma_tx_desc_set_next(sdma, tail->desc, head->desc_dma); 566 567 /* make sure descriptors are written */ 568 wmb(); 569 570 prestera_write(sdma->sw, PRESTERA_SDMA_TX_QUEUE_DESC_REG, 571 prestera_sdma_map(sdma, head->desc_dma)); 572 573 return 0; 574 } 575 576 static void prestera_sdma_tx_fini(struct prestera_sdma *sdma) 577 { 578 struct prestera_tx_ring *ring = &sdma->tx_ring; 579 int bnum = PRESTERA_SDMA_TX_DESC_PER_Q; 580 int b; 581 582 cancel_work_sync(&sdma->tx_work); 583 584 if (!ring->bufs) 585 return; 586 587 for (b = 0; b < bnum; b++) { 588 struct prestera_sdma_buf *buf = &ring->bufs[b]; 589 590 if (buf->desc) 591 dma_pool_free(sdma->desc_pool, buf->desc, 592 buf->desc_dma); 593 594 if (!buf->skb) 595 continue; 596 597 dma_unmap_single(sdma->sw->dev->dev, buf->buf_dma, 598 buf->skb->len, DMA_TO_DEVICE); 599 600 dev_consume_skb_any(buf->skb); 601 } 602 } 603 604 static void prestera_rxtx_handle_event(struct prestera_switch *sw, 605 struct prestera_event *evt, 606 void *arg) 607 { 608 struct prestera_sdma *sdma = arg; 609 610 if (evt->id != PRESTERA_RXTX_EVENT_RCV_PKT) 611 return; 612 613 prestera_write(sdma->sw, PRESTERA_SDMA_RX_INTR_MASK_REG, 0); 614 napi_schedule(&sdma->rx_napi); 615 } 616 617 static int prestera_sdma_switch_init(struct prestera_switch *sw) 618 { 619 struct prestera_sdma *sdma = &sw->rxtx->sdma; 620 struct device *dev = sw->dev->dev; 621 struct prestera_rxtx_params p; 622 int err; 623 624 p.use_sdma = true; 625 626 err = prestera_hw_rxtx_init(sw, &p); 627 if (err) { 628 dev_err(dev, "failed to init rxtx by hw\n"); 629 return err; 630 } 631 632 sdma->dma_mask = dma_get_mask(dev); 633 sdma->map_addr = p.map_addr; 634 sdma->sw = sw; 635 636 sdma->desc_pool = dma_pool_create("desc_pool", dev, 637 sizeof(struct prestera_sdma_desc), 638 16, 0); 639 if (!sdma->desc_pool) 640 return -ENOMEM; 641 642 err = prestera_sdma_rx_init(sdma); 643 if (err) { 644 dev_err(dev, "failed to init rx ring\n"); 645 goto err_rx_init; 646 } 647 648 err = prestera_sdma_tx_init(sdma); 649 if (err) { 650 dev_err(dev, "failed to init tx ring\n"); 651 goto err_tx_init; 652 } 653 654 err = prestera_hw_event_handler_register(sw, PRESTERA_EVENT_TYPE_RXTX, 655 prestera_rxtx_handle_event, 656 sdma); 657 if (err) 658 goto err_evt_register; 659 660 init_dummy_netdev(&sdma->napi_dev); 661 662 netif_napi_add(&sdma->napi_dev, &sdma->rx_napi, prestera_sdma_rx_poll, 64); 663 napi_enable(&sdma->rx_napi); 664 665 return 0; 666 667 err_evt_register: 668 err_tx_init: 669 prestera_sdma_tx_fini(sdma); 670 err_rx_init: 671 prestera_sdma_rx_fini(sdma); 672 673 dma_pool_destroy(sdma->desc_pool); 674 return err; 675 } 676 677 static void prestera_sdma_switch_fini(struct prestera_switch *sw) 678 { 679 struct prestera_sdma *sdma = &sw->rxtx->sdma; 680 681 napi_disable(&sdma->rx_napi); 682 netif_napi_del(&sdma->rx_napi); 683 prestera_hw_event_handler_unregister(sw, PRESTERA_EVENT_TYPE_RXTX, 684 prestera_rxtx_handle_event); 685 prestera_sdma_tx_fini(sdma); 686 prestera_sdma_rx_fini(sdma); 687 dma_pool_destroy(sdma->desc_pool); 688 } 689 690 static bool prestera_sdma_is_ready(struct prestera_sdma *sdma) 691 { 692 return !(prestera_read(sdma->sw, PRESTERA_SDMA_TX_QUEUE_START_REG) & 1); 693 } 694 695 static int prestera_sdma_tx_wait(struct prestera_sdma *sdma, 696 struct prestera_tx_ring *tx_ring) 697 { 698 int tx_wait_num = PRESTERA_SDMA_WAIT_MUL * tx_ring->max_burst; 699 700 do { 701 if (prestera_sdma_is_ready(sdma)) 702 return 0; 703 704 udelay(1); 705 } while (--tx_wait_num); 706 707 return -EBUSY; 708 } 709 710 static void prestera_sdma_tx_start(struct prestera_sdma *sdma) 711 { 712 prestera_write(sdma->sw, PRESTERA_SDMA_TX_QUEUE_START_REG, 1); 713 schedule_work(&sdma->tx_work); 714 } 715 716 static netdev_tx_t prestera_sdma_xmit(struct prestera_sdma *sdma, 717 struct sk_buff *skb) 718 { 719 struct device *dma_dev = sdma->sw->dev->dev; 720 struct net_device *dev = skb->dev; 721 struct prestera_tx_ring *tx_ring; 722 struct prestera_sdma_buf *buf; 723 int err; 724 725 spin_lock(&sdma->tx_lock); 726 727 tx_ring = &sdma->tx_ring; 728 729 buf = &tx_ring->bufs[tx_ring->next_tx]; 730 if (buf->is_used) { 731 schedule_work(&sdma->tx_work); 732 goto drop_skb; 733 } 734 735 if (unlikely(eth_skb_pad(skb))) 736 goto drop_skb_nofree; 737 738 err = prestera_sdma_tx_buf_map(sdma, buf, skb); 739 if (err) 740 goto drop_skb; 741 742 prestera_sdma_tx_desc_set_buf(sdma, buf->desc, buf->buf_dma, skb->len); 743 744 dma_sync_single_for_device(dma_dev, buf->buf_dma, skb->len, 745 DMA_TO_DEVICE); 746 747 if (tx_ring->burst) { 748 tx_ring->burst--; 749 } else { 750 tx_ring->burst = tx_ring->max_burst; 751 752 err = prestera_sdma_tx_wait(sdma, tx_ring); 753 if (err) 754 goto drop_skb_unmap; 755 } 756 757 tx_ring->next_tx = (tx_ring->next_tx + 1) % PRESTERA_SDMA_TX_DESC_PER_Q; 758 prestera_sdma_tx_desc_xmit(buf->desc); 759 buf->is_used = true; 760 761 prestera_sdma_tx_start(sdma); 762 763 goto tx_done; 764 765 drop_skb_unmap: 766 prestera_sdma_tx_buf_unmap(sdma, buf); 767 drop_skb: 768 dev_consume_skb_any(skb); 769 drop_skb_nofree: 770 dev->stats.tx_dropped++; 771 tx_done: 772 spin_unlock(&sdma->tx_lock); 773 return NETDEV_TX_OK; 774 } 775 776 int prestera_rxtx_switch_init(struct prestera_switch *sw) 777 { 778 struct prestera_rxtx *rxtx; 779 780 rxtx = kzalloc(sizeof(*rxtx), GFP_KERNEL); 781 if (!rxtx) 782 return -ENOMEM; 783 784 sw->rxtx = rxtx; 785 786 return prestera_sdma_switch_init(sw); 787 } 788 789 void prestera_rxtx_switch_fini(struct prestera_switch *sw) 790 { 791 prestera_sdma_switch_fini(sw); 792 kfree(sw->rxtx); 793 } 794 795 int prestera_rxtx_port_init(struct prestera_port *port) 796 { 797 port->dev->needed_headroom = PRESTERA_DSA_HLEN; 798 return 0; 799 } 800 801 netdev_tx_t prestera_rxtx_xmit(struct prestera_port *port, struct sk_buff *skb) 802 { 803 struct prestera_dsa dsa; 804 805 dsa.hw_dev_num = port->dev_id; 806 dsa.port_num = port->hw_id; 807 808 if (skb_cow_head(skb, PRESTERA_DSA_HLEN) < 0) 809 return NET_XMIT_DROP; 810 811 skb_push(skb, PRESTERA_DSA_HLEN); 812 memmove(skb->data, skb->data + PRESTERA_DSA_HLEN, 2 * ETH_ALEN); 813 814 if (prestera_dsa_build(&dsa, skb->data + 2 * ETH_ALEN) != 0) 815 return NET_XMIT_DROP; 816 817 return prestera_sdma_xmit(&port->sw->rxtx->sdma, skb); 818 } 819