1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/bpf_trace.h> 6 #include <linux/tcp.h> 7 #include <linux/udp.h> 8 #include <linux/vmalloc.h> 9 #include <linux/ptp_classify.h> 10 #include <net/ip6_checksum.h> 11 #include <net/pkt_sched.h> 12 #include <net/tso.h> 13 14 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) 15 { 16 int num_tx_rings = priv->num_tx_rings; 17 int i; 18 19 for (i = 0; i < priv->num_rx_rings; i++) 20 if (priv->rx_ring[i]->xdp.prog) 21 return num_tx_rings - num_possible_cpus(); 22 23 return num_tx_rings; 24 } 25 26 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv, 27 struct enetc_bdr *tx_ring) 28 { 29 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring; 30 31 return priv->rx_ring[index]; 32 } 33 34 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd) 35 { 36 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect) 37 return NULL; 38 39 return tx_swbd->skb; 40 } 41 42 static struct xdp_frame * 43 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd) 44 { 45 if (tx_swbd->is_xdp_redirect) 46 return tx_swbd->xdp_frame; 47 48 return NULL; 49 } 50 51 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 52 struct enetc_tx_swbd *tx_swbd) 53 { 54 /* For XDP_TX, pages come from RX, whereas for the other contexts where 55 * we have is_dma_page_set, those come from skb_frag_dma_map. We need 56 * to match the DMA mapping length, so we need to differentiate those. 57 */ 58 if (tx_swbd->is_dma_page) 59 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 60 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len, 61 tx_swbd->dir); 62 else 63 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 64 tx_swbd->len, tx_swbd->dir); 65 tx_swbd->dma = 0; 66 } 67 68 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring, 69 struct enetc_tx_swbd *tx_swbd) 70 { 71 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 72 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 73 74 if (tx_swbd->dma) 75 enetc_unmap_tx_buff(tx_ring, tx_swbd); 76 77 if (xdp_frame) { 78 xdp_return_frame(tx_swbd->xdp_frame); 79 tx_swbd->xdp_frame = NULL; 80 } else if (skb) { 81 dev_kfree_skb_any(skb); 82 tx_swbd->skb = NULL; 83 } 84 } 85 86 /* Let H/W know BD ring has been updated */ 87 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring) 88 { 89 /* includes wmb() */ 90 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use); 91 } 92 93 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp, 94 u8 *msgtype, u8 *twostep, 95 u16 *correction_offset, u16 *body_offset) 96 { 97 unsigned int ptp_class; 98 struct ptp_header *hdr; 99 unsigned int type; 100 u8 *base; 101 102 ptp_class = ptp_classify_raw(skb); 103 if (ptp_class == PTP_CLASS_NONE) 104 return -EINVAL; 105 106 hdr = ptp_parse_header(skb, ptp_class); 107 if (!hdr) 108 return -EINVAL; 109 110 type = ptp_class & PTP_CLASS_PMASK; 111 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6) 112 *udp = 1; 113 else 114 *udp = 0; 115 116 *msgtype = ptp_get_msgtype(hdr, ptp_class); 117 *twostep = hdr->flag_field[0] & 0x2; 118 119 base = skb_mac_header(skb); 120 *correction_offset = (u8 *)&hdr->correction - base; 121 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 122 123 return 0; 124 } 125 126 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 127 { 128 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false; 129 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 130 struct enetc_hw *hw = &priv->si->hw; 131 struct enetc_tx_swbd *tx_swbd; 132 int len = skb_headlen(skb); 133 union enetc_tx_bd temp_bd; 134 u8 msgtype, twostep, udp; 135 union enetc_tx_bd *txbd; 136 u16 offset1, offset2; 137 int i, count = 0; 138 skb_frag_t *frag; 139 unsigned int f; 140 dma_addr_t dma; 141 u8 flags = 0; 142 143 i = tx_ring->next_to_use; 144 txbd = ENETC_TXBD(*tx_ring, i); 145 prefetchw(txbd); 146 147 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 148 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 149 goto dma_err; 150 151 temp_bd.addr = cpu_to_le64(dma); 152 temp_bd.buf_len = cpu_to_le16(len); 153 temp_bd.lstatus = 0; 154 155 tx_swbd = &tx_ring->tx_swbd[i]; 156 tx_swbd->dma = dma; 157 tx_swbd->len = len; 158 tx_swbd->is_dma_page = 0; 159 tx_swbd->dir = DMA_TO_DEVICE; 160 count++; 161 162 do_vlan = skb_vlan_tag_present(skb); 163 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 164 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1, 165 &offset2) || 166 msgtype != PTP_MSGTYPE_SYNC || twostep) 167 WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 168 else 169 do_onestep_tstamp = true; 170 } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) { 171 do_twostep_tstamp = true; 172 } 173 174 tx_swbd->do_twostep_tstamp = do_twostep_tstamp; 175 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV); 176 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en; 177 178 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp) 179 flags |= ENETC_TXBD_FLAGS_EX; 180 181 if (tx_ring->tsd_enable) 182 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 183 184 /* first BD needs frm_len and offload flags set */ 185 temp_bd.frm_len = cpu_to_le16(skb->len); 186 temp_bd.flags = flags; 187 188 if (flags & ENETC_TXBD_FLAGS_TSE) 189 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 190 flags); 191 192 if (flags & ENETC_TXBD_FLAGS_EX) { 193 u8 e_flags = 0; 194 *txbd = temp_bd; 195 enetc_clear_tx_bd(&temp_bd); 196 197 /* add extension BD for VLAN and/or timestamping */ 198 flags = 0; 199 tx_swbd++; 200 txbd++; 201 i++; 202 if (unlikely(i == tx_ring->bd_count)) { 203 i = 0; 204 tx_swbd = tx_ring->tx_swbd; 205 txbd = ENETC_TXBD(*tx_ring, 0); 206 } 207 prefetchw(txbd); 208 209 if (do_vlan) { 210 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 211 temp_bd.ext.tpid = 0; /* < C-TAG */ 212 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 213 } 214 215 if (do_onestep_tstamp) { 216 u32 lo, hi, val; 217 u64 sec, nsec; 218 u8 *data; 219 220 lo = enetc_rd_hot(hw, ENETC_SICTR0); 221 hi = enetc_rd_hot(hw, ENETC_SICTR1); 222 sec = (u64)hi << 32 | lo; 223 nsec = do_div(sec, 1000000000); 224 225 /* Configure extension BD */ 226 temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff); 227 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP; 228 229 /* Update originTimestamp field of Sync packet 230 * - 48 bits seconds field 231 * - 32 bits nanseconds field 232 */ 233 data = skb_mac_header(skb); 234 *(__be16 *)(data + offset2) = 235 htons((sec >> 32) & 0xffff); 236 *(__be32 *)(data + offset2 + 2) = 237 htonl(sec & 0xffffffff); 238 *(__be32 *)(data + offset2 + 6) = htonl(nsec); 239 240 /* Configure single-step register */ 241 val = ENETC_PM0_SINGLE_STEP_EN; 242 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1); 243 if (udp) 244 val |= ENETC_PM0_SINGLE_STEP_CH; 245 246 enetc_port_wr(hw, ENETC_PM0_SINGLE_STEP, val); 247 enetc_port_wr(hw, ENETC_PM1_SINGLE_STEP, val); 248 } else if (do_twostep_tstamp) { 249 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 250 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 251 } 252 253 temp_bd.ext.e_flags = e_flags; 254 count++; 255 } 256 257 frag = &skb_shinfo(skb)->frags[0]; 258 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 259 len = skb_frag_size(frag); 260 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 261 DMA_TO_DEVICE); 262 if (dma_mapping_error(tx_ring->dev, dma)) 263 goto dma_err; 264 265 *txbd = temp_bd; 266 enetc_clear_tx_bd(&temp_bd); 267 268 flags = 0; 269 tx_swbd++; 270 txbd++; 271 i++; 272 if (unlikely(i == tx_ring->bd_count)) { 273 i = 0; 274 tx_swbd = tx_ring->tx_swbd; 275 txbd = ENETC_TXBD(*tx_ring, 0); 276 } 277 prefetchw(txbd); 278 279 temp_bd.addr = cpu_to_le64(dma); 280 temp_bd.buf_len = cpu_to_le16(len); 281 282 tx_swbd->dma = dma; 283 tx_swbd->len = len; 284 tx_swbd->is_dma_page = 1; 285 tx_swbd->dir = DMA_TO_DEVICE; 286 count++; 287 } 288 289 /* last BD needs 'F' bit set */ 290 flags |= ENETC_TXBD_FLAGS_F; 291 temp_bd.flags = flags; 292 *txbd = temp_bd; 293 294 tx_ring->tx_swbd[i].is_eof = true; 295 tx_ring->tx_swbd[i].skb = skb; 296 297 enetc_bdr_idx_inc(tx_ring, &i); 298 tx_ring->next_to_use = i; 299 300 skb_tx_timestamp(skb); 301 302 enetc_update_tx_ring_tail(tx_ring); 303 304 return count; 305 306 dma_err: 307 dev_err(tx_ring->dev, "DMA map error"); 308 309 do { 310 tx_swbd = &tx_ring->tx_swbd[i]; 311 enetc_free_tx_frame(tx_ring, tx_swbd); 312 if (i == 0) 313 i = tx_ring->bd_count; 314 i--; 315 } while (count--); 316 317 return 0; 318 } 319 320 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 321 struct enetc_tx_swbd *tx_swbd, 322 union enetc_tx_bd *txbd, int *i, int hdr_len, 323 int data_len) 324 { 325 union enetc_tx_bd txbd_tmp; 326 u8 flags = 0, e_flags = 0; 327 dma_addr_t addr; 328 329 enetc_clear_tx_bd(&txbd_tmp); 330 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 331 332 if (skb_vlan_tag_present(skb)) 333 flags |= ENETC_TXBD_FLAGS_EX; 334 335 txbd_tmp.addr = cpu_to_le64(addr); 336 txbd_tmp.buf_len = cpu_to_le16(hdr_len); 337 338 /* first BD needs frm_len and offload flags set */ 339 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len); 340 txbd_tmp.flags = flags; 341 342 /* For the TSO header we do not set the dma address since we do not 343 * want it unmapped when we do cleanup. We still set len so that we 344 * count the bytes sent. 345 */ 346 tx_swbd->len = hdr_len; 347 tx_swbd->do_twostep_tstamp = false; 348 tx_swbd->check_wb = false; 349 350 /* Actually write the header in the BD */ 351 *txbd = txbd_tmp; 352 353 /* Add extension BD for VLAN */ 354 if (flags & ENETC_TXBD_FLAGS_EX) { 355 /* Get the next BD */ 356 enetc_bdr_idx_inc(tx_ring, i); 357 txbd = ENETC_TXBD(*tx_ring, *i); 358 tx_swbd = &tx_ring->tx_swbd[*i]; 359 prefetchw(txbd); 360 361 /* Setup the VLAN fields */ 362 enetc_clear_tx_bd(&txbd_tmp); 363 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 364 txbd_tmp.ext.tpid = 0; /* < C-TAG */ 365 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 366 367 /* Write the BD */ 368 txbd_tmp.ext.e_flags = e_flags; 369 *txbd = txbd_tmp; 370 } 371 } 372 373 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 374 struct enetc_tx_swbd *tx_swbd, 375 union enetc_tx_bd *txbd, char *data, 376 int size, bool last_bd) 377 { 378 union enetc_tx_bd txbd_tmp; 379 dma_addr_t addr; 380 u8 flags = 0; 381 382 enetc_clear_tx_bd(&txbd_tmp); 383 384 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 385 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) { 386 netdev_err(tx_ring->ndev, "DMA map error\n"); 387 return -ENOMEM; 388 } 389 390 if (last_bd) { 391 flags |= ENETC_TXBD_FLAGS_F; 392 tx_swbd->is_eof = 1; 393 } 394 395 txbd_tmp.addr = cpu_to_le64(addr); 396 txbd_tmp.buf_len = cpu_to_le16(size); 397 txbd_tmp.flags = flags; 398 399 tx_swbd->dma = addr; 400 tx_swbd->len = size; 401 tx_swbd->dir = DMA_TO_DEVICE; 402 403 *txbd = txbd_tmp; 404 405 return 0; 406 } 407 408 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb, 409 char *hdr, int hdr_len, int *l4_hdr_len) 410 { 411 char *l4_hdr = hdr + skb_transport_offset(skb); 412 int mac_hdr_len = skb_network_offset(skb); 413 414 if (tso->tlen != sizeof(struct udphdr)) { 415 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 416 417 tcph->check = 0; 418 } else { 419 struct udphdr *udph = (struct udphdr *)(l4_hdr); 420 421 udph->check = 0; 422 } 423 424 /* Compute the IP checksum. This is necessary since tso_build_hdr() 425 * already incremented the IP ID field. 426 */ 427 if (!tso->ipv6) { 428 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 429 430 iph->check = 0; 431 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 432 } 433 434 /* Compute the checksum over the L4 header. */ 435 *l4_hdr_len = hdr_len - skb_transport_offset(skb); 436 return csum_partial(l4_hdr, *l4_hdr_len, 0); 437 } 438 439 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso, 440 struct sk_buff *skb, char *hdr, int len, 441 __wsum sum) 442 { 443 char *l4_hdr = hdr + skb_transport_offset(skb); 444 __sum16 csum_final; 445 446 /* Complete the L4 checksum by appending the pseudo-header to the 447 * already computed checksum. 448 */ 449 if (!tso->ipv6) 450 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr, 451 ip_hdr(skb)->daddr, 452 len, ip_hdr(skb)->protocol, sum); 453 else 454 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 455 &ipv6_hdr(skb)->daddr, 456 len, ipv6_hdr(skb)->nexthdr, sum); 457 458 if (tso->tlen != sizeof(struct udphdr)) { 459 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 460 461 tcph->check = csum_final; 462 } else { 463 struct udphdr *udph = (struct udphdr *)(l4_hdr); 464 465 udph->check = csum_final; 466 } 467 } 468 469 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 470 { 471 int hdr_len, total_len, data_len; 472 struct enetc_tx_swbd *tx_swbd; 473 union enetc_tx_bd *txbd; 474 struct tso_t tso; 475 __wsum csum, csum2; 476 int count = 0, pos; 477 int err, i, bd_data_num; 478 479 /* Initialize the TSO handler, and prepare the first payload */ 480 hdr_len = tso_start(skb, &tso); 481 total_len = skb->len - hdr_len; 482 i = tx_ring->next_to_use; 483 484 while (total_len > 0) { 485 char *hdr; 486 487 /* Get the BD */ 488 txbd = ENETC_TXBD(*tx_ring, i); 489 tx_swbd = &tx_ring->tx_swbd[i]; 490 prefetchw(txbd); 491 492 /* Determine the length of this packet */ 493 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len); 494 total_len -= data_len; 495 496 /* prepare packet headers: MAC + IP + TCP */ 497 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE; 498 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0); 499 500 /* compute the csum over the L4 header */ 501 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos); 502 enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len); 503 bd_data_num = 0; 504 count++; 505 506 while (data_len > 0) { 507 int size; 508 509 size = min_t(int, tso.size, data_len); 510 511 /* Advance the index in the BDR */ 512 enetc_bdr_idx_inc(tx_ring, &i); 513 txbd = ENETC_TXBD(*tx_ring, i); 514 tx_swbd = &tx_ring->tx_swbd[i]; 515 prefetchw(txbd); 516 517 /* Compute the checksum over this segment of data and 518 * add it to the csum already computed (over the L4 519 * header and possible other data segments). 520 */ 521 csum2 = csum_partial(tso.data, size, 0); 522 csum = csum_block_add(csum, csum2, pos); 523 pos += size; 524 525 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd, 526 tso.data, size, 527 size == data_len); 528 if (err) 529 goto err_map_data; 530 531 data_len -= size; 532 count++; 533 bd_data_num++; 534 tso_build_data(skb, &tso, size); 535 536 if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len)) 537 goto err_chained_bd; 538 } 539 540 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum); 541 542 if (total_len == 0) 543 tx_swbd->skb = skb; 544 545 /* Go to the next BD */ 546 enetc_bdr_idx_inc(tx_ring, &i); 547 } 548 549 tx_ring->next_to_use = i; 550 enetc_update_tx_ring_tail(tx_ring); 551 552 return count; 553 554 err_map_data: 555 dev_err(tx_ring->dev, "DMA map error"); 556 557 err_chained_bd: 558 do { 559 tx_swbd = &tx_ring->tx_swbd[i]; 560 enetc_free_tx_frame(tx_ring, tx_swbd); 561 if (i == 0) 562 i = tx_ring->bd_count; 563 i--; 564 } while (count--); 565 566 return 0; 567 } 568 569 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb, 570 struct net_device *ndev) 571 { 572 struct enetc_ndev_priv *priv = netdev_priv(ndev); 573 struct enetc_bdr *tx_ring; 574 int count, err; 575 576 /* Queue one-step Sync packet if already locked */ 577 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 578 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, 579 &priv->flags)) { 580 skb_queue_tail(&priv->tx_skbs, skb); 581 return NETDEV_TX_OK; 582 } 583 } 584 585 tx_ring = priv->tx_ring[skb->queue_mapping]; 586 587 if (skb_is_gso(skb)) { 588 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) { 589 netif_stop_subqueue(ndev, tx_ring->index); 590 return NETDEV_TX_BUSY; 591 } 592 593 enetc_lock_mdio(); 594 count = enetc_map_tx_tso_buffs(tx_ring, skb); 595 enetc_unlock_mdio(); 596 } else { 597 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 598 if (unlikely(skb_linearize(skb))) 599 goto drop_packet_err; 600 601 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 602 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 603 netif_stop_subqueue(ndev, tx_ring->index); 604 return NETDEV_TX_BUSY; 605 } 606 607 if (skb->ip_summed == CHECKSUM_PARTIAL) { 608 err = skb_checksum_help(skb); 609 if (err) 610 goto drop_packet_err; 611 } 612 enetc_lock_mdio(); 613 count = enetc_map_tx_buffs(tx_ring, skb); 614 enetc_unlock_mdio(); 615 } 616 617 if (unlikely(!count)) 618 goto drop_packet_err; 619 620 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 621 netif_stop_subqueue(ndev, tx_ring->index); 622 623 return NETDEV_TX_OK; 624 625 drop_packet_err: 626 dev_kfree_skb_any(skb); 627 return NETDEV_TX_OK; 628 } 629 630 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 631 { 632 struct enetc_ndev_priv *priv = netdev_priv(ndev); 633 u8 udp, msgtype, twostep; 634 u16 offset1, offset2; 635 636 /* Mark tx timestamp type on skb->cb[0] if requires */ 637 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 638 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) { 639 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK; 640 } else { 641 skb->cb[0] = 0; 642 } 643 644 /* Fall back to two-step timestamp if not one-step Sync packet */ 645 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 646 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, 647 &offset1, &offset2) || 648 msgtype != PTP_MSGTYPE_SYNC || twostep != 0) 649 skb->cb[0] = ENETC_F_TX_TSTAMP; 650 } 651 652 return enetc_start_xmit(skb, ndev); 653 } 654 655 static irqreturn_t enetc_msix(int irq, void *data) 656 { 657 struct enetc_int_vector *v = data; 658 int i; 659 660 enetc_lock_mdio(); 661 662 /* disable interrupts */ 663 enetc_wr_reg_hot(v->rbier, 0); 664 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 665 666 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 667 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 668 669 enetc_unlock_mdio(); 670 671 napi_schedule(&v->napi); 672 673 return IRQ_HANDLED; 674 } 675 676 static void enetc_rx_dim_work(struct work_struct *w) 677 { 678 struct dim *dim = container_of(w, struct dim, work); 679 struct dim_cq_moder moder = 680 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 681 struct enetc_int_vector *v = 682 container_of(dim, struct enetc_int_vector, rx_dim); 683 684 v->rx_ictt = enetc_usecs_to_cycles(moder.usec); 685 dim->state = DIM_START_MEASURE; 686 } 687 688 static void enetc_rx_net_dim(struct enetc_int_vector *v) 689 { 690 struct dim_sample dim_sample = {}; 691 692 v->comp_cnt++; 693 694 if (!v->rx_napi_work) 695 return; 696 697 dim_update_sample(v->comp_cnt, 698 v->rx_ring.stats.packets, 699 v->rx_ring.stats.bytes, 700 &dim_sample); 701 net_dim(&v->rx_dim, dim_sample); 702 } 703 704 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 705 { 706 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 707 708 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 709 } 710 711 static bool enetc_page_reusable(struct page *page) 712 { 713 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 714 } 715 716 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 717 struct enetc_rx_swbd *old) 718 { 719 struct enetc_rx_swbd *new; 720 721 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 722 723 /* next buf that may reuse a page */ 724 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 725 726 /* copy page reference */ 727 *new = *old; 728 } 729 730 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 731 u64 *tstamp) 732 { 733 u32 lo, hi, tstamp_lo; 734 735 lo = enetc_rd_hot(hw, ENETC_SICTR0); 736 hi = enetc_rd_hot(hw, ENETC_SICTR1); 737 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 738 if (lo <= tstamp_lo) 739 hi -= 1; 740 *tstamp = (u64)hi << 32 | tstamp_lo; 741 } 742 743 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 744 { 745 struct skb_shared_hwtstamps shhwtstamps; 746 747 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 748 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 749 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 750 skb_txtime_consumed(skb); 751 skb_tstamp_tx(skb, &shhwtstamps); 752 } 753 } 754 755 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring, 756 struct enetc_tx_swbd *tx_swbd) 757 { 758 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 759 struct enetc_rx_swbd rx_swbd = { 760 .dma = tx_swbd->dma, 761 .page = tx_swbd->page, 762 .page_offset = tx_swbd->page_offset, 763 .dir = tx_swbd->dir, 764 .len = tx_swbd->len, 765 }; 766 struct enetc_bdr *rx_ring; 767 768 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring); 769 770 if (likely(enetc_swbd_unused(rx_ring))) { 771 enetc_reuse_page(rx_ring, &rx_swbd); 772 773 /* sync for use by the device */ 774 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma, 775 rx_swbd.page_offset, 776 ENETC_RXB_DMA_SIZE_XDP, 777 rx_swbd.dir); 778 779 rx_ring->stats.recycles++; 780 } else { 781 /* RX ring is already full, we need to unmap and free the 782 * page, since there's nothing useful we can do with it. 783 */ 784 rx_ring->stats.recycle_failures++; 785 786 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE, 787 rx_swbd.dir); 788 __free_page(rx_swbd.page); 789 } 790 791 rx_ring->xdp.xdp_tx_in_flight--; 792 } 793 794 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 795 { 796 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0; 797 struct net_device *ndev = tx_ring->ndev; 798 struct enetc_ndev_priv *priv = netdev_priv(ndev); 799 struct enetc_tx_swbd *tx_swbd; 800 int i, bds_to_clean; 801 bool do_twostep_tstamp; 802 u64 tstamp = 0; 803 804 i = tx_ring->next_to_clean; 805 tx_swbd = &tx_ring->tx_swbd[i]; 806 807 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 808 809 do_twostep_tstamp = false; 810 811 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 812 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 813 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 814 bool is_eof = tx_swbd->is_eof; 815 816 if (unlikely(tx_swbd->check_wb)) { 817 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 818 819 if (txbd->flags & ENETC_TXBD_FLAGS_W && 820 tx_swbd->do_twostep_tstamp) { 821 enetc_get_tx_tstamp(&priv->si->hw, txbd, 822 &tstamp); 823 do_twostep_tstamp = true; 824 } 825 826 if (tx_swbd->qbv_en && 827 txbd->wb.status & ENETC_TXBD_STATS_WIN) 828 tx_win_drop++; 829 } 830 831 if (tx_swbd->is_xdp_tx) 832 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd); 833 else if (likely(tx_swbd->dma)) 834 enetc_unmap_tx_buff(tx_ring, tx_swbd); 835 836 if (xdp_frame) { 837 xdp_return_frame(xdp_frame); 838 } else if (skb) { 839 if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) { 840 /* Start work to release lock for next one-step 841 * timestamping packet. And send one skb in 842 * tx_skbs queue if has. 843 */ 844 schedule_work(&priv->tx_onestep_tstamp); 845 } else if (unlikely(do_twostep_tstamp)) { 846 enetc_tstamp_tx(skb, tstamp); 847 do_twostep_tstamp = false; 848 } 849 napi_consume_skb(skb, napi_budget); 850 } 851 852 tx_byte_cnt += tx_swbd->len; 853 /* Scrub the swbd here so we don't have to do that 854 * when we reuse it during xmit 855 */ 856 memset(tx_swbd, 0, sizeof(*tx_swbd)); 857 858 bds_to_clean--; 859 tx_swbd++; 860 i++; 861 if (unlikely(i == tx_ring->bd_count)) { 862 i = 0; 863 tx_swbd = tx_ring->tx_swbd; 864 } 865 866 /* BD iteration loop end */ 867 if (is_eof) { 868 tx_frm_cnt++; 869 /* re-arm interrupt source */ 870 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 871 BIT(16 + tx_ring->index)); 872 } 873 874 if (unlikely(!bds_to_clean)) 875 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 876 } 877 878 tx_ring->next_to_clean = i; 879 tx_ring->stats.packets += tx_frm_cnt; 880 tx_ring->stats.bytes += tx_byte_cnt; 881 tx_ring->stats.win_drop += tx_win_drop; 882 883 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 884 __netif_subqueue_stopped(ndev, tx_ring->index) && 885 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 886 netif_wake_subqueue(ndev, tx_ring->index); 887 } 888 889 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 890 } 891 892 static bool enetc_new_page(struct enetc_bdr *rx_ring, 893 struct enetc_rx_swbd *rx_swbd) 894 { 895 bool xdp = !!(rx_ring->xdp.prog); 896 struct page *page; 897 dma_addr_t addr; 898 899 page = dev_alloc_page(); 900 if (unlikely(!page)) 901 return false; 902 903 /* For XDP_TX, we forgo dma_unmap -> dma_map */ 904 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 905 906 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir); 907 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 908 __free_page(page); 909 910 return false; 911 } 912 913 rx_swbd->dma = addr; 914 rx_swbd->page = page; 915 rx_swbd->page_offset = rx_ring->buffer_offset; 916 917 return true; 918 } 919 920 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 921 { 922 struct enetc_rx_swbd *rx_swbd; 923 union enetc_rx_bd *rxbd; 924 int i, j; 925 926 i = rx_ring->next_to_use; 927 rx_swbd = &rx_ring->rx_swbd[i]; 928 rxbd = enetc_rxbd(rx_ring, i); 929 930 for (j = 0; j < buff_cnt; j++) { 931 /* try reuse page */ 932 if (unlikely(!rx_swbd->page)) { 933 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 934 rx_ring->stats.rx_alloc_errs++; 935 break; 936 } 937 } 938 939 /* update RxBD */ 940 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 941 rx_swbd->page_offset); 942 /* clear 'R" as well */ 943 rxbd->r.lstatus = 0; 944 945 enetc_rxbd_next(rx_ring, &rxbd, &i); 946 rx_swbd = &rx_ring->rx_swbd[i]; 947 } 948 949 if (likely(j)) { 950 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 951 rx_ring->next_to_use = i; 952 953 /* update ENETC's consumer index */ 954 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 955 } 956 957 return j; 958 } 959 960 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 961 static void enetc_get_rx_tstamp(struct net_device *ndev, 962 union enetc_rx_bd *rxbd, 963 struct sk_buff *skb) 964 { 965 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 966 struct enetc_ndev_priv *priv = netdev_priv(ndev); 967 struct enetc_hw *hw = &priv->si->hw; 968 u32 lo, hi, tstamp_lo; 969 u64 tstamp; 970 971 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 972 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 973 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 974 rxbd = enetc_rxbd_ext(rxbd); 975 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 976 if (lo <= tstamp_lo) 977 hi -= 1; 978 979 tstamp = (u64)hi << 32 | tstamp_lo; 980 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 981 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 982 } 983 } 984 #endif 985 986 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 987 union enetc_rx_bd *rxbd, struct sk_buff *skb) 988 { 989 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 990 991 /* TODO: hashing */ 992 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 993 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 994 995 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 996 skb->ip_summed = CHECKSUM_COMPLETE; 997 } 998 999 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 1000 __be16 tpid = 0; 1001 1002 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 1003 case 0: 1004 tpid = htons(ETH_P_8021Q); 1005 break; 1006 case 1: 1007 tpid = htons(ETH_P_8021AD); 1008 break; 1009 case 2: 1010 tpid = htons(enetc_port_rd(&priv->si->hw, 1011 ENETC_PCVLANR1)); 1012 break; 1013 case 3: 1014 tpid = htons(enetc_port_rd(&priv->si->hw, 1015 ENETC_PCVLANR2)); 1016 break; 1017 default: 1018 break; 1019 } 1020 1021 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 1022 } 1023 1024 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 1025 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 1026 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 1027 #endif 1028 } 1029 1030 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS, 1031 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL 1032 * mapped buffers. 1033 */ 1034 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 1035 int i, u16 size) 1036 { 1037 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1038 1039 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 1040 rx_swbd->page_offset, 1041 size, rx_swbd->dir); 1042 return rx_swbd; 1043 } 1044 1045 /* Reuse the current page without performing half-page buffer flipping */ 1046 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 1047 struct enetc_rx_swbd *rx_swbd) 1048 { 1049 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset; 1050 1051 enetc_reuse_page(rx_ring, rx_swbd); 1052 1053 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 1054 rx_swbd->page_offset, 1055 buffer_size, rx_swbd->dir); 1056 1057 rx_swbd->page = NULL; 1058 } 1059 1060 /* Reuse the current page by performing half-page buffer flipping */ 1061 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring, 1062 struct enetc_rx_swbd *rx_swbd) 1063 { 1064 if (likely(enetc_page_reusable(rx_swbd->page))) { 1065 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 1066 page_ref_inc(rx_swbd->page); 1067 1068 enetc_put_rx_buff(rx_ring, rx_swbd); 1069 } else { 1070 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1071 rx_swbd->dir); 1072 rx_swbd->page = NULL; 1073 } 1074 } 1075 1076 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 1077 int i, u16 size) 1078 { 1079 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1080 struct sk_buff *skb; 1081 void *ba; 1082 1083 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 1084 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE); 1085 if (unlikely(!skb)) { 1086 rx_ring->stats.rx_alloc_errs++; 1087 return NULL; 1088 } 1089 1090 skb_reserve(skb, rx_ring->buffer_offset); 1091 __skb_put(skb, size); 1092 1093 enetc_flip_rx_buff(rx_ring, rx_swbd); 1094 1095 return skb; 1096 } 1097 1098 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 1099 u16 size, struct sk_buff *skb) 1100 { 1101 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1102 1103 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 1104 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 1105 1106 enetc_flip_rx_buff(rx_ring, rx_swbd); 1107 } 1108 1109 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring, 1110 u32 bd_status, 1111 union enetc_rx_bd **rxbd, int *i) 1112 { 1113 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK)))) 1114 return false; 1115 1116 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1117 enetc_rxbd_next(rx_ring, rxbd, i); 1118 1119 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1120 dma_rmb(); 1121 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1122 1123 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1124 enetc_rxbd_next(rx_ring, rxbd, i); 1125 } 1126 1127 rx_ring->ndev->stats.rx_dropped++; 1128 rx_ring->ndev->stats.rx_errors++; 1129 1130 return true; 1131 } 1132 1133 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring, 1134 u32 bd_status, union enetc_rx_bd **rxbd, 1135 int *i, int *cleaned_cnt, int buffer_size) 1136 { 1137 struct sk_buff *skb; 1138 u16 size; 1139 1140 size = le16_to_cpu((*rxbd)->r.buf_len); 1141 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size); 1142 if (!skb) 1143 return NULL; 1144 1145 enetc_get_offloads(rx_ring, *rxbd, skb); 1146 1147 (*cleaned_cnt)++; 1148 1149 enetc_rxbd_next(rx_ring, rxbd, i); 1150 1151 /* not last BD in frame? */ 1152 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1153 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1154 size = buffer_size; 1155 1156 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1157 dma_rmb(); 1158 size = le16_to_cpu((*rxbd)->r.buf_len); 1159 } 1160 1161 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb); 1162 1163 (*cleaned_cnt)++; 1164 1165 enetc_rxbd_next(rx_ring, rxbd, i); 1166 } 1167 1168 skb_record_rx_queue(skb, rx_ring->index); 1169 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 1170 1171 return skb; 1172 } 1173 1174 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 1175 1176 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 1177 struct napi_struct *napi, int work_limit) 1178 { 1179 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1180 int cleaned_cnt, i; 1181 1182 cleaned_cnt = enetc_bd_unused(rx_ring); 1183 /* next descriptor to process */ 1184 i = rx_ring->next_to_clean; 1185 1186 while (likely(rx_frm_cnt < work_limit)) { 1187 union enetc_rx_bd *rxbd; 1188 struct sk_buff *skb; 1189 u32 bd_status; 1190 1191 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 1192 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 1193 cleaned_cnt); 1194 1195 rxbd = enetc_rxbd(rx_ring, i); 1196 bd_status = le32_to_cpu(rxbd->r.lstatus); 1197 if (!bd_status) 1198 break; 1199 1200 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1201 dma_rmb(); /* for reading other rxbd fields */ 1202 1203 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1204 &rxbd, &i)) 1205 break; 1206 1207 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i, 1208 &cleaned_cnt, ENETC_RXB_DMA_SIZE); 1209 if (!skb) 1210 break; 1211 1212 rx_byte_cnt += skb->len; 1213 rx_frm_cnt++; 1214 1215 napi_gro_receive(napi, skb); 1216 } 1217 1218 rx_ring->next_to_clean = i; 1219 1220 rx_ring->stats.packets += rx_frm_cnt; 1221 rx_ring->stats.bytes += rx_byte_cnt; 1222 1223 return rx_frm_cnt; 1224 } 1225 1226 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i, 1227 struct enetc_tx_swbd *tx_swbd, 1228 int frm_len) 1229 { 1230 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1231 1232 prefetchw(txbd); 1233 1234 enetc_clear_tx_bd(txbd); 1235 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset); 1236 txbd->buf_len = cpu_to_le16(tx_swbd->len); 1237 txbd->frm_len = cpu_to_le16(frm_len); 1238 1239 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd)); 1240 } 1241 1242 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer 1243 * descriptors. 1244 */ 1245 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring, 1246 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd) 1247 { 1248 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr; 1249 int i, k, frm_len = tmp_tx_swbd->len; 1250 1251 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd))) 1252 return false; 1253 1254 while (unlikely(!tmp_tx_swbd->is_eof)) { 1255 tmp_tx_swbd++; 1256 frm_len += tmp_tx_swbd->len; 1257 } 1258 1259 i = tx_ring->next_to_use; 1260 1261 for (k = 0; k < num_tx_swbd; k++) { 1262 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k]; 1263 1264 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len); 1265 1266 /* last BD needs 'F' bit set */ 1267 if (xdp_tx_swbd->is_eof) { 1268 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1269 1270 txbd->flags = ENETC_TXBD_FLAGS_F; 1271 } 1272 1273 enetc_bdr_idx_inc(tx_ring, &i); 1274 } 1275 1276 tx_ring->next_to_use = i; 1277 1278 return true; 1279 } 1280 1281 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring, 1282 struct enetc_tx_swbd *xdp_tx_arr, 1283 struct xdp_frame *xdp_frame) 1284 { 1285 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0]; 1286 struct skb_shared_info *shinfo; 1287 void *data = xdp_frame->data; 1288 int len = xdp_frame->len; 1289 skb_frag_t *frag; 1290 dma_addr_t dma; 1291 unsigned int f; 1292 int n = 0; 1293 1294 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1295 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1296 netdev_err(tx_ring->ndev, "DMA map error\n"); 1297 return -1; 1298 } 1299 1300 xdp_tx_swbd->dma = dma; 1301 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1302 xdp_tx_swbd->len = len; 1303 xdp_tx_swbd->is_xdp_redirect = true; 1304 xdp_tx_swbd->is_eof = false; 1305 xdp_tx_swbd->xdp_frame = NULL; 1306 1307 n++; 1308 xdp_tx_swbd = &xdp_tx_arr[n]; 1309 1310 shinfo = xdp_get_shared_info_from_frame(xdp_frame); 1311 1312 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags; 1313 f++, frag++) { 1314 data = skb_frag_address(frag); 1315 len = skb_frag_size(frag); 1316 1317 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1318 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1319 /* Undo the DMA mapping for all fragments */ 1320 while (--n >= 0) 1321 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]); 1322 1323 netdev_err(tx_ring->ndev, "DMA map error\n"); 1324 return -1; 1325 } 1326 1327 xdp_tx_swbd->dma = dma; 1328 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1329 xdp_tx_swbd->len = len; 1330 xdp_tx_swbd->is_xdp_redirect = true; 1331 xdp_tx_swbd->is_eof = false; 1332 xdp_tx_swbd->xdp_frame = NULL; 1333 1334 n++; 1335 xdp_tx_swbd = &xdp_tx_arr[n]; 1336 } 1337 1338 xdp_tx_arr[n - 1].is_eof = true; 1339 xdp_tx_arr[n - 1].xdp_frame = xdp_frame; 1340 1341 return n; 1342 } 1343 1344 int enetc_xdp_xmit(struct net_device *ndev, int num_frames, 1345 struct xdp_frame **frames, u32 flags) 1346 { 1347 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1348 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1349 struct enetc_bdr *tx_ring; 1350 int xdp_tx_bd_cnt, i, k; 1351 int xdp_tx_frm_cnt = 0; 1352 1353 enetc_lock_mdio(); 1354 1355 tx_ring = priv->xdp_tx_ring[smp_processor_id()]; 1356 1357 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); 1358 1359 for (k = 0; k < num_frames; k++) { 1360 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, 1361 xdp_redirect_arr, 1362 frames[k]); 1363 if (unlikely(xdp_tx_bd_cnt < 0)) 1364 break; 1365 1366 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr, 1367 xdp_tx_bd_cnt))) { 1368 for (i = 0; i < xdp_tx_bd_cnt; i++) 1369 enetc_unmap_tx_buff(tx_ring, 1370 &xdp_redirect_arr[i]); 1371 tx_ring->stats.xdp_tx_drops++; 1372 break; 1373 } 1374 1375 xdp_tx_frm_cnt++; 1376 } 1377 1378 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt)) 1379 enetc_update_tx_ring_tail(tx_ring); 1380 1381 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt; 1382 1383 enetc_unlock_mdio(); 1384 1385 return xdp_tx_frm_cnt; 1386 } 1387 1388 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1389 struct xdp_buff *xdp_buff, u16 size) 1390 { 1391 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1392 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset; 1393 struct skb_shared_info *shinfo; 1394 1395 /* To be used for XDP_TX */ 1396 rx_swbd->len = size; 1397 1398 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset, 1399 rx_ring->buffer_offset, size, false); 1400 1401 shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1402 shinfo->nr_frags = 0; 1403 } 1404 1405 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1406 u16 size, struct xdp_buff *xdp_buff) 1407 { 1408 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1409 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1410 skb_frag_t *frag = &shinfo->frags[shinfo->nr_frags]; 1411 1412 /* To be used for XDP_TX */ 1413 rx_swbd->len = size; 1414 1415 skb_frag_off_set(frag, rx_swbd->page_offset); 1416 skb_frag_size_set(frag, size); 1417 __skb_frag_set_page(frag, rx_swbd->page); 1418 1419 shinfo->nr_frags++; 1420 } 1421 1422 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status, 1423 union enetc_rx_bd **rxbd, int *i, 1424 int *cleaned_cnt, struct xdp_buff *xdp_buff) 1425 { 1426 u16 size = le16_to_cpu((*rxbd)->r.buf_len); 1427 1428 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq); 1429 1430 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size); 1431 (*cleaned_cnt)++; 1432 enetc_rxbd_next(rx_ring, rxbd, i); 1433 1434 /* not last BD in frame? */ 1435 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1436 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1437 size = ENETC_RXB_DMA_SIZE_XDP; 1438 1439 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1440 dma_rmb(); 1441 size = le16_to_cpu((*rxbd)->r.buf_len); 1442 } 1443 1444 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff); 1445 (*cleaned_cnt)++; 1446 enetc_rxbd_next(rx_ring, rxbd, i); 1447 } 1448 } 1449 1450 /* Convert RX buffer descriptors to TX buffer descriptors. These will be 1451 * recycled back into the RX ring in enetc_clean_tx_ring. 1452 */ 1453 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr, 1454 struct enetc_bdr *rx_ring, 1455 int rx_ring_first, int rx_ring_last) 1456 { 1457 int n = 0; 1458 1459 for (; rx_ring_first != rx_ring_last; 1460 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) { 1461 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1462 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n]; 1463 1464 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */ 1465 tx_swbd->dma = rx_swbd->dma; 1466 tx_swbd->dir = rx_swbd->dir; 1467 tx_swbd->page = rx_swbd->page; 1468 tx_swbd->page_offset = rx_swbd->page_offset; 1469 tx_swbd->len = rx_swbd->len; 1470 tx_swbd->is_dma_page = true; 1471 tx_swbd->is_xdp_tx = true; 1472 tx_swbd->is_eof = false; 1473 } 1474 1475 /* We rely on caller providing an rx_ring_last > rx_ring_first */ 1476 xdp_tx_arr[n - 1].is_eof = true; 1477 1478 return n; 1479 } 1480 1481 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, 1482 int rx_ring_last) 1483 { 1484 while (rx_ring_first != rx_ring_last) { 1485 enetc_put_rx_buff(rx_ring, 1486 &rx_ring->rx_swbd[rx_ring_first]); 1487 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1488 } 1489 rx_ring->stats.xdp_drops++; 1490 } 1491 1492 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, 1493 struct napi_struct *napi, int work_limit, 1494 struct bpf_prog *prog) 1495 { 1496 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0; 1497 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1498 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1499 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1500 struct enetc_bdr *tx_ring; 1501 int cleaned_cnt, i; 1502 u32 xdp_act; 1503 1504 cleaned_cnt = enetc_bd_unused(rx_ring); 1505 /* next descriptor to process */ 1506 i = rx_ring->next_to_clean; 1507 1508 while (likely(rx_frm_cnt < work_limit)) { 1509 union enetc_rx_bd *rxbd, *orig_rxbd; 1510 int orig_i, orig_cleaned_cnt; 1511 struct xdp_buff xdp_buff; 1512 struct sk_buff *skb; 1513 u32 bd_status; 1514 int err; 1515 1516 rxbd = enetc_rxbd(rx_ring, i); 1517 bd_status = le32_to_cpu(rxbd->r.lstatus); 1518 if (!bd_status) 1519 break; 1520 1521 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1522 dma_rmb(); /* for reading other rxbd fields */ 1523 1524 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1525 &rxbd, &i)) 1526 break; 1527 1528 orig_rxbd = rxbd; 1529 orig_cleaned_cnt = cleaned_cnt; 1530 orig_i = i; 1531 1532 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i, 1533 &cleaned_cnt, &xdp_buff); 1534 1535 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff); 1536 1537 switch (xdp_act) { 1538 default: 1539 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act); 1540 fallthrough; 1541 case XDP_ABORTED: 1542 trace_xdp_exception(rx_ring->ndev, prog, xdp_act); 1543 fallthrough; 1544 case XDP_DROP: 1545 enetc_xdp_drop(rx_ring, orig_i, i); 1546 break; 1547 case XDP_PASS: 1548 rxbd = orig_rxbd; 1549 cleaned_cnt = orig_cleaned_cnt; 1550 i = orig_i; 1551 1552 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, 1553 &i, &cleaned_cnt, 1554 ENETC_RXB_DMA_SIZE_XDP); 1555 if (unlikely(!skb)) 1556 goto out; 1557 1558 napi_gro_receive(napi, skb); 1559 break; 1560 case XDP_TX: 1561 tx_ring = priv->xdp_tx_ring[rx_ring->index]; 1562 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr, 1563 rx_ring, 1564 orig_i, i); 1565 1566 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) { 1567 enetc_xdp_drop(rx_ring, orig_i, i); 1568 tx_ring->stats.xdp_tx_drops++; 1569 } else { 1570 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt; 1571 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt; 1572 xdp_tx_frm_cnt++; 1573 /* The XDP_TX enqueue was successful, so we 1574 * need to scrub the RX software BDs because 1575 * the ownership of the buffers no longer 1576 * belongs to the RX ring, and we must prevent 1577 * enetc_refill_rx_ring() from reusing 1578 * rx_swbd->page. 1579 */ 1580 while (orig_i != i) { 1581 rx_ring->rx_swbd[orig_i].page = NULL; 1582 enetc_bdr_idx_inc(rx_ring, &orig_i); 1583 } 1584 } 1585 break; 1586 case XDP_REDIRECT: 1587 /* xdp_return_frame does not support S/G in the sense 1588 * that it leaks the fragments (__xdp_return should not 1589 * call page_frag_free only for the initial buffer). 1590 * Until XDP_REDIRECT gains support for S/G let's keep 1591 * the code structure in place, but dead. We drop the 1592 * S/G frames ourselves to avoid memory leaks which 1593 * would otherwise leave the kernel OOM. 1594 */ 1595 if (unlikely(cleaned_cnt - orig_cleaned_cnt != 1)) { 1596 enetc_xdp_drop(rx_ring, orig_i, i); 1597 rx_ring->stats.xdp_redirect_sg++; 1598 break; 1599 } 1600 1601 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); 1602 if (unlikely(err)) { 1603 enetc_xdp_drop(rx_ring, orig_i, i); 1604 rx_ring->stats.xdp_redirect_failures++; 1605 } else { 1606 while (orig_i != i) { 1607 enetc_flip_rx_buff(rx_ring, 1608 &rx_ring->rx_swbd[orig_i]); 1609 enetc_bdr_idx_inc(rx_ring, &orig_i); 1610 } 1611 xdp_redirect_frm_cnt++; 1612 rx_ring->stats.xdp_redirect++; 1613 } 1614 } 1615 1616 rx_frm_cnt++; 1617 } 1618 1619 out: 1620 rx_ring->next_to_clean = i; 1621 1622 rx_ring->stats.packets += rx_frm_cnt; 1623 rx_ring->stats.bytes += rx_byte_cnt; 1624 1625 if (xdp_redirect_frm_cnt) 1626 xdp_do_flush_map(); 1627 1628 if (xdp_tx_frm_cnt) 1629 enetc_update_tx_ring_tail(tx_ring); 1630 1631 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight) 1632 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - 1633 rx_ring->xdp.xdp_tx_in_flight); 1634 1635 return rx_frm_cnt; 1636 } 1637 1638 static int enetc_poll(struct napi_struct *napi, int budget) 1639 { 1640 struct enetc_int_vector 1641 *v = container_of(napi, struct enetc_int_vector, napi); 1642 struct enetc_bdr *rx_ring = &v->rx_ring; 1643 struct bpf_prog *prog; 1644 bool complete = true; 1645 int work_done; 1646 int i; 1647 1648 enetc_lock_mdio(); 1649 1650 for (i = 0; i < v->count_tx_rings; i++) 1651 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 1652 complete = false; 1653 1654 prog = rx_ring->xdp.prog; 1655 if (prog) 1656 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog); 1657 else 1658 work_done = enetc_clean_rx_ring(rx_ring, napi, budget); 1659 if (work_done == budget) 1660 complete = false; 1661 if (work_done) 1662 v->rx_napi_work = true; 1663 1664 if (!complete) { 1665 enetc_unlock_mdio(); 1666 return budget; 1667 } 1668 1669 napi_complete_done(napi, work_done); 1670 1671 if (likely(v->rx_dim_en)) 1672 enetc_rx_net_dim(v); 1673 1674 v->rx_napi_work = false; 1675 1676 /* enable interrupts */ 1677 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 1678 1679 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 1680 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 1681 ENETC_TBIER_TXTIE); 1682 1683 enetc_unlock_mdio(); 1684 1685 return work_done; 1686 } 1687 1688 /* Probing and Init */ 1689 #define ENETC_MAX_RFS_SIZE 64 1690 void enetc_get_si_caps(struct enetc_si *si) 1691 { 1692 struct enetc_hw *hw = &si->hw; 1693 u32 val; 1694 1695 /* find out how many of various resources we have to work with */ 1696 val = enetc_rd(hw, ENETC_SICAPR0); 1697 si->num_rx_rings = (val >> 16) & 0xff; 1698 si->num_tx_rings = val & 0xff; 1699 1700 val = enetc_rd(hw, ENETC_SIRFSCAPR); 1701 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 1702 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 1703 1704 si->num_rss = 0; 1705 val = enetc_rd(hw, ENETC_SIPCAPR0); 1706 if (val & ENETC_SIPCAPR0_RSS) { 1707 u32 rss; 1708 1709 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 1710 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 1711 } 1712 1713 if (val & ENETC_SIPCAPR0_QBV) 1714 si->hw_features |= ENETC_SI_F_QBV; 1715 1716 if (val & ENETC_SIPCAPR0_PSFP) 1717 si->hw_features |= ENETC_SI_F_PSFP; 1718 } 1719 1720 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size) 1721 { 1722 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size, 1723 &r->bd_dma_base, GFP_KERNEL); 1724 if (!r->bd_base) 1725 return -ENOMEM; 1726 1727 /* h/w requires 128B alignment */ 1728 if (!IS_ALIGNED(r->bd_dma_base, 128)) { 1729 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base, 1730 r->bd_dma_base); 1731 return -EINVAL; 1732 } 1733 1734 return 0; 1735 } 1736 1737 static int enetc_alloc_txbdr(struct enetc_bdr *txr) 1738 { 1739 int err; 1740 1741 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd)); 1742 if (!txr->tx_swbd) 1743 return -ENOMEM; 1744 1745 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd)); 1746 if (err) 1747 goto err_alloc_bdr; 1748 1749 txr->tso_headers = dma_alloc_coherent(txr->dev, 1750 txr->bd_count * TSO_HEADER_SIZE, 1751 &txr->tso_headers_dma, 1752 GFP_KERNEL); 1753 if (!txr->tso_headers) { 1754 err = -ENOMEM; 1755 goto err_alloc_tso; 1756 } 1757 1758 txr->next_to_clean = 0; 1759 txr->next_to_use = 0; 1760 1761 return 0; 1762 1763 err_alloc_tso: 1764 dma_free_coherent(txr->dev, txr->bd_count * sizeof(union enetc_tx_bd), 1765 txr->bd_base, txr->bd_dma_base); 1766 txr->bd_base = NULL; 1767 err_alloc_bdr: 1768 vfree(txr->tx_swbd); 1769 txr->tx_swbd = NULL; 1770 1771 return err; 1772 } 1773 1774 static void enetc_free_txbdr(struct enetc_bdr *txr) 1775 { 1776 int size, i; 1777 1778 for (i = 0; i < txr->bd_count; i++) 1779 enetc_free_tx_frame(txr, &txr->tx_swbd[i]); 1780 1781 size = txr->bd_count * sizeof(union enetc_tx_bd); 1782 1783 dma_free_coherent(txr->dev, txr->bd_count * TSO_HEADER_SIZE, 1784 txr->tso_headers, txr->tso_headers_dma); 1785 txr->tso_headers = NULL; 1786 1787 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base); 1788 txr->bd_base = NULL; 1789 1790 vfree(txr->tx_swbd); 1791 txr->tx_swbd = NULL; 1792 } 1793 1794 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 1795 { 1796 int i, err; 1797 1798 for (i = 0; i < priv->num_tx_rings; i++) { 1799 err = enetc_alloc_txbdr(priv->tx_ring[i]); 1800 1801 if (err) 1802 goto fail; 1803 } 1804 1805 return 0; 1806 1807 fail: 1808 while (i-- > 0) 1809 enetc_free_txbdr(priv->tx_ring[i]); 1810 1811 return err; 1812 } 1813 1814 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv) 1815 { 1816 int i; 1817 1818 for (i = 0; i < priv->num_tx_rings; i++) 1819 enetc_free_txbdr(priv->tx_ring[i]); 1820 } 1821 1822 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended) 1823 { 1824 size_t size = sizeof(union enetc_rx_bd); 1825 int err; 1826 1827 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd)); 1828 if (!rxr->rx_swbd) 1829 return -ENOMEM; 1830 1831 if (extended) 1832 size *= 2; 1833 1834 err = enetc_dma_alloc_bdr(rxr, size); 1835 if (err) { 1836 vfree(rxr->rx_swbd); 1837 return err; 1838 } 1839 1840 rxr->next_to_clean = 0; 1841 rxr->next_to_use = 0; 1842 rxr->next_to_alloc = 0; 1843 rxr->ext_en = extended; 1844 1845 return 0; 1846 } 1847 1848 static void enetc_free_rxbdr(struct enetc_bdr *rxr) 1849 { 1850 int size; 1851 1852 size = rxr->bd_count * sizeof(union enetc_rx_bd); 1853 1854 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base); 1855 rxr->bd_base = NULL; 1856 1857 vfree(rxr->rx_swbd); 1858 rxr->rx_swbd = NULL; 1859 } 1860 1861 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv) 1862 { 1863 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 1864 int i, err; 1865 1866 for (i = 0; i < priv->num_rx_rings; i++) { 1867 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended); 1868 1869 if (err) 1870 goto fail; 1871 } 1872 1873 return 0; 1874 1875 fail: 1876 while (i-- > 0) 1877 enetc_free_rxbdr(priv->rx_ring[i]); 1878 1879 return err; 1880 } 1881 1882 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv) 1883 { 1884 int i; 1885 1886 for (i = 0; i < priv->num_rx_rings; i++) 1887 enetc_free_rxbdr(priv->rx_ring[i]); 1888 } 1889 1890 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 1891 { 1892 int i; 1893 1894 if (!tx_ring->tx_swbd) 1895 return; 1896 1897 for (i = 0; i < tx_ring->bd_count; i++) { 1898 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 1899 1900 enetc_free_tx_frame(tx_ring, tx_swbd); 1901 } 1902 1903 tx_ring->next_to_clean = 0; 1904 tx_ring->next_to_use = 0; 1905 } 1906 1907 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 1908 { 1909 int i; 1910 1911 if (!rx_ring->rx_swbd) 1912 return; 1913 1914 for (i = 0; i < rx_ring->bd_count; i++) { 1915 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1916 1917 if (!rx_swbd->page) 1918 continue; 1919 1920 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1921 rx_swbd->dir); 1922 __free_page(rx_swbd->page); 1923 rx_swbd->page = NULL; 1924 } 1925 1926 rx_ring->next_to_clean = 0; 1927 rx_ring->next_to_use = 0; 1928 rx_ring->next_to_alloc = 0; 1929 } 1930 1931 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 1932 { 1933 int i; 1934 1935 for (i = 0; i < priv->num_rx_rings; i++) 1936 enetc_free_rx_ring(priv->rx_ring[i]); 1937 1938 for (i = 0; i < priv->num_tx_rings; i++) 1939 enetc_free_tx_ring(priv->tx_ring[i]); 1940 } 1941 1942 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 1943 { 1944 int *rss_table; 1945 int i; 1946 1947 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 1948 if (!rss_table) 1949 return -ENOMEM; 1950 1951 /* Set up RSS table defaults */ 1952 for (i = 0; i < si->num_rss; i++) 1953 rss_table[i] = i % num_groups; 1954 1955 enetc_set_rss_table(si, rss_table, si->num_rss); 1956 1957 kfree(rss_table); 1958 1959 return 0; 1960 } 1961 1962 int enetc_configure_si(struct enetc_ndev_priv *priv) 1963 { 1964 struct enetc_si *si = priv->si; 1965 struct enetc_hw *hw = &si->hw; 1966 int err; 1967 1968 /* set SI cache attributes */ 1969 enetc_wr(hw, ENETC_SICAR0, 1970 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1971 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 1972 /* enable SI */ 1973 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 1974 1975 if (si->num_rss) { 1976 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 1977 if (err) 1978 return err; 1979 } 1980 1981 return 0; 1982 } 1983 1984 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 1985 { 1986 struct enetc_si *si = priv->si; 1987 int cpus = num_online_cpus(); 1988 1989 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 1990 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 1991 1992 /* Enable all available TX rings in order to configure as many 1993 * priorities as possible, when needed. 1994 * TODO: Make # of TX rings run-time configurable 1995 */ 1996 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 1997 priv->num_tx_rings = si->num_tx_rings; 1998 priv->bdr_int_num = cpus; 1999 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 2000 priv->tx_ictt = ENETC_TXIC_TIMETHR; 2001 } 2002 2003 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 2004 { 2005 struct enetc_si *si = priv->si; 2006 2007 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 2008 GFP_KERNEL); 2009 if (!priv->cls_rules) 2010 return -ENOMEM; 2011 2012 return 0; 2013 } 2014 2015 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 2016 { 2017 kfree(priv->cls_rules); 2018 } 2019 2020 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2021 { 2022 int idx = tx_ring->index; 2023 u32 tbmr; 2024 2025 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 2026 lower_32_bits(tx_ring->bd_dma_base)); 2027 2028 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 2029 upper_32_bits(tx_ring->bd_dma_base)); 2030 2031 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 2032 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 2033 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 2034 2035 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 2036 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 2037 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 2038 2039 /* enable Tx ints by setting pkt thr to 1 */ 2040 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 2041 2042 tbmr = ENETC_TBMR_EN | ENETC_TBMR_SET_PRIO(tx_ring->prio); 2043 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 2044 tbmr |= ENETC_TBMR_VIH; 2045 2046 /* enable ring */ 2047 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2048 2049 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 2050 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 2051 tx_ring->idr = hw->reg + ENETC_SITXIDR; 2052 } 2053 2054 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2055 { 2056 int idx = rx_ring->index; 2057 u32 rbmr; 2058 2059 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 2060 lower_32_bits(rx_ring->bd_dma_base)); 2061 2062 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 2063 upper_32_bits(rx_ring->bd_dma_base)); 2064 2065 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 2066 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 2067 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 2068 2069 if (rx_ring->xdp.prog) 2070 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP); 2071 else 2072 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 2073 2074 /* Also prepare the consumer index in case page allocation never 2075 * succeeds. In that case, hardware will never advance producer index 2076 * to match consumer index, and will drop all frames. 2077 */ 2078 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 2079 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); 2080 2081 /* enable Rx ints by setting pkt thr to 1 */ 2082 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 2083 2084 rbmr = ENETC_RBMR_EN; 2085 2086 if (rx_ring->ext_en) 2087 rbmr |= ENETC_RBMR_BDS; 2088 2089 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 2090 rbmr |= ENETC_RBMR_VTE; 2091 2092 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 2093 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 2094 2095 enetc_lock_mdio(); 2096 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 2097 enetc_unlock_mdio(); 2098 2099 /* enable ring */ 2100 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2101 } 2102 2103 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv) 2104 { 2105 struct enetc_hw *hw = &priv->si->hw; 2106 int i; 2107 2108 for (i = 0; i < priv->num_tx_rings; i++) 2109 enetc_setup_txbdr(hw, priv->tx_ring[i]); 2110 2111 for (i = 0; i < priv->num_rx_rings; i++) 2112 enetc_setup_rxbdr(hw, priv->rx_ring[i]); 2113 } 2114 2115 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2116 { 2117 int idx = rx_ring->index; 2118 2119 /* disable EN bit on ring */ 2120 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 2121 } 2122 2123 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2124 { 2125 int delay = 8, timeout = 100; 2126 int idx = tx_ring->index; 2127 2128 /* disable EN bit on ring */ 2129 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 2130 2131 /* wait for busy to clear */ 2132 while (delay < timeout && 2133 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 2134 msleep(delay); 2135 delay *= 2; 2136 } 2137 2138 if (delay >= timeout) 2139 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 2140 idx); 2141 } 2142 2143 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) 2144 { 2145 struct enetc_hw *hw = &priv->si->hw; 2146 int i; 2147 2148 for (i = 0; i < priv->num_tx_rings; i++) 2149 enetc_clear_txbdr(hw, priv->tx_ring[i]); 2150 2151 for (i = 0; i < priv->num_rx_rings; i++) 2152 enetc_clear_rxbdr(hw, priv->rx_ring[i]); 2153 2154 udelay(1); 2155 } 2156 2157 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 2158 { 2159 struct pci_dev *pdev = priv->si->pdev; 2160 struct enetc_hw *hw = &priv->si->hw; 2161 int i, j, err; 2162 2163 for (i = 0; i < priv->bdr_int_num; i++) { 2164 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2165 struct enetc_int_vector *v = priv->int_vector[i]; 2166 int entry = ENETC_BDR_INT_BASE_IDX + i; 2167 2168 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 2169 priv->ndev->name, i); 2170 err = request_irq(irq, enetc_msix, 0, v->name, v); 2171 if (err) { 2172 dev_err(priv->dev, "request_irq() failed!\n"); 2173 goto irq_err; 2174 } 2175 disable_irq(irq); 2176 2177 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 2178 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 2179 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 2180 2181 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 2182 2183 for (j = 0; j < v->count_tx_rings; j++) { 2184 int idx = v->tx_ring[j].index; 2185 2186 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 2187 } 2188 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 2189 } 2190 2191 return 0; 2192 2193 irq_err: 2194 while (i--) { 2195 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2196 2197 irq_set_affinity_hint(irq, NULL); 2198 free_irq(irq, priv->int_vector[i]); 2199 } 2200 2201 return err; 2202 } 2203 2204 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 2205 { 2206 struct pci_dev *pdev = priv->si->pdev; 2207 int i; 2208 2209 for (i = 0; i < priv->bdr_int_num; i++) { 2210 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2211 2212 irq_set_affinity_hint(irq, NULL); 2213 free_irq(irq, priv->int_vector[i]); 2214 } 2215 } 2216 2217 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 2218 { 2219 struct enetc_hw *hw = &priv->si->hw; 2220 u32 icpt, ictt; 2221 int i; 2222 2223 /* enable Tx & Rx event indication */ 2224 if (priv->ic_mode & 2225 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 2226 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 2227 /* init to non-0 minimum, will be adjusted later */ 2228 ictt = 0x1; 2229 } else { 2230 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 2231 ictt = 0; 2232 } 2233 2234 for (i = 0; i < priv->num_rx_rings; i++) { 2235 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 2236 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 2237 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 2238 } 2239 2240 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 2241 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 2242 else 2243 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 2244 2245 for (i = 0; i < priv->num_tx_rings; i++) { 2246 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 2247 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 2248 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 2249 } 2250 } 2251 2252 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 2253 { 2254 struct enetc_hw *hw = &priv->si->hw; 2255 int i; 2256 2257 for (i = 0; i < priv->num_tx_rings; i++) 2258 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); 2259 2260 for (i = 0; i < priv->num_rx_rings; i++) 2261 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); 2262 } 2263 2264 static int enetc_phylink_connect(struct net_device *ndev) 2265 { 2266 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2267 struct ethtool_eee edata; 2268 int err; 2269 2270 if (!priv->phylink) 2271 return 0; /* phy-less mode */ 2272 2273 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 2274 if (err) { 2275 dev_err(&ndev->dev, "could not attach to PHY\n"); 2276 return err; 2277 } 2278 2279 /* disable EEE autoneg, until ENETC driver supports it */ 2280 memset(&edata, 0, sizeof(struct ethtool_eee)); 2281 phylink_ethtool_set_eee(priv->phylink, &edata); 2282 2283 return 0; 2284 } 2285 2286 static void enetc_tx_onestep_tstamp(struct work_struct *work) 2287 { 2288 struct enetc_ndev_priv *priv; 2289 struct sk_buff *skb; 2290 2291 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp); 2292 2293 netif_tx_lock(priv->ndev); 2294 2295 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags); 2296 skb = skb_dequeue(&priv->tx_skbs); 2297 if (skb) 2298 enetc_start_xmit(skb, priv->ndev); 2299 2300 netif_tx_unlock(priv->ndev); 2301 } 2302 2303 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv) 2304 { 2305 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp); 2306 skb_queue_head_init(&priv->tx_skbs); 2307 } 2308 2309 void enetc_start(struct net_device *ndev) 2310 { 2311 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2312 int i; 2313 2314 enetc_setup_interrupts(priv); 2315 2316 for (i = 0; i < priv->bdr_int_num; i++) { 2317 int irq = pci_irq_vector(priv->si->pdev, 2318 ENETC_BDR_INT_BASE_IDX + i); 2319 2320 napi_enable(&priv->int_vector[i]->napi); 2321 enable_irq(irq); 2322 } 2323 2324 if (priv->phylink) 2325 phylink_start(priv->phylink); 2326 else 2327 netif_carrier_on(ndev); 2328 2329 netif_tx_start_all_queues(ndev); 2330 } 2331 2332 int enetc_open(struct net_device *ndev) 2333 { 2334 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2335 int num_stack_tx_queues; 2336 int err; 2337 2338 err = enetc_setup_irqs(priv); 2339 if (err) 2340 return err; 2341 2342 err = enetc_phylink_connect(ndev); 2343 if (err) 2344 goto err_phy_connect; 2345 2346 err = enetc_alloc_tx_resources(priv); 2347 if (err) 2348 goto err_alloc_tx; 2349 2350 err = enetc_alloc_rx_resources(priv); 2351 if (err) 2352 goto err_alloc_rx; 2353 2354 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2355 2356 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2357 if (err) 2358 goto err_set_queues; 2359 2360 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings); 2361 if (err) 2362 goto err_set_queues; 2363 2364 enetc_tx_onestep_tstamp_init(priv); 2365 enetc_setup_bdrs(priv); 2366 enetc_start(ndev); 2367 2368 return 0; 2369 2370 err_set_queues: 2371 enetc_free_rx_resources(priv); 2372 err_alloc_rx: 2373 enetc_free_tx_resources(priv); 2374 err_alloc_tx: 2375 if (priv->phylink) 2376 phylink_disconnect_phy(priv->phylink); 2377 err_phy_connect: 2378 enetc_free_irqs(priv); 2379 2380 return err; 2381 } 2382 2383 void enetc_stop(struct net_device *ndev) 2384 { 2385 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2386 int i; 2387 2388 netif_tx_stop_all_queues(ndev); 2389 2390 for (i = 0; i < priv->bdr_int_num; i++) { 2391 int irq = pci_irq_vector(priv->si->pdev, 2392 ENETC_BDR_INT_BASE_IDX + i); 2393 2394 disable_irq(irq); 2395 napi_synchronize(&priv->int_vector[i]->napi); 2396 napi_disable(&priv->int_vector[i]->napi); 2397 } 2398 2399 if (priv->phylink) 2400 phylink_stop(priv->phylink); 2401 else 2402 netif_carrier_off(ndev); 2403 2404 enetc_clear_interrupts(priv); 2405 } 2406 2407 int enetc_close(struct net_device *ndev) 2408 { 2409 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2410 2411 enetc_stop(ndev); 2412 enetc_clear_bdrs(priv); 2413 2414 if (priv->phylink) 2415 phylink_disconnect_phy(priv->phylink); 2416 enetc_free_rxtx_rings(priv); 2417 enetc_free_rx_resources(priv); 2418 enetc_free_tx_resources(priv); 2419 enetc_free_irqs(priv); 2420 2421 return 0; 2422 } 2423 2424 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 2425 { 2426 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2427 struct tc_mqprio_qopt *mqprio = type_data; 2428 struct enetc_hw *hw = &priv->si->hw; 2429 struct enetc_bdr *tx_ring; 2430 int num_stack_tx_queues; 2431 u8 num_tc; 2432 int i; 2433 2434 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2435 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 2436 num_tc = mqprio->num_tc; 2437 2438 if (!num_tc) { 2439 netdev_reset_tc(ndev); 2440 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2441 2442 /* Reset all ring priorities to 0 */ 2443 for (i = 0; i < priv->num_tx_rings; i++) { 2444 tx_ring = priv->tx_ring[i]; 2445 tx_ring->prio = 0; 2446 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 2447 } 2448 2449 return 0; 2450 } 2451 2452 /* Check if we have enough BD rings available to accommodate all TCs */ 2453 if (num_tc > num_stack_tx_queues) { 2454 netdev_err(ndev, "Max %d traffic classes supported\n", 2455 priv->num_tx_rings); 2456 return -EINVAL; 2457 } 2458 2459 /* For the moment, we use only one BD ring per TC. 2460 * 2461 * Configure num_tc BD rings with increasing priorities. 2462 */ 2463 for (i = 0; i < num_tc; i++) { 2464 tx_ring = priv->tx_ring[i]; 2465 tx_ring->prio = i; 2466 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 2467 } 2468 2469 /* Reset the number of netdev queues based on the TC count */ 2470 netif_set_real_num_tx_queues(ndev, num_tc); 2471 2472 netdev_set_num_tc(ndev, num_tc); 2473 2474 /* Each TC is associated with one netdev queue */ 2475 for (i = 0; i < num_tc; i++) 2476 netdev_set_tc_queue(ndev, i, 1, i); 2477 2478 return 0; 2479 } 2480 2481 static int enetc_setup_xdp_prog(struct net_device *dev, struct bpf_prog *prog, 2482 struct netlink_ext_ack *extack) 2483 { 2484 struct enetc_ndev_priv *priv = netdev_priv(dev); 2485 struct bpf_prog *old_prog; 2486 bool is_up; 2487 int i; 2488 2489 /* The buffer layout is changing, so we need to drain the old 2490 * RX buffers and seed new ones. 2491 */ 2492 is_up = netif_running(dev); 2493 if (is_up) 2494 dev_close(dev); 2495 2496 old_prog = xchg(&priv->xdp_prog, prog); 2497 if (old_prog) 2498 bpf_prog_put(old_prog); 2499 2500 for (i = 0; i < priv->num_rx_rings; i++) { 2501 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 2502 2503 rx_ring->xdp.prog = prog; 2504 2505 if (prog) 2506 rx_ring->buffer_offset = XDP_PACKET_HEADROOM; 2507 else 2508 rx_ring->buffer_offset = ENETC_RXB_PAD; 2509 } 2510 2511 if (is_up) 2512 return dev_open(dev, extack); 2513 2514 return 0; 2515 } 2516 2517 int enetc_setup_bpf(struct net_device *dev, struct netdev_bpf *xdp) 2518 { 2519 switch (xdp->command) { 2520 case XDP_SETUP_PROG: 2521 return enetc_setup_xdp_prog(dev, xdp->prog, xdp->extack); 2522 default: 2523 return -EINVAL; 2524 } 2525 2526 return 0; 2527 } 2528 2529 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 2530 { 2531 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2532 struct net_device_stats *stats = &ndev->stats; 2533 unsigned long packets = 0, bytes = 0; 2534 unsigned long tx_dropped = 0; 2535 int i; 2536 2537 for (i = 0; i < priv->num_rx_rings; i++) { 2538 packets += priv->rx_ring[i]->stats.packets; 2539 bytes += priv->rx_ring[i]->stats.bytes; 2540 } 2541 2542 stats->rx_packets = packets; 2543 stats->rx_bytes = bytes; 2544 bytes = 0; 2545 packets = 0; 2546 2547 for (i = 0; i < priv->num_tx_rings; i++) { 2548 packets += priv->tx_ring[i]->stats.packets; 2549 bytes += priv->tx_ring[i]->stats.bytes; 2550 tx_dropped += priv->tx_ring[i]->stats.win_drop; 2551 } 2552 2553 stats->tx_packets = packets; 2554 stats->tx_bytes = bytes; 2555 stats->tx_dropped = tx_dropped; 2556 2557 return stats; 2558 } 2559 2560 static int enetc_set_rss(struct net_device *ndev, int en) 2561 { 2562 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2563 struct enetc_hw *hw = &priv->si->hw; 2564 u32 reg; 2565 2566 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 2567 2568 reg = enetc_rd(hw, ENETC_SIMR); 2569 reg &= ~ENETC_SIMR_RSSE; 2570 reg |= (en) ? ENETC_SIMR_RSSE : 0; 2571 enetc_wr(hw, ENETC_SIMR, reg); 2572 2573 return 0; 2574 } 2575 2576 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 2577 { 2578 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2579 struct enetc_hw *hw = &priv->si->hw; 2580 int i; 2581 2582 for (i = 0; i < priv->num_rx_rings; i++) 2583 enetc_bdr_enable_rxvlan(hw, i, en); 2584 } 2585 2586 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 2587 { 2588 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2589 struct enetc_hw *hw = &priv->si->hw; 2590 int i; 2591 2592 for (i = 0; i < priv->num_tx_rings; i++) 2593 enetc_bdr_enable_txvlan(hw, i, en); 2594 } 2595 2596 void enetc_set_features(struct net_device *ndev, netdev_features_t features) 2597 { 2598 netdev_features_t changed = ndev->features ^ features; 2599 2600 if (changed & NETIF_F_RXHASH) 2601 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 2602 2603 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 2604 enetc_enable_rxvlan(ndev, 2605 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 2606 2607 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 2608 enetc_enable_txvlan(ndev, 2609 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 2610 } 2611 2612 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 2613 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 2614 { 2615 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2616 struct hwtstamp_config config; 2617 int ao; 2618 2619 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 2620 return -EFAULT; 2621 2622 switch (config.tx_type) { 2623 case HWTSTAMP_TX_OFF: 2624 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2625 break; 2626 case HWTSTAMP_TX_ON: 2627 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2628 priv->active_offloads |= ENETC_F_TX_TSTAMP; 2629 break; 2630 case HWTSTAMP_TX_ONESTEP_SYNC: 2631 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2632 priv->active_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP; 2633 break; 2634 default: 2635 return -ERANGE; 2636 } 2637 2638 ao = priv->active_offloads; 2639 switch (config.rx_filter) { 2640 case HWTSTAMP_FILTER_NONE: 2641 priv->active_offloads &= ~ENETC_F_RX_TSTAMP; 2642 break; 2643 default: 2644 priv->active_offloads |= ENETC_F_RX_TSTAMP; 2645 config.rx_filter = HWTSTAMP_FILTER_ALL; 2646 } 2647 2648 if (netif_running(ndev) && ao != priv->active_offloads) { 2649 enetc_close(ndev); 2650 enetc_open(ndev); 2651 } 2652 2653 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2654 -EFAULT : 0; 2655 } 2656 2657 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 2658 { 2659 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2660 struct hwtstamp_config config; 2661 2662 config.flags = 0; 2663 2664 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) 2665 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC; 2666 else if (priv->active_offloads & ENETC_F_TX_TSTAMP) 2667 config.tx_type = HWTSTAMP_TX_ON; 2668 else 2669 config.tx_type = HWTSTAMP_TX_OFF; 2670 2671 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 2672 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 2673 2674 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2675 -EFAULT : 0; 2676 } 2677 #endif 2678 2679 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 2680 { 2681 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2682 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK 2683 if (cmd == SIOCSHWTSTAMP) 2684 return enetc_hwtstamp_set(ndev, rq); 2685 if (cmd == SIOCGHWTSTAMP) 2686 return enetc_hwtstamp_get(ndev, rq); 2687 #endif 2688 2689 if (!priv->phylink) 2690 return -EOPNOTSUPP; 2691 2692 return phylink_mii_ioctl(priv->phylink, rq, cmd); 2693 } 2694 2695 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 2696 { 2697 struct pci_dev *pdev = priv->si->pdev; 2698 int first_xdp_tx_ring; 2699 int i, n, err, nvec; 2700 int v_tx_rings; 2701 2702 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 2703 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 2704 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 2705 2706 if (n < 0) 2707 return n; 2708 2709 if (n != nvec) 2710 return -EPERM; 2711 2712 /* # of tx rings per int vector */ 2713 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 2714 2715 for (i = 0; i < priv->bdr_int_num; i++) { 2716 struct enetc_int_vector *v; 2717 struct enetc_bdr *bdr; 2718 int j; 2719 2720 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 2721 if (!v) { 2722 err = -ENOMEM; 2723 goto fail; 2724 } 2725 2726 priv->int_vector[i] = v; 2727 2728 bdr = &v->rx_ring; 2729 bdr->index = i; 2730 bdr->ndev = priv->ndev; 2731 bdr->dev = priv->dev; 2732 bdr->bd_count = priv->rx_bd_count; 2733 bdr->buffer_offset = ENETC_RXB_PAD; 2734 priv->rx_ring[i] = bdr; 2735 2736 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0); 2737 if (err) { 2738 kfree(v); 2739 goto fail; 2740 } 2741 2742 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, 2743 MEM_TYPE_PAGE_SHARED, NULL); 2744 if (err) { 2745 xdp_rxq_info_unreg(&bdr->xdp.rxq); 2746 kfree(v); 2747 goto fail; 2748 } 2749 2750 /* init defaults for adaptive IC */ 2751 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 2752 v->rx_ictt = 0x1; 2753 v->rx_dim_en = true; 2754 } 2755 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 2756 netif_napi_add(priv->ndev, &v->napi, enetc_poll); 2757 v->count_tx_rings = v_tx_rings; 2758 2759 for (j = 0; j < v_tx_rings; j++) { 2760 int idx; 2761 2762 /* default tx ring mapping policy */ 2763 idx = priv->bdr_int_num * j + i; 2764 __set_bit(idx, &v->tx_rings_map); 2765 bdr = &v->tx_ring[j]; 2766 bdr->index = idx; 2767 bdr->ndev = priv->ndev; 2768 bdr->dev = priv->dev; 2769 bdr->bd_count = priv->tx_bd_count; 2770 priv->tx_ring[idx] = bdr; 2771 } 2772 } 2773 2774 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus(); 2775 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring]; 2776 2777 return 0; 2778 2779 fail: 2780 while (i--) { 2781 struct enetc_int_vector *v = priv->int_vector[i]; 2782 struct enetc_bdr *rx_ring = &v->rx_ring; 2783 2784 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 2785 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 2786 netif_napi_del(&v->napi); 2787 cancel_work_sync(&v->rx_dim.work); 2788 kfree(v); 2789 } 2790 2791 pci_free_irq_vectors(pdev); 2792 2793 return err; 2794 } 2795 2796 void enetc_free_msix(struct enetc_ndev_priv *priv) 2797 { 2798 int i; 2799 2800 for (i = 0; i < priv->bdr_int_num; i++) { 2801 struct enetc_int_vector *v = priv->int_vector[i]; 2802 struct enetc_bdr *rx_ring = &v->rx_ring; 2803 2804 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 2805 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 2806 netif_napi_del(&v->napi); 2807 cancel_work_sync(&v->rx_dim.work); 2808 } 2809 2810 for (i = 0; i < priv->num_rx_rings; i++) 2811 priv->rx_ring[i] = NULL; 2812 2813 for (i = 0; i < priv->num_tx_rings; i++) 2814 priv->tx_ring[i] = NULL; 2815 2816 for (i = 0; i < priv->bdr_int_num; i++) { 2817 kfree(priv->int_vector[i]); 2818 priv->int_vector[i] = NULL; 2819 } 2820 2821 /* disable all MSIX for this device */ 2822 pci_free_irq_vectors(priv->si->pdev); 2823 } 2824 2825 static void enetc_kfree_si(struct enetc_si *si) 2826 { 2827 char *p = (char *)si - si->pad; 2828 2829 kfree(p); 2830 } 2831 2832 static void enetc_detect_errata(struct enetc_si *si) 2833 { 2834 if (si->pdev->revision == ENETC_REV1) 2835 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 2836 } 2837 2838 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 2839 { 2840 struct enetc_si *si, *p; 2841 struct enetc_hw *hw; 2842 size_t alloc_size; 2843 int err, len; 2844 2845 pcie_flr(pdev); 2846 err = pci_enable_device_mem(pdev); 2847 if (err) 2848 return dev_err_probe(&pdev->dev, err, "device enable failed\n"); 2849 2850 /* set up for high or low dma */ 2851 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 2852 if (err) { 2853 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err); 2854 goto err_dma; 2855 } 2856 2857 err = pci_request_mem_regions(pdev, name); 2858 if (err) { 2859 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 2860 goto err_pci_mem_reg; 2861 } 2862 2863 pci_set_master(pdev); 2864 2865 alloc_size = sizeof(struct enetc_si); 2866 if (sizeof_priv) { 2867 /* align priv to 32B */ 2868 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 2869 alloc_size += sizeof_priv; 2870 } 2871 /* force 32B alignment for enetc_si */ 2872 alloc_size += ENETC_SI_ALIGN - 1; 2873 2874 p = kzalloc(alloc_size, GFP_KERNEL); 2875 if (!p) { 2876 err = -ENOMEM; 2877 goto err_alloc_si; 2878 } 2879 2880 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 2881 si->pad = (char *)si - (char *)p; 2882 2883 pci_set_drvdata(pdev, si); 2884 si->pdev = pdev; 2885 hw = &si->hw; 2886 2887 len = pci_resource_len(pdev, ENETC_BAR_REGS); 2888 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 2889 if (!hw->reg) { 2890 err = -ENXIO; 2891 dev_err(&pdev->dev, "ioremap() failed\n"); 2892 goto err_ioremap; 2893 } 2894 if (len > ENETC_PORT_BASE) 2895 hw->port = hw->reg + ENETC_PORT_BASE; 2896 if (len > ENETC_GLOBAL_BASE) 2897 hw->global = hw->reg + ENETC_GLOBAL_BASE; 2898 2899 enetc_detect_errata(si); 2900 2901 return 0; 2902 2903 err_ioremap: 2904 enetc_kfree_si(si); 2905 err_alloc_si: 2906 pci_release_mem_regions(pdev); 2907 err_pci_mem_reg: 2908 err_dma: 2909 pci_disable_device(pdev); 2910 2911 return err; 2912 } 2913 2914 void enetc_pci_remove(struct pci_dev *pdev) 2915 { 2916 struct enetc_si *si = pci_get_drvdata(pdev); 2917 struct enetc_hw *hw = &si->hw; 2918 2919 iounmap(hw->reg); 2920 enetc_kfree_si(si); 2921 pci_release_mem_regions(pdev); 2922 pci_disable_device(pdev); 2923 } 2924