1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2015 Jakub Kicinski <kubakici@wp.pl> 4 */ 5 6 #include "mt7601u.h" 7 #include "dma.h" 8 #include "usb.h" 9 #include "trace.h" 10 11 static int mt7601u_submit_rx_buf(struct mt7601u_dev *dev, 12 struct mt7601u_dma_buf_rx *e, gfp_t gfp); 13 14 static unsigned int ieee80211_get_hdrlen_from_buf(const u8 *data, unsigned len) 15 { 16 const struct ieee80211_hdr *hdr = (const struct ieee80211_hdr *)data; 17 unsigned int hdrlen; 18 19 if (unlikely(len < 10)) 20 return 0; 21 hdrlen = ieee80211_hdrlen(hdr->frame_control); 22 if (unlikely(hdrlen > len)) 23 return 0; 24 return hdrlen; 25 } 26 27 static struct sk_buff * 28 mt7601u_rx_skb_from_seg(struct mt7601u_dev *dev, struct mt7601u_rxwi *rxwi, 29 void *data, u32 seg_len, u32 truesize, struct page *p) 30 { 31 struct sk_buff *skb; 32 u32 true_len, hdr_len = 0, copy, frag; 33 34 skb = alloc_skb(p ? 128 : seg_len, GFP_ATOMIC); 35 if (!skb) 36 return NULL; 37 38 true_len = mt76_mac_process_rx(dev, skb, data, rxwi); 39 if (!true_len || true_len > seg_len) 40 goto bad_frame; 41 42 hdr_len = ieee80211_get_hdrlen_from_buf(data, true_len); 43 if (!hdr_len) 44 goto bad_frame; 45 46 if (rxwi->rxinfo & cpu_to_le32(MT_RXINFO_L2PAD)) { 47 skb_put_data(skb, data, hdr_len); 48 49 data += hdr_len + 2; 50 true_len -= hdr_len; 51 hdr_len = 0; 52 } 53 54 /* If not doing paged RX allocated skb will always have enough space */ 55 copy = (true_len <= skb_tailroom(skb)) ? true_len : hdr_len + 8; 56 frag = true_len - copy; 57 58 skb_put_data(skb, data, copy); 59 data += copy; 60 61 if (frag) { 62 skb_add_rx_frag(skb, 0, p, data - page_address(p), 63 frag, truesize); 64 get_page(p); 65 } 66 67 return skb; 68 69 bad_frame: 70 dev_err_ratelimited(dev->dev, "Error: incorrect frame len:%u hdr:%u\n", 71 true_len, hdr_len); 72 dev_kfree_skb(skb); 73 return NULL; 74 } 75 76 static void mt7601u_rx_process_seg(struct mt7601u_dev *dev, u8 *data, 77 u32 seg_len, struct page *p, 78 struct list_head *list) 79 { 80 struct sk_buff *skb; 81 struct mt7601u_rxwi *rxwi; 82 u32 fce_info, truesize = seg_len; 83 84 /* DMA_INFO field at the beginning of the segment contains only some of 85 * the information, we need to read the FCE descriptor from the end. 86 */ 87 fce_info = get_unaligned_le32(data + seg_len - MT_FCE_INFO_LEN); 88 seg_len -= MT_FCE_INFO_LEN; 89 90 data += MT_DMA_HDR_LEN; 91 seg_len -= MT_DMA_HDR_LEN; 92 93 rxwi = (struct mt7601u_rxwi *) data; 94 data += sizeof(struct mt7601u_rxwi); 95 seg_len -= sizeof(struct mt7601u_rxwi); 96 97 if (unlikely(rxwi->zero[0] || rxwi->zero[1] || rxwi->zero[2])) 98 dev_err_once(dev->dev, "Error: RXWI zero fields are set\n"); 99 if (unlikely(FIELD_GET(MT_RXD_INFO_TYPE, fce_info))) 100 dev_err_once(dev->dev, "Error: RX path seen a non-pkt urb\n"); 101 102 trace_mt_rx(dev, rxwi, fce_info); 103 104 skb = mt7601u_rx_skb_from_seg(dev, rxwi, data, seg_len, truesize, p); 105 if (!skb) 106 return; 107 108 local_bh_disable(); 109 rcu_read_lock(); 110 111 ieee80211_rx_list(dev->hw, NULL, skb, list); 112 113 rcu_read_unlock(); 114 local_bh_enable(); 115 } 116 117 static u16 mt7601u_rx_next_seg_len(u8 *data, u32 data_len) 118 { 119 u32 min_seg_len = MT_DMA_HDR_LEN + MT_RX_INFO_LEN + 120 sizeof(struct mt7601u_rxwi) + MT_FCE_INFO_LEN; 121 u16 dma_len = get_unaligned_le16(data); 122 123 if (data_len < min_seg_len || 124 WARN_ON_ONCE(!dma_len) || 125 WARN_ON_ONCE(dma_len + MT_DMA_HDRS > data_len) || 126 WARN_ON_ONCE(dma_len & 0x3) || 127 WARN_ON_ONCE(dma_len < min_seg_len)) 128 return 0; 129 130 return MT_DMA_HDRS + dma_len; 131 } 132 133 static void 134 mt7601u_rx_process_entry(struct mt7601u_dev *dev, struct mt7601u_dma_buf_rx *e) 135 { 136 u32 seg_len, data_len = e->urb->actual_length; 137 u8 *data = page_address(e->p); 138 struct page *new_p = NULL; 139 LIST_HEAD(list); 140 int cnt = 0; 141 142 if (!test_bit(MT7601U_STATE_INITIALIZED, &dev->state)) 143 return; 144 145 /* Copy if there is very little data in the buffer. */ 146 if (data_len > 512) 147 new_p = dev_alloc_pages(MT_RX_ORDER); 148 149 while ((seg_len = mt7601u_rx_next_seg_len(data, data_len))) { 150 mt7601u_rx_process_seg(dev, data, seg_len, 151 new_p ? e->p : NULL, &list); 152 153 data_len -= seg_len; 154 data += seg_len; 155 cnt++; 156 } 157 158 if (cnt > 1) 159 trace_mt_rx_dma_aggr(dev, cnt, !!new_p); 160 161 netif_receive_skb_list(&list); 162 163 if (new_p) { 164 /* we have one extra ref from the allocator */ 165 put_page(e->p); 166 e->p = new_p; 167 } 168 } 169 170 static struct mt7601u_dma_buf_rx * 171 mt7601u_rx_get_pending_entry(struct mt7601u_dev *dev) 172 { 173 struct mt7601u_rx_queue *q = &dev->rx_q; 174 struct mt7601u_dma_buf_rx *buf = NULL; 175 unsigned long flags; 176 177 spin_lock_irqsave(&dev->rx_lock, flags); 178 179 if (!q->pending) 180 goto out; 181 182 buf = &q->e[q->start]; 183 q->pending--; 184 q->start = (q->start + 1) % q->entries; 185 out: 186 spin_unlock_irqrestore(&dev->rx_lock, flags); 187 188 return buf; 189 } 190 191 static void mt7601u_complete_rx(struct urb *urb) 192 { 193 struct mt7601u_dev *dev = urb->context; 194 struct mt7601u_rx_queue *q = &dev->rx_q; 195 unsigned long flags; 196 197 /* do no schedule rx tasklet if urb has been unlinked 198 * or the device has been removed 199 */ 200 switch (urb->status) { 201 case -ECONNRESET: 202 case -ESHUTDOWN: 203 case -ENOENT: 204 case -EPROTO: 205 return; 206 default: 207 dev_err_ratelimited(dev->dev, "rx urb failed: %d\n", 208 urb->status); 209 fallthrough; 210 case 0: 211 break; 212 } 213 214 spin_lock_irqsave(&dev->rx_lock, flags); 215 if (WARN_ONCE(q->e[q->end].urb != urb, "RX urb mismatch")) 216 goto out; 217 218 q->end = (q->end + 1) % q->entries; 219 q->pending++; 220 tasklet_schedule(&dev->rx_tasklet); 221 out: 222 spin_unlock_irqrestore(&dev->rx_lock, flags); 223 } 224 225 static void mt7601u_rx_tasklet(struct tasklet_struct *t) 226 { 227 struct mt7601u_dev *dev = from_tasklet(dev, t, rx_tasklet); 228 struct mt7601u_dma_buf_rx *e; 229 230 while ((e = mt7601u_rx_get_pending_entry(dev))) { 231 if (e->urb->status) 232 continue; 233 234 mt7601u_rx_process_entry(dev, e); 235 mt7601u_submit_rx_buf(dev, e, GFP_ATOMIC); 236 } 237 } 238 239 static void mt7601u_complete_tx(struct urb *urb) 240 { 241 struct mt7601u_tx_queue *q = urb->context; 242 struct mt7601u_dev *dev = q->dev; 243 struct sk_buff *skb; 244 unsigned long flags; 245 246 switch (urb->status) { 247 case -ECONNRESET: 248 case -ESHUTDOWN: 249 case -ENOENT: 250 case -EPROTO: 251 return; 252 default: 253 dev_err_ratelimited(dev->dev, "tx urb failed: %d\n", 254 urb->status); 255 fallthrough; 256 case 0: 257 break; 258 } 259 260 spin_lock_irqsave(&dev->tx_lock, flags); 261 if (WARN_ONCE(q->e[q->start].urb != urb, "TX urb mismatch")) 262 goto out; 263 264 skb = q->e[q->start].skb; 265 q->e[q->start].skb = NULL; 266 trace_mt_tx_dma_done(dev, skb); 267 268 __skb_queue_tail(&dev->tx_skb_done, skb); 269 tasklet_schedule(&dev->tx_tasklet); 270 271 if (q->used == q->entries - q->entries / 8) 272 ieee80211_wake_queue(dev->hw, skb_get_queue_mapping(skb)); 273 274 q->start = (q->start + 1) % q->entries; 275 q->used--; 276 out: 277 spin_unlock_irqrestore(&dev->tx_lock, flags); 278 } 279 280 static void mt7601u_tx_tasklet(struct tasklet_struct *t) 281 { 282 struct mt7601u_dev *dev = from_tasklet(dev, t, tx_tasklet); 283 struct sk_buff_head skbs; 284 unsigned long flags; 285 286 __skb_queue_head_init(&skbs); 287 288 spin_lock_irqsave(&dev->tx_lock, flags); 289 290 set_bit(MT7601U_STATE_MORE_STATS, &dev->state); 291 if (!test_and_set_bit(MT7601U_STATE_READING_STATS, &dev->state)) 292 queue_delayed_work(dev->stat_wq, &dev->stat_work, 293 msecs_to_jiffies(10)); 294 295 skb_queue_splice_init(&dev->tx_skb_done, &skbs); 296 297 spin_unlock_irqrestore(&dev->tx_lock, flags); 298 299 while (!skb_queue_empty(&skbs)) { 300 struct sk_buff *skb = __skb_dequeue(&skbs); 301 302 mt7601u_tx_status(dev, skb); 303 } 304 } 305 306 static int mt7601u_dma_submit_tx(struct mt7601u_dev *dev, 307 struct sk_buff *skb, u8 ep) 308 { 309 struct usb_device *usb_dev = mt7601u_to_usb_dev(dev); 310 unsigned snd_pipe = usb_sndbulkpipe(usb_dev, dev->out_eps[ep]); 311 struct mt7601u_dma_buf_tx *e; 312 struct mt7601u_tx_queue *q = &dev->tx_q[ep]; 313 unsigned long flags; 314 int ret; 315 316 spin_lock_irqsave(&dev->tx_lock, flags); 317 318 if (WARN_ON(q->entries <= q->used)) { 319 ret = -ENOSPC; 320 goto out; 321 } 322 323 e = &q->e[q->end]; 324 usb_fill_bulk_urb(e->urb, usb_dev, snd_pipe, skb->data, skb->len, 325 mt7601u_complete_tx, q); 326 ret = usb_submit_urb(e->urb, GFP_ATOMIC); 327 if (ret) { 328 /* Special-handle ENODEV from TX urb submission because it will 329 * often be the first ENODEV we see after device is removed. 330 */ 331 if (ret == -ENODEV) 332 set_bit(MT7601U_STATE_REMOVED, &dev->state); 333 else 334 dev_err(dev->dev, "Error: TX urb submit failed:%d\n", 335 ret); 336 goto out; 337 } 338 339 q->end = (q->end + 1) % q->entries; 340 q->used++; 341 e->skb = skb; 342 343 if (q->used >= q->entries) 344 ieee80211_stop_queue(dev->hw, skb_get_queue_mapping(skb)); 345 out: 346 spin_unlock_irqrestore(&dev->tx_lock, flags); 347 348 return ret; 349 } 350 351 /* Map hardware Q to USB endpoint number */ 352 static u8 q2ep(u8 qid) 353 { 354 /* TODO: take management packets to queue 5 */ 355 return qid + 1; 356 } 357 358 /* Map USB endpoint number to Q id in the DMA engine */ 359 static enum mt76_qsel ep2dmaq(u8 ep) 360 { 361 if (ep == 5) 362 return MT_QSEL_MGMT; 363 return MT_QSEL_EDCA; 364 } 365 366 int mt7601u_dma_enqueue_tx(struct mt7601u_dev *dev, struct sk_buff *skb, 367 struct mt76_wcid *wcid, int hw_q) 368 { 369 u8 ep = q2ep(hw_q); 370 u32 dma_flags; 371 int ret; 372 373 dma_flags = MT_TXD_PKT_INFO_80211; 374 if (wcid->hw_key_idx == 0xff) 375 dma_flags |= MT_TXD_PKT_INFO_WIV; 376 377 ret = mt7601u_dma_skb_wrap_pkt(skb, ep2dmaq(ep), dma_flags); 378 if (ret) 379 return ret; 380 381 ret = mt7601u_dma_submit_tx(dev, skb, ep); 382 if (ret) { 383 ieee80211_free_txskb(dev->hw, skb); 384 return ret; 385 } 386 387 return 0; 388 } 389 390 static void mt7601u_kill_rx(struct mt7601u_dev *dev) 391 { 392 int i; 393 394 for (i = 0; i < dev->rx_q.entries; i++) 395 usb_poison_urb(dev->rx_q.e[i].urb); 396 } 397 398 static int mt7601u_submit_rx_buf(struct mt7601u_dev *dev, 399 struct mt7601u_dma_buf_rx *e, gfp_t gfp) 400 { 401 struct usb_device *usb_dev = mt7601u_to_usb_dev(dev); 402 u8 *buf = page_address(e->p); 403 unsigned pipe; 404 int ret; 405 406 pipe = usb_rcvbulkpipe(usb_dev, dev->in_eps[MT_EP_IN_PKT_RX]); 407 408 usb_fill_bulk_urb(e->urb, usb_dev, pipe, buf, MT_RX_URB_SIZE, 409 mt7601u_complete_rx, dev); 410 411 trace_mt_submit_urb(dev, e->urb); 412 ret = usb_submit_urb(e->urb, gfp); 413 if (ret) 414 dev_err(dev->dev, "Error: submit RX URB failed:%d\n", ret); 415 416 return ret; 417 } 418 419 static int mt7601u_submit_rx(struct mt7601u_dev *dev) 420 { 421 int i, ret; 422 423 for (i = 0; i < dev->rx_q.entries; i++) { 424 ret = mt7601u_submit_rx_buf(dev, &dev->rx_q.e[i], GFP_KERNEL); 425 if (ret) 426 return ret; 427 } 428 429 return 0; 430 } 431 432 static void mt7601u_free_rx(struct mt7601u_dev *dev) 433 { 434 int i; 435 436 for (i = 0; i < dev->rx_q.entries; i++) { 437 __free_pages(dev->rx_q.e[i].p, MT_RX_ORDER); 438 usb_free_urb(dev->rx_q.e[i].urb); 439 } 440 } 441 442 static int mt7601u_alloc_rx(struct mt7601u_dev *dev) 443 { 444 int i; 445 446 memset(&dev->rx_q, 0, sizeof(dev->rx_q)); 447 dev->rx_q.dev = dev; 448 dev->rx_q.entries = N_RX_ENTRIES; 449 450 for (i = 0; i < N_RX_ENTRIES; i++) { 451 dev->rx_q.e[i].urb = usb_alloc_urb(0, GFP_KERNEL); 452 dev->rx_q.e[i].p = dev_alloc_pages(MT_RX_ORDER); 453 454 if (!dev->rx_q.e[i].urb || !dev->rx_q.e[i].p) 455 return -ENOMEM; 456 } 457 458 return 0; 459 } 460 461 static void mt7601u_free_tx_queue(struct mt7601u_tx_queue *q) 462 { 463 int i; 464 465 for (i = 0; i < q->entries; i++) { 466 usb_poison_urb(q->e[i].urb); 467 if (q->e[i].skb) 468 mt7601u_tx_status(q->dev, q->e[i].skb); 469 usb_free_urb(q->e[i].urb); 470 } 471 } 472 473 static void mt7601u_free_tx(struct mt7601u_dev *dev) 474 { 475 int i; 476 477 if (!dev->tx_q) 478 return; 479 480 for (i = 0; i < __MT_EP_OUT_MAX; i++) 481 mt7601u_free_tx_queue(&dev->tx_q[i]); 482 } 483 484 static int mt7601u_alloc_tx_queue(struct mt7601u_dev *dev, 485 struct mt7601u_tx_queue *q) 486 { 487 int i; 488 489 q->dev = dev; 490 q->entries = N_TX_ENTRIES; 491 492 for (i = 0; i < N_TX_ENTRIES; i++) { 493 q->e[i].urb = usb_alloc_urb(0, GFP_KERNEL); 494 if (!q->e[i].urb) 495 return -ENOMEM; 496 } 497 498 return 0; 499 } 500 501 static int mt7601u_alloc_tx(struct mt7601u_dev *dev) 502 { 503 int i; 504 505 dev->tx_q = devm_kcalloc(dev->dev, __MT_EP_OUT_MAX, 506 sizeof(*dev->tx_q), GFP_KERNEL); 507 if (!dev->tx_q) 508 return -ENOMEM; 509 510 for (i = 0; i < __MT_EP_OUT_MAX; i++) 511 if (mt7601u_alloc_tx_queue(dev, &dev->tx_q[i])) 512 return -ENOMEM; 513 514 return 0; 515 } 516 517 int mt7601u_dma_init(struct mt7601u_dev *dev) 518 { 519 int ret; 520 521 tasklet_setup(&dev->tx_tasklet, mt7601u_tx_tasklet); 522 tasklet_setup(&dev->rx_tasklet, mt7601u_rx_tasklet); 523 524 ret = mt7601u_alloc_tx(dev); 525 if (ret) 526 goto err; 527 ret = mt7601u_alloc_rx(dev); 528 if (ret) 529 goto err; 530 531 ret = mt7601u_submit_rx(dev); 532 if (ret) 533 goto err; 534 535 return 0; 536 err: 537 mt7601u_dma_cleanup(dev); 538 return ret; 539 } 540 541 void mt7601u_dma_cleanup(struct mt7601u_dev *dev) 542 { 543 mt7601u_kill_rx(dev); 544 545 tasklet_kill(&dev->rx_tasklet); 546 547 mt7601u_free_rx(dev); 548 mt7601u_free_tx(dev); 549 550 tasklet_kill(&dev->tx_tasklet); 551 } 552