1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright(c) 2020 Intel Corporation. 4 * 5 */ 6 7 /* 8 * This file contains HFI1 support for netdev RX functionality 9 */ 10 11 #include "sdma.h" 12 #include "verbs.h" 13 #include "netdev.h" 14 #include "hfi.h" 15 16 #include <linux/netdevice.h> 17 #include <linux/etherdevice.h> 18 #include <rdma/ib_verbs.h> 19 20 static int hfi1_netdev_setup_ctxt(struct hfi1_netdev_priv *priv, 21 struct hfi1_ctxtdata *uctxt) 22 { 23 unsigned int rcvctrl_ops; 24 struct hfi1_devdata *dd = priv->dd; 25 int ret; 26 27 uctxt->rhf_rcv_function_map = netdev_rhf_rcv_functions; 28 uctxt->do_interrupt = &handle_receive_interrupt_napi_sp; 29 30 /* Now allocate the RcvHdr queue and eager buffers. */ 31 ret = hfi1_create_rcvhdrq(dd, uctxt); 32 if (ret) 33 goto done; 34 35 ret = hfi1_setup_eagerbufs(uctxt); 36 if (ret) 37 goto done; 38 39 clear_rcvhdrtail(uctxt); 40 41 rcvctrl_ops = HFI1_RCVCTRL_CTXT_DIS; 42 rcvctrl_ops |= HFI1_RCVCTRL_INTRAVAIL_DIS; 43 44 if (!HFI1_CAP_KGET_MASK(uctxt->flags, MULTI_PKT_EGR)) 45 rcvctrl_ops |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB; 46 if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_EGR_FULL)) 47 rcvctrl_ops |= HFI1_RCVCTRL_NO_EGR_DROP_ENB; 48 if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_RHQ_FULL)) 49 rcvctrl_ops |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB; 50 if (HFI1_CAP_KGET_MASK(uctxt->flags, DMA_RTAIL)) 51 rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB; 52 53 hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt); 54 done: 55 return ret; 56 } 57 58 static int hfi1_netdev_allocate_ctxt(struct hfi1_devdata *dd, 59 struct hfi1_ctxtdata **ctxt) 60 { 61 struct hfi1_ctxtdata *uctxt; 62 int ret; 63 64 if (dd->flags & HFI1_FROZEN) 65 return -EIO; 66 67 ret = hfi1_create_ctxtdata(dd->pport, dd->node, &uctxt); 68 if (ret < 0) { 69 dd_dev_err(dd, "Unable to create ctxtdata, failing open\n"); 70 return -ENOMEM; 71 } 72 73 uctxt->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) | 74 HFI1_CAP_KGET(NODROP_RHQ_FULL) | 75 HFI1_CAP_KGET(NODROP_EGR_FULL) | 76 HFI1_CAP_KGET(DMA_RTAIL); 77 /* Netdev contexts are always NO_RDMA_RTAIL */ 78 uctxt->fast_handler = handle_receive_interrupt_napi_fp; 79 uctxt->slow_handler = handle_receive_interrupt_napi_sp; 80 hfi1_set_seq_cnt(uctxt, 1); 81 uctxt->is_vnic = true; 82 83 hfi1_stats.sps_ctxts++; 84 85 dd_dev_info(dd, "created netdev context %d\n", uctxt->ctxt); 86 *ctxt = uctxt; 87 88 return 0; 89 } 90 91 static void hfi1_netdev_deallocate_ctxt(struct hfi1_devdata *dd, 92 struct hfi1_ctxtdata *uctxt) 93 { 94 flush_wc(); 95 96 /* 97 * Disable receive context and interrupt available, reset all 98 * RcvCtxtCtrl bits to default values. 99 */ 100 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS | 101 HFI1_RCVCTRL_TIDFLOW_DIS | 102 HFI1_RCVCTRL_INTRAVAIL_DIS | 103 HFI1_RCVCTRL_ONE_PKT_EGR_DIS | 104 HFI1_RCVCTRL_NO_RHQ_DROP_DIS | 105 HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt); 106 107 if (uctxt->msix_intr != CCE_NUM_MSIX_VECTORS) 108 msix_free_irq(dd, uctxt->msix_intr); 109 110 uctxt->msix_intr = CCE_NUM_MSIX_VECTORS; 111 uctxt->event_flags = 0; 112 113 hfi1_clear_tids(uctxt); 114 hfi1_clear_ctxt_pkey(dd, uctxt); 115 116 hfi1_stats.sps_ctxts--; 117 118 hfi1_free_ctxt(uctxt); 119 } 120 121 static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_priv *priv, 122 struct hfi1_ctxtdata **ctxt) 123 { 124 int rc; 125 struct hfi1_devdata *dd = priv->dd; 126 127 rc = hfi1_netdev_allocate_ctxt(dd, ctxt); 128 if (rc) { 129 dd_dev_err(dd, "netdev ctxt alloc failed %d\n", rc); 130 return rc; 131 } 132 133 rc = hfi1_netdev_setup_ctxt(priv, *ctxt); 134 if (rc) { 135 dd_dev_err(dd, "netdev ctxt setup failed %d\n", rc); 136 hfi1_netdev_deallocate_ctxt(dd, *ctxt); 137 *ctxt = NULL; 138 } 139 140 return rc; 141 } 142 143 /** 144 * hfi1_num_netdev_contexts - Count of netdev recv contexts to use. 145 * @dd: device on which to allocate netdev contexts 146 * @available_contexts: count of available receive contexts 147 * @cpu_mask: mask of possible cpus to include for contexts 148 * 149 * Return: count of physical cores on a node or the remaining available recv 150 * contexts for netdev recv context usage up to the maximum of 151 * HFI1_MAX_NETDEV_CTXTS. 152 * A value of 0 can be returned when acceleration is explicitly turned off, 153 * a memory allocation error occurs or when there are no available contexts. 154 * 155 */ 156 u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts, 157 struct cpumask *cpu_mask) 158 { 159 cpumask_var_t node_cpu_mask; 160 unsigned int available_cpus; 161 162 if (!HFI1_CAP_IS_KSET(AIP)) 163 return 0; 164 165 /* Always give user contexts priority over netdev contexts */ 166 if (available_contexts == 0) { 167 dd_dev_info(dd, "No receive contexts available for netdevs.\n"); 168 return 0; 169 } 170 171 if (!zalloc_cpumask_var(&node_cpu_mask, GFP_KERNEL)) { 172 dd_dev_err(dd, "Unable to allocate cpu_mask for netdevs.\n"); 173 return 0; 174 } 175 176 cpumask_and(node_cpu_mask, cpu_mask, 177 cpumask_of_node(pcibus_to_node(dd->pcidev->bus))); 178 179 available_cpus = cpumask_weight(node_cpu_mask); 180 181 free_cpumask_var(node_cpu_mask); 182 183 return min3(available_cpus, available_contexts, 184 (u32)HFI1_MAX_NETDEV_CTXTS); 185 } 186 187 static int hfi1_netdev_rxq_init(struct net_device *dev) 188 { 189 int i; 190 int rc; 191 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev); 192 struct hfi1_devdata *dd = priv->dd; 193 194 priv->num_rx_q = dd->num_netdev_contexts; 195 priv->rxq = kcalloc_node(priv->num_rx_q, sizeof(struct hfi1_netdev_rxq), 196 GFP_KERNEL, dd->node); 197 198 if (!priv->rxq) { 199 dd_dev_err(dd, "Unable to allocate netdev queue data\n"); 200 return (-ENOMEM); 201 } 202 203 for (i = 0; i < priv->num_rx_q; i++) { 204 struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; 205 206 rc = hfi1_netdev_allot_ctxt(priv, &rxq->rcd); 207 if (rc) 208 goto bail_context_irq_failure; 209 210 hfi1_rcd_get(rxq->rcd); 211 rxq->priv = priv; 212 rxq->rcd->napi = &rxq->napi; 213 dd_dev_info(dd, "Setting rcv queue %d napi to context %d\n", 214 i, rxq->rcd->ctxt); 215 /* 216 * Disable BUSY_POLL on this NAPI as this is not supported 217 * right now. 218 */ 219 set_bit(NAPI_STATE_NO_BUSY_POLL, &rxq->napi.state); 220 netif_napi_add(dev, &rxq->napi, hfi1_netdev_rx_napi, 64); 221 rc = msix_netdev_request_rcd_irq(rxq->rcd); 222 if (rc) 223 goto bail_context_irq_failure; 224 } 225 226 return 0; 227 228 bail_context_irq_failure: 229 dd_dev_err(dd, "Unable to allot receive context\n"); 230 for (; i >= 0; i--) { 231 struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; 232 233 if (rxq->rcd) { 234 hfi1_netdev_deallocate_ctxt(dd, rxq->rcd); 235 hfi1_rcd_put(rxq->rcd); 236 rxq->rcd = NULL; 237 } 238 } 239 kfree(priv->rxq); 240 priv->rxq = NULL; 241 242 return rc; 243 } 244 245 static void hfi1_netdev_rxq_deinit(struct net_device *dev) 246 { 247 int i; 248 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev); 249 struct hfi1_devdata *dd = priv->dd; 250 251 for (i = 0; i < priv->num_rx_q; i++) { 252 struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; 253 254 netif_napi_del(&rxq->napi); 255 hfi1_netdev_deallocate_ctxt(dd, rxq->rcd); 256 hfi1_rcd_put(rxq->rcd); 257 rxq->rcd = NULL; 258 } 259 260 kfree(priv->rxq); 261 priv->rxq = NULL; 262 priv->num_rx_q = 0; 263 } 264 265 static void enable_queues(struct hfi1_netdev_priv *priv) 266 { 267 int i; 268 269 for (i = 0; i < priv->num_rx_q; i++) { 270 struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; 271 272 dd_dev_info(priv->dd, "enabling queue %d on context %d\n", i, 273 rxq->rcd->ctxt); 274 napi_enable(&rxq->napi); 275 hfi1_rcvctrl(priv->dd, 276 HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB, 277 rxq->rcd); 278 } 279 } 280 281 static void disable_queues(struct hfi1_netdev_priv *priv) 282 { 283 int i; 284 285 msix_netdev_synchronize_irq(priv->dd); 286 287 for (i = 0; i < priv->num_rx_q; i++) { 288 struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; 289 290 dd_dev_info(priv->dd, "disabling queue %d on context %d\n", i, 291 rxq->rcd->ctxt); 292 293 /* wait for napi if it was scheduled */ 294 hfi1_rcvctrl(priv->dd, 295 HFI1_RCVCTRL_CTXT_DIS | HFI1_RCVCTRL_INTRAVAIL_DIS, 296 rxq->rcd); 297 napi_synchronize(&rxq->napi); 298 napi_disable(&rxq->napi); 299 } 300 } 301 302 /** 303 * hfi1_netdev_rx_init - Incrememnts netdevs counter. When called first time, 304 * it allocates receive queue data and calls netif_napi_add 305 * for each queue. 306 * 307 * @dd: hfi1 dev data 308 */ 309 int hfi1_netdev_rx_init(struct hfi1_devdata *dd) 310 { 311 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); 312 int res; 313 314 if (atomic_fetch_inc(&priv->netdevs)) 315 return 0; 316 317 mutex_lock(&hfi1_mutex); 318 init_dummy_netdev(dd->dummy_netdev); 319 res = hfi1_netdev_rxq_init(dd->dummy_netdev); 320 mutex_unlock(&hfi1_mutex); 321 return res; 322 } 323 324 /** 325 * hfi1_netdev_rx_destroy - Decrements netdevs counter, when it reaches 0 326 * napi is deleted and receive queses memory is freed. 327 * 328 * @dd: hfi1 dev data 329 */ 330 int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd) 331 { 332 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); 333 334 /* destroy the RX queues only if it is the last netdev going away */ 335 if (atomic_fetch_add_unless(&priv->netdevs, -1, 0) == 1) { 336 mutex_lock(&hfi1_mutex); 337 hfi1_netdev_rxq_deinit(dd->dummy_netdev); 338 mutex_unlock(&hfi1_mutex); 339 } 340 341 return 0; 342 } 343 344 /** 345 * hfi1_netdev_alloc - Allocates netdev and private data. It is required 346 * because RMT index and MSI-X interrupt can be set only 347 * during driver initialization. 348 * 349 * @dd: hfi1 dev data 350 */ 351 int hfi1_netdev_alloc(struct hfi1_devdata *dd) 352 { 353 struct hfi1_netdev_priv *priv; 354 const int netdev_size = sizeof(*dd->dummy_netdev) + 355 sizeof(struct hfi1_netdev_priv); 356 357 dd_dev_info(dd, "allocating netdev size %d\n", netdev_size); 358 dd->dummy_netdev = kcalloc_node(1, netdev_size, GFP_KERNEL, dd->node); 359 360 if (!dd->dummy_netdev) 361 return -ENOMEM; 362 363 priv = hfi1_netdev_priv(dd->dummy_netdev); 364 priv->dd = dd; 365 xa_init(&priv->dev_tbl); 366 atomic_set(&priv->enabled, 0); 367 atomic_set(&priv->netdevs, 0); 368 369 return 0; 370 } 371 372 void hfi1_netdev_free(struct hfi1_devdata *dd) 373 { 374 if (dd->dummy_netdev) { 375 dd_dev_info(dd, "hfi1 netdev freed\n"); 376 kfree(dd->dummy_netdev); 377 dd->dummy_netdev = NULL; 378 } 379 } 380 381 /** 382 * hfi1_netdev_enable_queues - This is napi enable function. 383 * It enables napi objects associated with queues. 384 * When at least one device has called it it increments atomic counter. 385 * Disable function decrements counter and when it is 0, 386 * calls napi_disable for every queue. 387 * 388 * @dd: hfi1 dev data 389 */ 390 void hfi1_netdev_enable_queues(struct hfi1_devdata *dd) 391 { 392 struct hfi1_netdev_priv *priv; 393 394 if (!dd->dummy_netdev) 395 return; 396 397 priv = hfi1_netdev_priv(dd->dummy_netdev); 398 if (atomic_fetch_inc(&priv->enabled)) 399 return; 400 401 mutex_lock(&hfi1_mutex); 402 enable_queues(priv); 403 mutex_unlock(&hfi1_mutex); 404 } 405 406 void hfi1_netdev_disable_queues(struct hfi1_devdata *dd) 407 { 408 struct hfi1_netdev_priv *priv; 409 410 if (!dd->dummy_netdev) 411 return; 412 413 priv = hfi1_netdev_priv(dd->dummy_netdev); 414 if (atomic_dec_if_positive(&priv->enabled)) 415 return; 416 417 mutex_lock(&hfi1_mutex); 418 disable_queues(priv); 419 mutex_unlock(&hfi1_mutex); 420 } 421 422 /** 423 * hfi1_netdev_add_data - Registers data with unique identifier 424 * to be requested later this is needed for VNIC and IPoIB VLANs 425 * implementations. 426 * This call is protected by mutex idr_lock. 427 * 428 * @dd: hfi1 dev data 429 * @id: requested integer id up to INT_MAX 430 * @data: data to be associated with index 431 */ 432 int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data) 433 { 434 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); 435 436 return xa_insert(&priv->dev_tbl, id, data, GFP_NOWAIT); 437 } 438 439 /** 440 * hfi1_netdev_remove_data - Removes data with previously given id. 441 * Returns the reference to removed entry. 442 * 443 * @dd: hfi1 dev data 444 * @id: requested integer id up to INT_MAX 445 */ 446 void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id) 447 { 448 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); 449 450 return xa_erase(&priv->dev_tbl, id); 451 } 452 453 /** 454 * hfi1_netdev_get_data - Gets data with given id 455 * 456 * @dd: hfi1 dev data 457 * @id: requested integer id up to INT_MAX 458 */ 459 void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id) 460 { 461 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); 462 463 return xa_load(&priv->dev_tbl, id); 464 } 465 466 /** 467 * hfi1_netdev_get_first_dat - Gets first entry with greater or equal id. 468 * 469 * @dd: hfi1 dev data 470 * @id: requested integer id up to INT_MAX 471 */ 472 void *hfi1_netdev_get_first_data(struct hfi1_devdata *dd, int *start_id) 473 { 474 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); 475 unsigned long index = *start_id; 476 void *ret; 477 478 ret = xa_find(&priv->dev_tbl, &index, UINT_MAX, XA_PRESENT); 479 *start_id = (int)index; 480 return ret; 481 } 482