1 /* 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <net/tc_act/tc_gact.h> 34 #include <net/pkt_cls.h> 35 #include <linux/mlx5/fs.h> 36 #include <net/vxlan.h> 37 #include <net/geneve.h> 38 #include <linux/bpf.h> 39 #include <linux/if_bridge.h> 40 #include <net/page_pool.h> 41 #include <net/xdp_sock_drv.h> 42 #include "eswitch.h" 43 #include "en.h" 44 #include "en/txrx.h" 45 #include "en_tc.h" 46 #include "en_rep.h" 47 #include "en_accel/ipsec.h" 48 #include "en_accel/ipsec_rxtx.h" 49 #include "en_accel/en_accel.h" 50 #include "en_accel/tls.h" 51 #include "accel/ipsec.h" 52 #include "accel/tls.h" 53 #include "lib/vxlan.h" 54 #include "lib/clock.h" 55 #include "en/port.h" 56 #include "en/xdp.h" 57 #include "lib/eq.h" 58 #include "en/monitor_stats.h" 59 #include "en/health.h" 60 #include "en/params.h" 61 #include "en/xsk/umem.h" 62 #include "en/xsk/setup.h" 63 #include "en/xsk/rx.h" 64 #include "en/xsk/tx.h" 65 #include "en/hv_vhca_stats.h" 66 #include "en/devlink.h" 67 #include "lib/mlx5.h" 68 69 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) 70 { 71 bool striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && 72 MLX5_CAP_GEN(mdev, umr_ptr_rlky) && 73 MLX5_CAP_ETH(mdev, reg_umr_sq); 74 u16 max_wqe_sz_cap = MLX5_CAP_GEN(mdev, max_wqe_sz_sq); 75 bool inline_umr = MLX5E_UMR_WQE_INLINE_SZ <= max_wqe_sz_cap; 76 77 if (!striding_rq_umr) 78 return false; 79 if (!inline_umr) { 80 mlx5_core_warn(mdev, "Cannot support Striding RQ: UMR WQE size (%d) exceeds maximum supported (%d).\n", 81 (int)MLX5E_UMR_WQE_INLINE_SZ, max_wqe_sz_cap); 82 return false; 83 } 84 return true; 85 } 86 87 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, 88 struct mlx5e_params *params) 89 { 90 params->log_rq_mtu_frames = is_kdump_kernel() ? 91 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE : 92 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE; 93 94 mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n", 95 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ, 96 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ? 97 BIT(mlx5e_mpwqe_get_log_rq_size(params, NULL)) : 98 BIT(params->log_rq_mtu_frames), 99 BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)), 100 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)); 101 } 102 103 bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev, 104 struct mlx5e_params *params) 105 { 106 if (!mlx5e_check_fragmented_striding_rq_cap(mdev)) 107 return false; 108 109 if (MLX5_IPSEC_DEV(mdev)) 110 return false; 111 112 if (params->xdp_prog) { 113 /* XSK params are not considered here. If striding RQ is in use, 114 * and an XSK is being opened, mlx5e_rx_mpwqe_is_linear_skb will 115 * be called with the known XSK params. 116 */ 117 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL)) 118 return false; 119 } 120 121 return true; 122 } 123 124 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params) 125 { 126 params->rq_wq_type = mlx5e_striding_rq_possible(mdev, params) && 127 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ? 128 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ : 129 MLX5_WQ_TYPE_CYCLIC; 130 } 131 132 void mlx5e_update_carrier(struct mlx5e_priv *priv) 133 { 134 struct mlx5_core_dev *mdev = priv->mdev; 135 u8 port_state; 136 137 port_state = mlx5_query_vport_state(mdev, 138 MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 139 0); 140 141 if (port_state == VPORT_STATE_UP) { 142 netdev_info(priv->netdev, "Link up\n"); 143 netif_carrier_on(priv->netdev); 144 } else { 145 netdev_info(priv->netdev, "Link down\n"); 146 netif_carrier_off(priv->netdev); 147 } 148 } 149 150 static void mlx5e_update_carrier_work(struct work_struct *work) 151 { 152 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 153 update_carrier_work); 154 155 mutex_lock(&priv->state_lock); 156 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 157 if (priv->profile->update_carrier) 158 priv->profile->update_carrier(priv); 159 mutex_unlock(&priv->state_lock); 160 } 161 162 void mlx5e_update_ndo_stats(struct mlx5e_priv *priv) 163 { 164 int i; 165 166 for (i = mlx5e_nic_stats_grps_num(priv) - 1; i >= 0; i--) 167 if (mlx5e_nic_stats_grps[i]->update_stats_mask & 168 MLX5E_NDO_UPDATE_STATS) 169 mlx5e_nic_stats_grps[i]->update_stats(priv); 170 } 171 172 static void mlx5e_update_stats_work(struct work_struct *work) 173 { 174 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 175 update_stats_work); 176 177 mutex_lock(&priv->state_lock); 178 priv->profile->update_stats(priv); 179 mutex_unlock(&priv->state_lock); 180 } 181 182 void mlx5e_queue_update_stats(struct mlx5e_priv *priv) 183 { 184 if (!priv->profile->update_stats) 185 return; 186 187 if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state))) 188 return; 189 190 queue_work(priv->wq, &priv->update_stats_work); 191 } 192 193 static int async_event(struct notifier_block *nb, unsigned long event, void *data) 194 { 195 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); 196 struct mlx5_eqe *eqe = data; 197 198 if (event != MLX5_EVENT_TYPE_PORT_CHANGE) 199 return NOTIFY_DONE; 200 201 switch (eqe->sub_type) { 202 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 203 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 204 queue_work(priv->wq, &priv->update_carrier_work); 205 break; 206 default: 207 return NOTIFY_DONE; 208 } 209 210 return NOTIFY_OK; 211 } 212 213 static void mlx5e_enable_async_events(struct mlx5e_priv *priv) 214 { 215 priv->events_nb.notifier_call = async_event; 216 mlx5_notifier_register(priv->mdev, &priv->events_nb); 217 } 218 219 static void mlx5e_disable_async_events(struct mlx5e_priv *priv) 220 { 221 mlx5_notifier_unregister(priv->mdev, &priv->events_nb); 222 } 223 224 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, 225 struct mlx5e_icosq *sq, 226 struct mlx5e_umr_wqe *wqe) 227 { 228 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 229 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; 230 u8 ds_cnt = DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_DS); 231 232 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | 233 ds_cnt); 234 cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 235 cseg->umr_mkey = rq->mkey_be; 236 237 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; 238 ucseg->xlt_octowords = 239 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE)); 240 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 241 } 242 243 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, 244 struct mlx5e_channel *c) 245 { 246 int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 247 248 rq->mpwqe.info = kvzalloc_node(array_size(wq_sz, 249 sizeof(*rq->mpwqe.info)), 250 GFP_KERNEL, cpu_to_node(c->cpu)); 251 if (!rq->mpwqe.info) 252 return -ENOMEM; 253 254 mlx5e_build_umr_wqe(rq, &c->icosq, &rq->mpwqe.umr_wqe); 255 256 return 0; 257 } 258 259 static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, 260 u64 npages, u8 page_shift, 261 struct mlx5_core_mkey *umr_mkey) 262 { 263 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 264 void *mkc; 265 u32 *in; 266 int err; 267 268 in = kvzalloc(inlen, GFP_KERNEL); 269 if (!in) 270 return -ENOMEM; 271 272 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 273 274 MLX5_SET(mkc, mkc, free, 1); 275 MLX5_SET(mkc, mkc, umr_en, 1); 276 MLX5_SET(mkc, mkc, lw, 1); 277 MLX5_SET(mkc, mkc, lr, 1); 278 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 279 280 MLX5_SET(mkc, mkc, qpn, 0xffffff); 281 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn); 282 MLX5_SET64(mkc, mkc, len, npages << page_shift); 283 MLX5_SET(mkc, mkc, translations_octword_size, 284 MLX5_MTT_OCTW(npages)); 285 MLX5_SET(mkc, mkc, log_page_size, page_shift); 286 287 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 288 289 kvfree(in); 290 return err; 291 } 292 293 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) 294 { 295 u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->mpwqe.wq)); 296 297 return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey); 298 } 299 300 static inline u64 mlx5e_get_mpwqe_offset(struct mlx5e_rq *rq, u16 wqe_ix) 301 { 302 return (wqe_ix << MLX5E_LOG_ALIGNED_MPWQE_PPW) << PAGE_SHIFT; 303 } 304 305 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) 306 { 307 struct mlx5e_wqe_frag_info next_frag = {}; 308 struct mlx5e_wqe_frag_info *prev = NULL; 309 int i; 310 311 next_frag.di = &rq->wqe.di[0]; 312 313 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { 314 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; 315 struct mlx5e_wqe_frag_info *frag = 316 &rq->wqe.frags[i << rq->wqe.info.log_num_frags]; 317 int f; 318 319 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) { 320 if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) { 321 next_frag.di++; 322 next_frag.offset = 0; 323 if (prev) 324 prev->last_in_page = true; 325 } 326 *frag = next_frag; 327 328 /* prepare next */ 329 next_frag.offset += frag_info[f].frag_stride; 330 prev = frag; 331 } 332 } 333 334 if (prev) 335 prev->last_in_page = true; 336 } 337 338 static int mlx5e_init_di_list(struct mlx5e_rq *rq, 339 int wq_sz, int cpu) 340 { 341 int len = wq_sz << rq->wqe.info.log_num_frags; 342 343 rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)), 344 GFP_KERNEL, cpu_to_node(cpu)); 345 if (!rq->wqe.di) 346 return -ENOMEM; 347 348 mlx5e_init_frags_partition(rq); 349 350 return 0; 351 } 352 353 static void mlx5e_free_di_list(struct mlx5e_rq *rq) 354 { 355 kvfree(rq->wqe.di); 356 } 357 358 static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work) 359 { 360 struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work); 361 362 mlx5e_reporter_rq_cqe_err(rq); 363 } 364 365 static int mlx5e_alloc_rq(struct mlx5e_channel *c, 366 struct mlx5e_params *params, 367 struct mlx5e_xsk_param *xsk, 368 struct xdp_umem *umem, 369 struct mlx5e_rq_param *rqp, 370 struct mlx5e_rq *rq) 371 { 372 struct page_pool_params pp_params = { 0 }; 373 struct mlx5_core_dev *mdev = c->mdev; 374 void *rqc = rqp->rqc; 375 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); 376 u32 rq_xdp_ix; 377 u32 pool_size; 378 int wq_sz; 379 int err; 380 int i; 381 382 rqp->wq.db_numa_node = cpu_to_node(c->cpu); 383 384 rq->wq_type = params->rq_wq_type; 385 rq->pdev = c->pdev; 386 rq->netdev = c->netdev; 387 rq->tstamp = c->tstamp; 388 rq->clock = &mdev->clock; 389 rq->channel = c; 390 rq->ix = c->ix; 391 rq->mdev = mdev; 392 rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 393 rq->xdpsq = &c->rq_xdpsq; 394 rq->umem = umem; 395 396 if (rq->umem) 397 rq->stats = &c->priv->channel_stats[c->ix].xskrq; 398 else 399 rq->stats = &c->priv->channel_stats[c->ix].rq; 400 INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work); 401 402 if (params->xdp_prog) 403 bpf_prog_inc(params->xdp_prog); 404 rq->xdp_prog = params->xdp_prog; 405 406 rq_xdp_ix = rq->ix; 407 if (xsk) 408 rq_xdp_ix += params->num_channels * MLX5E_RQ_GROUP_XSK; 409 err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix); 410 if (err < 0) 411 goto err_rq_wq_destroy; 412 413 rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 414 rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk); 415 pool_size = 1 << params->log_rq_mtu_frames; 416 417 switch (rq->wq_type) { 418 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 419 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq, 420 &rq->wq_ctrl); 421 if (err) 422 goto err_rq_wq_destroy; 423 424 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR]; 425 426 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 427 428 pool_size = MLX5_MPWRQ_PAGES_PER_WQE << 429 mlx5e_mpwqe_get_log_rq_size(params, xsk); 430 431 rq->post_wqes = mlx5e_post_rx_mpwqes; 432 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe; 433 434 rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe; 435 #ifdef CONFIG_MLX5_EN_IPSEC 436 if (MLX5_IPSEC_DEV(mdev)) { 437 err = -EINVAL; 438 netdev_err(c->netdev, "MPWQE RQ with IPSec offload not supported\n"); 439 goto err_rq_wq_destroy; 440 } 441 #endif 442 if (!rq->handle_rx_cqe) { 443 err = -EINVAL; 444 netdev_err(c->netdev, "RX handler of MPWQE RQ is not set, err %d\n", err); 445 goto err_rq_wq_destroy; 446 } 447 448 rq->mpwqe.skb_from_cqe_mpwrq = xsk ? 449 mlx5e_xsk_skb_from_cqe_mpwrq_linear : 450 mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ? 451 mlx5e_skb_from_cqe_mpwrq_linear : 452 mlx5e_skb_from_cqe_mpwrq_nonlinear; 453 454 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); 455 rq->mpwqe.num_strides = 456 BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); 457 458 rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz); 459 460 err = mlx5e_create_rq_umr_mkey(mdev, rq); 461 if (err) 462 goto err_rq_wq_destroy; 463 rq->mkey_be = cpu_to_be32(rq->umr_mkey.key); 464 465 err = mlx5e_rq_alloc_mpwqe_info(rq, c); 466 if (err) 467 goto err_free; 468 break; 469 default: /* MLX5_WQ_TYPE_CYCLIC */ 470 err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq, 471 &rq->wq_ctrl); 472 if (err) 473 goto err_rq_wq_destroy; 474 475 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR]; 476 477 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); 478 479 rq->wqe.info = rqp->frags_info; 480 rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride; 481 482 rq->wqe.frags = 483 kvzalloc_node(array_size(sizeof(*rq->wqe.frags), 484 (wq_sz << rq->wqe.info.log_num_frags)), 485 GFP_KERNEL, cpu_to_node(c->cpu)); 486 if (!rq->wqe.frags) { 487 err = -ENOMEM; 488 goto err_free; 489 } 490 491 err = mlx5e_init_di_list(rq, wq_sz, c->cpu); 492 if (err) 493 goto err_free; 494 495 rq->post_wqes = mlx5e_post_rx_wqes; 496 rq->dealloc_wqe = mlx5e_dealloc_rx_wqe; 497 498 #ifdef CONFIG_MLX5_EN_IPSEC 499 if (c->priv->ipsec) 500 rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe; 501 else 502 #endif 503 rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe; 504 if (!rq->handle_rx_cqe) { 505 err = -EINVAL; 506 netdev_err(c->netdev, "RX handler of RQ is not set, err %d\n", err); 507 goto err_free; 508 } 509 510 rq->wqe.skb_from_cqe = xsk ? 511 mlx5e_xsk_skb_from_cqe_linear : 512 mlx5e_rx_is_linear_skb(params, NULL) ? 513 mlx5e_skb_from_cqe_linear : 514 mlx5e_skb_from_cqe_nonlinear; 515 rq->mkey_be = c->mkey_be; 516 } 517 518 if (xsk) { 519 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 520 MEM_TYPE_XSK_BUFF_POOL, NULL); 521 xsk_buff_set_rxq_info(rq->umem, &rq->xdp_rxq); 522 } else { 523 /* Create a page_pool and register it with rxq */ 524 pp_params.order = 0; 525 pp_params.flags = 0; /* No-internal DMA mapping in page_pool */ 526 pp_params.pool_size = pool_size; 527 pp_params.nid = cpu_to_node(c->cpu); 528 pp_params.dev = c->pdev; 529 pp_params.dma_dir = rq->buff.map_dir; 530 531 /* page_pool can be used even when there is no rq->xdp_prog, 532 * given page_pool does not handle DMA mapping there is no 533 * required state to clear. And page_pool gracefully handle 534 * elevated refcnt. 535 */ 536 rq->page_pool = page_pool_create(&pp_params); 537 if (IS_ERR(rq->page_pool)) { 538 err = PTR_ERR(rq->page_pool); 539 rq->page_pool = NULL; 540 goto err_free; 541 } 542 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 543 MEM_TYPE_PAGE_POOL, rq->page_pool); 544 } 545 if (err) 546 goto err_free; 547 548 for (i = 0; i < wq_sz; i++) { 549 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 550 struct mlx5e_rx_wqe_ll *wqe = 551 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i); 552 u32 byte_count = 553 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz; 554 u64 dma_offset = mlx5e_get_mpwqe_offset(rq, i); 555 556 wqe->data[0].addr = cpu_to_be64(dma_offset + rq->buff.headroom); 557 wqe->data[0].byte_count = cpu_to_be32(byte_count); 558 wqe->data[0].lkey = rq->mkey_be; 559 } else { 560 struct mlx5e_rx_wqe_cyc *wqe = 561 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i); 562 int f; 563 564 for (f = 0; f < rq->wqe.info.num_frags; f++) { 565 u32 frag_size = rq->wqe.info.arr[f].frag_size | 566 MLX5_HW_START_PADDING; 567 568 wqe->data[f].byte_count = cpu_to_be32(frag_size); 569 wqe->data[f].lkey = rq->mkey_be; 570 } 571 /* check if num_frags is not a pow of two */ 572 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) { 573 wqe->data[f].byte_count = 0; 574 wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 575 wqe->data[f].addr = 0; 576 } 577 } 578 } 579 580 INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work); 581 582 switch (params->rx_cq_moderation.cq_period_mode) { 583 case MLX5_CQ_PERIOD_MODE_START_FROM_CQE: 584 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE; 585 break; 586 case MLX5_CQ_PERIOD_MODE_START_FROM_EQE: 587 default: 588 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 589 } 590 591 rq->page_cache.head = 0; 592 rq->page_cache.tail = 0; 593 594 return 0; 595 596 err_free: 597 switch (rq->wq_type) { 598 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 599 kvfree(rq->mpwqe.info); 600 mlx5_core_destroy_mkey(mdev, &rq->umr_mkey); 601 break; 602 default: /* MLX5_WQ_TYPE_CYCLIC */ 603 kvfree(rq->wqe.frags); 604 mlx5e_free_di_list(rq); 605 } 606 607 err_rq_wq_destroy: 608 if (rq->xdp_prog) 609 bpf_prog_put(rq->xdp_prog); 610 xdp_rxq_info_unreg(&rq->xdp_rxq); 611 page_pool_destroy(rq->page_pool); 612 mlx5_wq_destroy(&rq->wq_ctrl); 613 614 return err; 615 } 616 617 static void mlx5e_free_rq(struct mlx5e_rq *rq) 618 { 619 int i; 620 621 if (rq->xdp_prog) 622 bpf_prog_put(rq->xdp_prog); 623 624 switch (rq->wq_type) { 625 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 626 kvfree(rq->mpwqe.info); 627 mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey); 628 break; 629 default: /* MLX5_WQ_TYPE_CYCLIC */ 630 kvfree(rq->wqe.frags); 631 mlx5e_free_di_list(rq); 632 } 633 634 for (i = rq->page_cache.head; i != rq->page_cache.tail; 635 i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) { 636 struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i]; 637 638 /* With AF_XDP, page_cache is not used, so this loop is not 639 * entered, and it's safe to call mlx5e_page_release_dynamic 640 * directly. 641 */ 642 mlx5e_page_release_dynamic(rq, dma_info, false); 643 } 644 645 xdp_rxq_info_unreg(&rq->xdp_rxq); 646 page_pool_destroy(rq->page_pool); 647 mlx5_wq_destroy(&rq->wq_ctrl); 648 } 649 650 static int mlx5e_create_rq(struct mlx5e_rq *rq, 651 struct mlx5e_rq_param *param) 652 { 653 struct mlx5_core_dev *mdev = rq->mdev; 654 655 void *in; 656 void *rqc; 657 void *wq; 658 int inlen; 659 int err; 660 661 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + 662 sizeof(u64) * rq->wq_ctrl.buf.npages; 663 in = kvzalloc(inlen, GFP_KERNEL); 664 if (!in) 665 return -ENOMEM; 666 667 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 668 wq = MLX5_ADDR_OF(rqc, rqc, wq); 669 670 memcpy(rqc, param->rqc, sizeof(param->rqc)); 671 672 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn); 673 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 674 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift - 675 MLX5_ADAPTER_PAGE_SHIFT); 676 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma); 677 678 mlx5_fill_page_frag_array(&rq->wq_ctrl.buf, 679 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 680 681 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn); 682 683 kvfree(in); 684 685 return err; 686 } 687 688 int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state) 689 { 690 struct mlx5_core_dev *mdev = rq->mdev; 691 692 void *in; 693 void *rqc; 694 int inlen; 695 int err; 696 697 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 698 in = kvzalloc(inlen, GFP_KERNEL); 699 if (!in) 700 return -ENOMEM; 701 702 if (curr_state == MLX5_RQC_STATE_RST && next_state == MLX5_RQC_STATE_RDY) 703 mlx5e_rqwq_reset(rq); 704 705 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 706 707 MLX5_SET(modify_rq_in, in, rq_state, curr_state); 708 MLX5_SET(rqc, rqc, state, next_state); 709 710 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 711 712 kvfree(in); 713 714 return err; 715 } 716 717 static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable) 718 { 719 struct mlx5e_channel *c = rq->channel; 720 struct mlx5e_priv *priv = c->priv; 721 struct mlx5_core_dev *mdev = priv->mdev; 722 723 void *in; 724 void *rqc; 725 int inlen; 726 int err; 727 728 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 729 in = kvzalloc(inlen, GFP_KERNEL); 730 if (!in) 731 return -ENOMEM; 732 733 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 734 735 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); 736 MLX5_SET64(modify_rq_in, in, modify_bitmask, 737 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS); 738 MLX5_SET(rqc, rqc, scatter_fcs, enable); 739 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); 740 741 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 742 743 kvfree(in); 744 745 return err; 746 } 747 748 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd) 749 { 750 struct mlx5e_channel *c = rq->channel; 751 struct mlx5_core_dev *mdev = c->mdev; 752 void *in; 753 void *rqc; 754 int inlen; 755 int err; 756 757 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 758 in = kvzalloc(inlen, GFP_KERNEL); 759 if (!in) 760 return -ENOMEM; 761 762 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 763 764 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); 765 MLX5_SET64(modify_rq_in, in, modify_bitmask, 766 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD); 767 MLX5_SET(rqc, rqc, vsd, vsd); 768 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); 769 770 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 771 772 kvfree(in); 773 774 return err; 775 } 776 777 static void mlx5e_destroy_rq(struct mlx5e_rq *rq) 778 { 779 mlx5_core_destroy_rq(rq->mdev, rq->rqn); 780 } 781 782 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time) 783 { 784 unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time); 785 struct mlx5e_channel *c = rq->channel; 786 787 u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq)); 788 789 do { 790 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes) 791 return 0; 792 793 msleep(20); 794 } while (time_before(jiffies, exp_time)); 795 796 netdev_warn(c->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n", 797 c->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes); 798 799 mlx5e_reporter_rx_timeout(rq); 800 return -ETIMEDOUT; 801 } 802 803 void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq) 804 { 805 struct mlx5_wq_ll *wq; 806 u16 head; 807 int i; 808 809 if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) 810 return; 811 812 wq = &rq->mpwqe.wq; 813 head = wq->head; 814 815 /* Outstanding UMR WQEs (in progress) start at wq->head */ 816 for (i = 0; i < rq->mpwqe.umr_in_progress; i++) { 817 rq->dealloc_wqe(rq, head); 818 head = mlx5_wq_ll_get_wqe_next_ix(wq, head); 819 } 820 821 rq->mpwqe.actual_wq_head = wq->head; 822 rq->mpwqe.umr_in_progress = 0; 823 rq->mpwqe.umr_completed = 0; 824 } 825 826 void mlx5e_free_rx_descs(struct mlx5e_rq *rq) 827 { 828 __be16 wqe_ix_be; 829 u16 wqe_ix; 830 831 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 832 struct mlx5_wq_ll *wq = &rq->mpwqe.wq; 833 834 mlx5e_free_rx_in_progress_descs(rq); 835 836 while (!mlx5_wq_ll_is_empty(wq)) { 837 struct mlx5e_rx_wqe_ll *wqe; 838 839 wqe_ix_be = *wq->tail_next; 840 wqe_ix = be16_to_cpu(wqe_ix_be); 841 wqe = mlx5_wq_ll_get_wqe(wq, wqe_ix); 842 rq->dealloc_wqe(rq, wqe_ix); 843 mlx5_wq_ll_pop(wq, wqe_ix_be, 844 &wqe->next.next_wqe_index); 845 } 846 } else { 847 struct mlx5_wq_cyc *wq = &rq->wqe.wq; 848 849 while (!mlx5_wq_cyc_is_empty(wq)) { 850 wqe_ix = mlx5_wq_cyc_get_tail(wq); 851 rq->dealloc_wqe(rq, wqe_ix); 852 mlx5_wq_cyc_pop(wq); 853 } 854 } 855 856 } 857 858 int mlx5e_open_rq(struct mlx5e_channel *c, struct mlx5e_params *params, 859 struct mlx5e_rq_param *param, struct mlx5e_xsk_param *xsk, 860 struct xdp_umem *umem, struct mlx5e_rq *rq) 861 { 862 int err; 863 864 err = mlx5e_alloc_rq(c, params, xsk, umem, param, rq); 865 if (err) 866 return err; 867 868 err = mlx5e_create_rq(rq, param); 869 if (err) 870 goto err_free_rq; 871 872 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 873 if (err) 874 goto err_destroy_rq; 875 876 if (MLX5_CAP_ETH(c->mdev, cqe_checksum_full)) 877 __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &c->rq.state); 878 879 if (params->rx_dim_enabled) 880 __set_bit(MLX5E_RQ_STATE_AM, &c->rq.state); 881 882 /* We disable csum_complete when XDP is enabled since 883 * XDP programs might manipulate packets which will render 884 * skb->checksum incorrect. 885 */ 886 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || c->xdp) 887 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &c->rq.state); 888 889 return 0; 890 891 err_destroy_rq: 892 mlx5e_destroy_rq(rq); 893 err_free_rq: 894 mlx5e_free_rq(rq); 895 896 return err; 897 } 898 899 void mlx5e_activate_rq(struct mlx5e_rq *rq) 900 { 901 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 902 mlx5e_trigger_irq(&rq->channel->icosq); 903 } 904 905 void mlx5e_deactivate_rq(struct mlx5e_rq *rq) 906 { 907 clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 908 napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */ 909 } 910 911 void mlx5e_close_rq(struct mlx5e_rq *rq) 912 { 913 cancel_work_sync(&rq->dim.work); 914 cancel_work_sync(&rq->channel->icosq.recover_work); 915 cancel_work_sync(&rq->recover_work); 916 mlx5e_destroy_rq(rq); 917 mlx5e_free_rx_descs(rq); 918 mlx5e_free_rq(rq); 919 } 920 921 static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq) 922 { 923 kvfree(sq->db.xdpi_fifo.xi); 924 kvfree(sq->db.wqe_info); 925 } 926 927 static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa) 928 { 929 struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo; 930 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 931 int dsegs_per_wq = wq_sz * MLX5_SEND_WQEBB_NUM_DS; 932 933 xdpi_fifo->xi = kvzalloc_node(sizeof(*xdpi_fifo->xi) * dsegs_per_wq, 934 GFP_KERNEL, numa); 935 if (!xdpi_fifo->xi) 936 return -ENOMEM; 937 938 xdpi_fifo->pc = &sq->xdpi_fifo_pc; 939 xdpi_fifo->cc = &sq->xdpi_fifo_cc; 940 xdpi_fifo->mask = dsegs_per_wq - 1; 941 942 return 0; 943 } 944 945 static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa) 946 { 947 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 948 int err; 949 950 sq->db.wqe_info = kvzalloc_node(sizeof(*sq->db.wqe_info) * wq_sz, 951 GFP_KERNEL, numa); 952 if (!sq->db.wqe_info) 953 return -ENOMEM; 954 955 err = mlx5e_alloc_xdpsq_fifo(sq, numa); 956 if (err) { 957 mlx5e_free_xdpsq_db(sq); 958 return err; 959 } 960 961 return 0; 962 } 963 964 static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, 965 struct mlx5e_params *params, 966 struct xdp_umem *umem, 967 struct mlx5e_sq_param *param, 968 struct mlx5e_xdpsq *sq, 969 bool is_redirect) 970 { 971 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 972 struct mlx5_core_dev *mdev = c->mdev; 973 struct mlx5_wq_cyc *wq = &sq->wq; 974 int err; 975 976 sq->pdev = c->pdev; 977 sq->mkey_be = c->mkey_be; 978 sq->channel = c; 979 sq->uar_map = mdev->mlx5e_res.bfreg.map; 980 sq->min_inline_mode = params->tx_min_inline_mode; 981 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 982 sq->umem = umem; 983 984 sq->stats = sq->umem ? 985 &c->priv->channel_stats[c->ix].xsksq : 986 is_redirect ? 987 &c->priv->channel_stats[c->ix].xdpsq : 988 &c->priv->channel_stats[c->ix].rq_xdpsq; 989 990 param->wq.db_numa_node = cpu_to_node(c->cpu); 991 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 992 if (err) 993 return err; 994 wq->db = &wq->db[MLX5_SND_DBR]; 995 996 err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu)); 997 if (err) 998 goto err_sq_wq_destroy; 999 1000 return 0; 1001 1002 err_sq_wq_destroy: 1003 mlx5_wq_destroy(&sq->wq_ctrl); 1004 1005 return err; 1006 } 1007 1008 static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq) 1009 { 1010 mlx5e_free_xdpsq_db(sq); 1011 mlx5_wq_destroy(&sq->wq_ctrl); 1012 } 1013 1014 static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq) 1015 { 1016 kvfree(sq->db.wqe_info); 1017 } 1018 1019 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa) 1020 { 1021 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1022 size_t size; 1023 1024 size = array_size(wq_sz, sizeof(*sq->db.wqe_info)); 1025 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa); 1026 if (!sq->db.wqe_info) 1027 return -ENOMEM; 1028 1029 return 0; 1030 } 1031 1032 static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work) 1033 { 1034 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq, 1035 recover_work); 1036 1037 mlx5e_reporter_icosq_cqe_err(sq); 1038 } 1039 1040 static int mlx5e_alloc_icosq(struct mlx5e_channel *c, 1041 struct mlx5e_sq_param *param, 1042 struct mlx5e_icosq *sq) 1043 { 1044 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1045 struct mlx5_core_dev *mdev = c->mdev; 1046 struct mlx5_wq_cyc *wq = &sq->wq; 1047 int err; 1048 1049 sq->channel = c; 1050 sq->uar_map = mdev->mlx5e_res.bfreg.map; 1051 1052 param->wq.db_numa_node = cpu_to_node(c->cpu); 1053 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1054 if (err) 1055 return err; 1056 wq->db = &wq->db[MLX5_SND_DBR]; 1057 1058 err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu)); 1059 if (err) 1060 goto err_sq_wq_destroy; 1061 1062 INIT_WORK(&sq->recover_work, mlx5e_icosq_err_cqe_work); 1063 1064 return 0; 1065 1066 err_sq_wq_destroy: 1067 mlx5_wq_destroy(&sq->wq_ctrl); 1068 1069 return err; 1070 } 1071 1072 static void mlx5e_free_icosq(struct mlx5e_icosq *sq) 1073 { 1074 mlx5e_free_icosq_db(sq); 1075 mlx5_wq_destroy(&sq->wq_ctrl); 1076 } 1077 1078 static void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq) 1079 { 1080 kvfree(sq->db.wqe_info); 1081 kvfree(sq->db.dma_fifo); 1082 } 1083 1084 static int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa) 1085 { 1086 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1087 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS; 1088 1089 sq->db.dma_fifo = kvzalloc_node(array_size(df_sz, 1090 sizeof(*sq->db.dma_fifo)), 1091 GFP_KERNEL, numa); 1092 sq->db.wqe_info = kvzalloc_node(array_size(wq_sz, 1093 sizeof(*sq->db.wqe_info)), 1094 GFP_KERNEL, numa); 1095 if (!sq->db.dma_fifo || !sq->db.wqe_info) { 1096 mlx5e_free_txqsq_db(sq); 1097 return -ENOMEM; 1098 } 1099 1100 sq->dma_fifo_mask = df_sz - 1; 1101 1102 return 0; 1103 } 1104 1105 static int mlx5e_calc_sq_stop_room(struct mlx5e_txqsq *sq, u8 log_sq_size) 1106 { 1107 int sq_size = 1 << log_sq_size; 1108 1109 sq->stop_room = mlx5e_tls_get_stop_room(sq); 1110 sq->stop_room += mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS); 1111 1112 if (WARN_ON(sq->stop_room >= sq_size)) { 1113 netdev_err(sq->channel->netdev, "Stop room %hu is bigger than the SQ size %d\n", 1114 sq->stop_room, sq_size); 1115 return -ENOSPC; 1116 } 1117 1118 return 0; 1119 } 1120 1121 static void mlx5e_tx_err_cqe_work(struct work_struct *recover_work); 1122 static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, 1123 int txq_ix, 1124 struct mlx5e_params *params, 1125 struct mlx5e_sq_param *param, 1126 struct mlx5e_txqsq *sq, 1127 int tc) 1128 { 1129 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1130 struct mlx5_core_dev *mdev = c->mdev; 1131 struct mlx5_wq_cyc *wq = &sq->wq; 1132 int err; 1133 1134 sq->pdev = c->pdev; 1135 sq->tstamp = c->tstamp; 1136 sq->clock = &mdev->clock; 1137 sq->mkey_be = c->mkey_be; 1138 sq->channel = c; 1139 sq->ch_ix = c->ix; 1140 sq->txq_ix = txq_ix; 1141 sq->uar_map = mdev->mlx5e_res.bfreg.map; 1142 sq->min_inline_mode = params->tx_min_inline_mode; 1143 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 1144 sq->stats = &c->priv->channel_stats[c->ix].sq[tc]; 1145 INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work); 1146 if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert)) 1147 set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state); 1148 if (MLX5_IPSEC_DEV(c->priv->mdev)) 1149 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state); 1150 if (mlx5_accel_is_tls_device(c->priv->mdev)) 1151 set_bit(MLX5E_SQ_STATE_TLS, &sq->state); 1152 err = mlx5e_calc_sq_stop_room(sq, params->log_sq_size); 1153 if (err) 1154 return err; 1155 1156 param->wq.db_numa_node = cpu_to_node(c->cpu); 1157 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1158 if (err) 1159 return err; 1160 wq->db = &wq->db[MLX5_SND_DBR]; 1161 1162 err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu)); 1163 if (err) 1164 goto err_sq_wq_destroy; 1165 1166 INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work); 1167 sq->dim.mode = params->tx_cq_moderation.cq_period_mode; 1168 1169 return 0; 1170 1171 err_sq_wq_destroy: 1172 mlx5_wq_destroy(&sq->wq_ctrl); 1173 1174 return err; 1175 } 1176 1177 static void mlx5e_free_txqsq(struct mlx5e_txqsq *sq) 1178 { 1179 mlx5e_free_txqsq_db(sq); 1180 mlx5_wq_destroy(&sq->wq_ctrl); 1181 } 1182 1183 struct mlx5e_create_sq_param { 1184 struct mlx5_wq_ctrl *wq_ctrl; 1185 u32 cqn; 1186 u32 tisn; 1187 u8 tis_lst_sz; 1188 u8 min_inline_mode; 1189 }; 1190 1191 static int mlx5e_create_sq(struct mlx5_core_dev *mdev, 1192 struct mlx5e_sq_param *param, 1193 struct mlx5e_create_sq_param *csp, 1194 u32 *sqn) 1195 { 1196 void *in; 1197 void *sqc; 1198 void *wq; 1199 int inlen; 1200 int err; 1201 1202 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + 1203 sizeof(u64) * csp->wq_ctrl->buf.npages; 1204 in = kvzalloc(inlen, GFP_KERNEL); 1205 if (!in) 1206 return -ENOMEM; 1207 1208 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 1209 wq = MLX5_ADDR_OF(sqc, sqc, wq); 1210 1211 memcpy(sqc, param->sqc, sizeof(param->sqc)); 1212 MLX5_SET(sqc, sqc, tis_lst_sz, csp->tis_lst_sz); 1213 MLX5_SET(sqc, sqc, tis_num_0, csp->tisn); 1214 MLX5_SET(sqc, sqc, cqn, csp->cqn); 1215 1216 if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT) 1217 MLX5_SET(sqc, sqc, min_wqe_inline_mode, csp->min_inline_mode); 1218 1219 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 1220 MLX5_SET(sqc, sqc, flush_in_error_en, 1); 1221 1222 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 1223 MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.bfreg.index); 1224 MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift - 1225 MLX5_ADAPTER_PAGE_SHIFT); 1226 MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma); 1227 1228 mlx5_fill_page_frag_array(&csp->wq_ctrl->buf, 1229 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 1230 1231 err = mlx5_core_create_sq(mdev, in, inlen, sqn); 1232 1233 kvfree(in); 1234 1235 return err; 1236 } 1237 1238 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn, 1239 struct mlx5e_modify_sq_param *p) 1240 { 1241 void *in; 1242 void *sqc; 1243 int inlen; 1244 int err; 1245 1246 inlen = MLX5_ST_SZ_BYTES(modify_sq_in); 1247 in = kvzalloc(inlen, GFP_KERNEL); 1248 if (!in) 1249 return -ENOMEM; 1250 1251 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 1252 1253 MLX5_SET(modify_sq_in, in, sq_state, p->curr_state); 1254 MLX5_SET(sqc, sqc, state, p->next_state); 1255 if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) { 1256 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1); 1257 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index); 1258 } 1259 1260 err = mlx5_core_modify_sq(mdev, sqn, in); 1261 1262 kvfree(in); 1263 1264 return err; 1265 } 1266 1267 static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn) 1268 { 1269 mlx5_core_destroy_sq(mdev, sqn); 1270 } 1271 1272 static int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, 1273 struct mlx5e_sq_param *param, 1274 struct mlx5e_create_sq_param *csp, 1275 u32 *sqn) 1276 { 1277 struct mlx5e_modify_sq_param msp = {0}; 1278 int err; 1279 1280 err = mlx5e_create_sq(mdev, param, csp, sqn); 1281 if (err) 1282 return err; 1283 1284 msp.curr_state = MLX5_SQC_STATE_RST; 1285 msp.next_state = MLX5_SQC_STATE_RDY; 1286 err = mlx5e_modify_sq(mdev, *sqn, &msp); 1287 if (err) 1288 mlx5e_destroy_sq(mdev, *sqn); 1289 1290 return err; 1291 } 1292 1293 static int mlx5e_set_sq_maxrate(struct net_device *dev, 1294 struct mlx5e_txqsq *sq, u32 rate); 1295 1296 static int mlx5e_open_txqsq(struct mlx5e_channel *c, 1297 u32 tisn, 1298 int txq_ix, 1299 struct mlx5e_params *params, 1300 struct mlx5e_sq_param *param, 1301 struct mlx5e_txqsq *sq, 1302 int tc) 1303 { 1304 struct mlx5e_create_sq_param csp = {}; 1305 u32 tx_rate; 1306 int err; 1307 1308 err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc); 1309 if (err) 1310 return err; 1311 1312 csp.tisn = tisn; 1313 csp.tis_lst_sz = 1; 1314 csp.cqn = sq->cq.mcq.cqn; 1315 csp.wq_ctrl = &sq->wq_ctrl; 1316 csp.min_inline_mode = sq->min_inline_mode; 1317 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn); 1318 if (err) 1319 goto err_free_txqsq; 1320 1321 tx_rate = c->priv->tx_rates[sq->txq_ix]; 1322 if (tx_rate) 1323 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate); 1324 1325 if (params->tx_dim_enabled) 1326 sq->state |= BIT(MLX5E_SQ_STATE_AM); 1327 1328 return 0; 1329 1330 err_free_txqsq: 1331 mlx5e_free_txqsq(sq); 1332 1333 return err; 1334 } 1335 1336 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq) 1337 { 1338 sq->txq = netdev_get_tx_queue(sq->channel->netdev, sq->txq_ix); 1339 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1340 netdev_tx_reset_queue(sq->txq); 1341 netif_tx_start_queue(sq->txq); 1342 } 1343 1344 void mlx5e_tx_disable_queue(struct netdev_queue *txq) 1345 { 1346 __netif_tx_lock_bh(txq); 1347 netif_tx_stop_queue(txq); 1348 __netif_tx_unlock_bh(txq); 1349 } 1350 1351 static void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq) 1352 { 1353 struct mlx5e_channel *c = sq->channel; 1354 struct mlx5_wq_cyc *wq = &sq->wq; 1355 1356 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1357 /* prevent netif_tx_wake_queue */ 1358 napi_synchronize(&c->napi); 1359 1360 mlx5e_tx_disable_queue(sq->txq); 1361 1362 /* last doorbell out, godspeed .. */ 1363 if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) { 1364 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1365 struct mlx5e_tx_wqe *nop; 1366 1367 sq->db.wqe_info[pi] = (struct mlx5e_tx_wqe_info) { 1368 .num_wqebbs = 1, 1369 }; 1370 1371 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc); 1372 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl); 1373 } 1374 } 1375 1376 static void mlx5e_close_txqsq(struct mlx5e_txqsq *sq) 1377 { 1378 struct mlx5e_channel *c = sq->channel; 1379 struct mlx5_core_dev *mdev = c->mdev; 1380 struct mlx5_rate_limit rl = {0}; 1381 1382 cancel_work_sync(&sq->dim.work); 1383 cancel_work_sync(&sq->recover_work); 1384 mlx5e_destroy_sq(mdev, sq->sqn); 1385 if (sq->rate_limit) { 1386 rl.rate = sq->rate_limit; 1387 mlx5_rl_remove_rate(mdev, &rl); 1388 } 1389 mlx5e_free_txqsq_descs(sq); 1390 mlx5e_free_txqsq(sq); 1391 } 1392 1393 static void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) 1394 { 1395 struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq, 1396 recover_work); 1397 1398 mlx5e_reporter_tx_err_cqe(sq); 1399 } 1400 1401 int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params, 1402 struct mlx5e_sq_param *param, struct mlx5e_icosq *sq) 1403 { 1404 struct mlx5e_create_sq_param csp = {}; 1405 int err; 1406 1407 err = mlx5e_alloc_icosq(c, param, sq); 1408 if (err) 1409 return err; 1410 1411 csp.cqn = sq->cq.mcq.cqn; 1412 csp.wq_ctrl = &sq->wq_ctrl; 1413 csp.min_inline_mode = params->tx_min_inline_mode; 1414 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn); 1415 if (err) 1416 goto err_free_icosq; 1417 1418 return 0; 1419 1420 err_free_icosq: 1421 mlx5e_free_icosq(sq); 1422 1423 return err; 1424 } 1425 1426 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq) 1427 { 1428 set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 1429 } 1430 1431 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq) 1432 { 1433 struct mlx5e_channel *c = icosq->channel; 1434 1435 clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 1436 napi_synchronize(&c->napi); 1437 } 1438 1439 void mlx5e_close_icosq(struct mlx5e_icosq *sq) 1440 { 1441 struct mlx5e_channel *c = sq->channel; 1442 1443 mlx5e_destroy_sq(c->mdev, sq->sqn); 1444 mlx5e_free_icosq(sq); 1445 } 1446 1447 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, 1448 struct mlx5e_sq_param *param, struct xdp_umem *umem, 1449 struct mlx5e_xdpsq *sq, bool is_redirect) 1450 { 1451 struct mlx5e_create_sq_param csp = {}; 1452 int err; 1453 1454 err = mlx5e_alloc_xdpsq(c, params, umem, param, sq, is_redirect); 1455 if (err) 1456 return err; 1457 1458 csp.tis_lst_sz = 1; 1459 csp.tisn = c->priv->tisn[c->lag_port][0]; /* tc = 0 */ 1460 csp.cqn = sq->cq.mcq.cqn; 1461 csp.wq_ctrl = &sq->wq_ctrl; 1462 csp.min_inline_mode = sq->min_inline_mode; 1463 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1464 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn); 1465 if (err) 1466 goto err_free_xdpsq; 1467 1468 mlx5e_set_xmit_fp(sq, param->is_mpw); 1469 1470 if (!param->is_mpw) { 1471 unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT; 1472 unsigned int inline_hdr_sz = 0; 1473 int i; 1474 1475 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) { 1476 inline_hdr_sz = MLX5E_XDP_MIN_INLINE; 1477 ds_cnt++; 1478 } 1479 1480 /* Pre initialize fixed WQE fields */ 1481 for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) { 1482 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, i); 1483 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 1484 struct mlx5_wqe_eth_seg *eseg = &wqe->eth; 1485 struct mlx5_wqe_data_seg *dseg; 1486 1487 sq->db.wqe_info[i] = (struct mlx5e_xdp_wqe_info) { 1488 .num_wqebbs = 1, 1489 .num_pkts = 1, 1490 }; 1491 1492 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 1493 eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz); 1494 1495 dseg = (struct mlx5_wqe_data_seg *)cseg + (ds_cnt - 1); 1496 dseg->lkey = sq->mkey_be; 1497 } 1498 } 1499 1500 return 0; 1501 1502 err_free_xdpsq: 1503 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1504 mlx5e_free_xdpsq(sq); 1505 1506 return err; 1507 } 1508 1509 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq) 1510 { 1511 struct mlx5e_channel *c = sq->channel; 1512 1513 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1514 napi_synchronize(&c->napi); 1515 1516 mlx5e_destroy_sq(c->mdev, sq->sqn); 1517 mlx5e_free_xdpsq_descs(sq); 1518 mlx5e_free_xdpsq(sq); 1519 } 1520 1521 static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev, 1522 struct mlx5e_cq_param *param, 1523 struct mlx5e_cq *cq) 1524 { 1525 struct mlx5_core_cq *mcq = &cq->mcq; 1526 int eqn_not_used; 1527 unsigned int irqn; 1528 int err; 1529 u32 i; 1530 1531 err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn); 1532 if (err) 1533 return err; 1534 1535 err = mlx5_cqwq_create(mdev, ¶m->wq, param->cqc, &cq->wq, 1536 &cq->wq_ctrl); 1537 if (err) 1538 return err; 1539 1540 mcq->cqe_sz = 64; 1541 mcq->set_ci_db = cq->wq_ctrl.db.db; 1542 mcq->arm_db = cq->wq_ctrl.db.db + 1; 1543 *mcq->set_ci_db = 0; 1544 *mcq->arm_db = 0; 1545 mcq->vector = param->eq_ix; 1546 mcq->comp = mlx5e_completion_event; 1547 mcq->event = mlx5e_cq_error_event; 1548 mcq->irqn = irqn; 1549 1550 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { 1551 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); 1552 1553 cqe->op_own = 0xf1; 1554 } 1555 1556 cq->mdev = mdev; 1557 1558 return 0; 1559 } 1560 1561 static int mlx5e_alloc_cq(struct mlx5e_channel *c, 1562 struct mlx5e_cq_param *param, 1563 struct mlx5e_cq *cq) 1564 { 1565 struct mlx5_core_dev *mdev = c->priv->mdev; 1566 int err; 1567 1568 param->wq.buf_numa_node = cpu_to_node(c->cpu); 1569 param->wq.db_numa_node = cpu_to_node(c->cpu); 1570 param->eq_ix = c->ix; 1571 1572 err = mlx5e_alloc_cq_common(mdev, param, cq); 1573 1574 cq->napi = &c->napi; 1575 cq->channel = c; 1576 1577 return err; 1578 } 1579 1580 static void mlx5e_free_cq(struct mlx5e_cq *cq) 1581 { 1582 mlx5_wq_destroy(&cq->wq_ctrl); 1583 } 1584 1585 static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param) 1586 { 1587 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 1588 struct mlx5_core_dev *mdev = cq->mdev; 1589 struct mlx5_core_cq *mcq = &cq->mcq; 1590 1591 void *in; 1592 void *cqc; 1593 int inlen; 1594 unsigned int irqn_not_used; 1595 int eqn; 1596 int err; 1597 1598 err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used); 1599 if (err) 1600 return err; 1601 1602 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 1603 sizeof(u64) * cq->wq_ctrl.buf.npages; 1604 in = kvzalloc(inlen, GFP_KERNEL); 1605 if (!in) 1606 return -ENOMEM; 1607 1608 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 1609 1610 memcpy(cqc, param->cqc, sizeof(param->cqc)); 1611 1612 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, 1613 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas)); 1614 1615 MLX5_SET(cqc, cqc, cq_period_mode, param->cq_period_mode); 1616 MLX5_SET(cqc, cqc, c_eqn, eqn); 1617 MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); 1618 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - 1619 MLX5_ADAPTER_PAGE_SHIFT); 1620 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); 1621 1622 err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out)); 1623 1624 kvfree(in); 1625 1626 if (err) 1627 return err; 1628 1629 mlx5e_cq_arm(cq); 1630 1631 return 0; 1632 } 1633 1634 static void mlx5e_destroy_cq(struct mlx5e_cq *cq) 1635 { 1636 mlx5_core_destroy_cq(cq->mdev, &cq->mcq); 1637 } 1638 1639 int mlx5e_open_cq(struct mlx5e_channel *c, struct dim_cq_moder moder, 1640 struct mlx5e_cq_param *param, struct mlx5e_cq *cq) 1641 { 1642 struct mlx5_core_dev *mdev = c->mdev; 1643 int err; 1644 1645 err = mlx5e_alloc_cq(c, param, cq); 1646 if (err) 1647 return err; 1648 1649 err = mlx5e_create_cq(cq, param); 1650 if (err) 1651 goto err_free_cq; 1652 1653 if (MLX5_CAP_GEN(mdev, cq_moderation)) 1654 mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts); 1655 return 0; 1656 1657 err_free_cq: 1658 mlx5e_free_cq(cq); 1659 1660 return err; 1661 } 1662 1663 void mlx5e_close_cq(struct mlx5e_cq *cq) 1664 { 1665 mlx5e_destroy_cq(cq); 1666 mlx5e_free_cq(cq); 1667 } 1668 1669 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c, 1670 struct mlx5e_params *params, 1671 struct mlx5e_channel_param *cparam) 1672 { 1673 int err; 1674 int tc; 1675 1676 for (tc = 0; tc < c->num_tc; tc++) { 1677 err = mlx5e_open_cq(c, params->tx_cq_moderation, 1678 &cparam->tx_cq, &c->sq[tc].cq); 1679 if (err) 1680 goto err_close_tx_cqs; 1681 } 1682 1683 return 0; 1684 1685 err_close_tx_cqs: 1686 for (tc--; tc >= 0; tc--) 1687 mlx5e_close_cq(&c->sq[tc].cq); 1688 1689 return err; 1690 } 1691 1692 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c) 1693 { 1694 int tc; 1695 1696 for (tc = 0; tc < c->num_tc; tc++) 1697 mlx5e_close_cq(&c->sq[tc].cq); 1698 } 1699 1700 static int mlx5e_open_sqs(struct mlx5e_channel *c, 1701 struct mlx5e_params *params, 1702 struct mlx5e_channel_param *cparam) 1703 { 1704 int err, tc; 1705 1706 for (tc = 0; tc < params->num_tc; tc++) { 1707 int txq_ix = c->ix + tc * params->num_channels; 1708 1709 err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix, 1710 params, &cparam->sq, &c->sq[tc], tc); 1711 if (err) 1712 goto err_close_sqs; 1713 } 1714 1715 return 0; 1716 1717 err_close_sqs: 1718 for (tc--; tc >= 0; tc--) 1719 mlx5e_close_txqsq(&c->sq[tc]); 1720 1721 return err; 1722 } 1723 1724 static void mlx5e_close_sqs(struct mlx5e_channel *c) 1725 { 1726 int tc; 1727 1728 for (tc = 0; tc < c->num_tc; tc++) 1729 mlx5e_close_txqsq(&c->sq[tc]); 1730 } 1731 1732 static int mlx5e_set_sq_maxrate(struct net_device *dev, 1733 struct mlx5e_txqsq *sq, u32 rate) 1734 { 1735 struct mlx5e_priv *priv = netdev_priv(dev); 1736 struct mlx5_core_dev *mdev = priv->mdev; 1737 struct mlx5e_modify_sq_param msp = {0}; 1738 struct mlx5_rate_limit rl = {0}; 1739 u16 rl_index = 0; 1740 int err; 1741 1742 if (rate == sq->rate_limit) 1743 /* nothing to do */ 1744 return 0; 1745 1746 if (sq->rate_limit) { 1747 rl.rate = sq->rate_limit; 1748 /* remove current rl index to free space to next ones */ 1749 mlx5_rl_remove_rate(mdev, &rl); 1750 } 1751 1752 sq->rate_limit = 0; 1753 1754 if (rate) { 1755 rl.rate = rate; 1756 err = mlx5_rl_add_rate(mdev, &rl_index, &rl); 1757 if (err) { 1758 netdev_err(dev, "Failed configuring rate %u: %d\n", 1759 rate, err); 1760 return err; 1761 } 1762 } 1763 1764 msp.curr_state = MLX5_SQC_STATE_RDY; 1765 msp.next_state = MLX5_SQC_STATE_RDY; 1766 msp.rl_index = rl_index; 1767 msp.rl_update = true; 1768 err = mlx5e_modify_sq(mdev, sq->sqn, &msp); 1769 if (err) { 1770 netdev_err(dev, "Failed configuring rate %u: %d\n", 1771 rate, err); 1772 /* remove the rate from the table */ 1773 if (rate) 1774 mlx5_rl_remove_rate(mdev, &rl); 1775 return err; 1776 } 1777 1778 sq->rate_limit = rate; 1779 return 0; 1780 } 1781 1782 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate) 1783 { 1784 struct mlx5e_priv *priv = netdev_priv(dev); 1785 struct mlx5_core_dev *mdev = priv->mdev; 1786 struct mlx5e_txqsq *sq = priv->txq2sq[index]; 1787 int err = 0; 1788 1789 if (!mlx5_rl_is_supported(mdev)) { 1790 netdev_err(dev, "Rate limiting is not supported on this device\n"); 1791 return -EINVAL; 1792 } 1793 1794 /* rate is given in Mb/sec, HW config is in Kb/sec */ 1795 rate = rate << 10; 1796 1797 /* Check whether rate in valid range, 0 is always valid */ 1798 if (rate && !mlx5_rl_is_in_range(mdev, rate)) { 1799 netdev_err(dev, "TX rate %u, is not in range\n", rate); 1800 return -ERANGE; 1801 } 1802 1803 mutex_lock(&priv->state_lock); 1804 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 1805 err = mlx5e_set_sq_maxrate(dev, sq, rate); 1806 if (!err) 1807 priv->tx_rates[index] = rate; 1808 mutex_unlock(&priv->state_lock); 1809 1810 return err; 1811 } 1812 1813 static int mlx5e_open_queues(struct mlx5e_channel *c, 1814 struct mlx5e_params *params, 1815 struct mlx5e_channel_param *cparam) 1816 { 1817 struct dim_cq_moder icocq_moder = {0, 0}; 1818 int err; 1819 1820 err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq); 1821 if (err) 1822 return err; 1823 1824 err = mlx5e_open_tx_cqs(c, params, cparam); 1825 if (err) 1826 goto err_close_icosq_cq; 1827 1828 err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->tx_cq, &c->xdpsq.cq); 1829 if (err) 1830 goto err_close_tx_cqs; 1831 1832 err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq); 1833 if (err) 1834 goto err_close_xdp_tx_cqs; 1835 1836 /* XDP SQ CQ params are same as normal TXQ sq CQ params */ 1837 err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation, 1838 &cparam->tx_cq, &c->rq_xdpsq.cq) : 0; 1839 if (err) 1840 goto err_close_rx_cq; 1841 1842 napi_enable(&c->napi); 1843 1844 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq); 1845 if (err) 1846 goto err_disable_napi; 1847 1848 err = mlx5e_open_sqs(c, params, cparam); 1849 if (err) 1850 goto err_close_icosq; 1851 1852 if (c->xdp) { 1853 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, 1854 &c->rq_xdpsq, false); 1855 if (err) 1856 goto err_close_sqs; 1857 } 1858 1859 err = mlx5e_open_rq(c, params, &cparam->rq, NULL, NULL, &c->rq); 1860 if (err) 1861 goto err_close_xdp_sq; 1862 1863 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, &c->xdpsq, true); 1864 if (err) 1865 goto err_close_rq; 1866 1867 return 0; 1868 1869 err_close_rq: 1870 mlx5e_close_rq(&c->rq); 1871 1872 err_close_xdp_sq: 1873 if (c->xdp) 1874 mlx5e_close_xdpsq(&c->rq_xdpsq); 1875 1876 err_close_sqs: 1877 mlx5e_close_sqs(c); 1878 1879 err_close_icosq: 1880 mlx5e_close_icosq(&c->icosq); 1881 1882 err_disable_napi: 1883 napi_disable(&c->napi); 1884 1885 if (c->xdp) 1886 mlx5e_close_cq(&c->rq_xdpsq.cq); 1887 1888 err_close_rx_cq: 1889 mlx5e_close_cq(&c->rq.cq); 1890 1891 err_close_xdp_tx_cqs: 1892 mlx5e_close_cq(&c->xdpsq.cq); 1893 1894 err_close_tx_cqs: 1895 mlx5e_close_tx_cqs(c); 1896 1897 err_close_icosq_cq: 1898 mlx5e_close_cq(&c->icosq.cq); 1899 1900 return err; 1901 } 1902 1903 static void mlx5e_close_queues(struct mlx5e_channel *c) 1904 { 1905 mlx5e_close_xdpsq(&c->xdpsq); 1906 mlx5e_close_rq(&c->rq); 1907 if (c->xdp) 1908 mlx5e_close_xdpsq(&c->rq_xdpsq); 1909 mlx5e_close_sqs(c); 1910 mlx5e_close_icosq(&c->icosq); 1911 napi_disable(&c->napi); 1912 if (c->xdp) 1913 mlx5e_close_cq(&c->rq_xdpsq.cq); 1914 mlx5e_close_cq(&c->rq.cq); 1915 mlx5e_close_cq(&c->xdpsq.cq); 1916 mlx5e_close_tx_cqs(c); 1917 mlx5e_close_cq(&c->icosq.cq); 1918 } 1919 1920 static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix) 1921 { 1922 u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id); 1923 1924 return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev); 1925 } 1926 1927 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, 1928 struct mlx5e_params *params, 1929 struct mlx5e_channel_param *cparam, 1930 struct xdp_umem *umem, 1931 struct mlx5e_channel **cp) 1932 { 1933 int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(priv->mdev, ix)); 1934 struct net_device *netdev = priv->netdev; 1935 struct mlx5e_xsk_param xsk; 1936 struct mlx5e_channel *c; 1937 unsigned int irq; 1938 int err; 1939 int eqn; 1940 1941 err = mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq); 1942 if (err) 1943 return err; 1944 1945 c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu)); 1946 if (!c) 1947 return -ENOMEM; 1948 1949 c->priv = priv; 1950 c->mdev = priv->mdev; 1951 c->tstamp = &priv->tstamp; 1952 c->ix = ix; 1953 c->cpu = cpu; 1954 c->pdev = priv->mdev->device; 1955 c->netdev = priv->netdev; 1956 c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key); 1957 c->num_tc = params->num_tc; 1958 c->xdp = !!params->xdp_prog; 1959 c->stats = &priv->channel_stats[ix].ch; 1960 c->irq_desc = irq_to_desc(irq); 1961 c->lag_port = mlx5e_enumerate_lag_port(priv->mdev, ix); 1962 1963 netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64); 1964 1965 err = mlx5e_open_queues(c, params, cparam); 1966 if (unlikely(err)) 1967 goto err_napi_del; 1968 1969 if (umem) { 1970 mlx5e_build_xsk_param(umem, &xsk); 1971 err = mlx5e_open_xsk(priv, params, &xsk, umem, c); 1972 if (unlikely(err)) 1973 goto err_close_queues; 1974 } 1975 1976 *cp = c; 1977 1978 return 0; 1979 1980 err_close_queues: 1981 mlx5e_close_queues(c); 1982 1983 err_napi_del: 1984 netif_napi_del(&c->napi); 1985 1986 kvfree(c); 1987 1988 return err; 1989 } 1990 1991 static void mlx5e_activate_channel(struct mlx5e_channel *c) 1992 { 1993 int tc; 1994 1995 for (tc = 0; tc < c->num_tc; tc++) 1996 mlx5e_activate_txqsq(&c->sq[tc]); 1997 mlx5e_activate_icosq(&c->icosq); 1998 mlx5e_activate_rq(&c->rq); 1999 2000 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2001 mlx5e_activate_xsk(c); 2002 } 2003 2004 static void mlx5e_deactivate_channel(struct mlx5e_channel *c) 2005 { 2006 int tc; 2007 2008 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2009 mlx5e_deactivate_xsk(c); 2010 2011 mlx5e_deactivate_rq(&c->rq); 2012 mlx5e_deactivate_icosq(&c->icosq); 2013 for (tc = 0; tc < c->num_tc; tc++) 2014 mlx5e_deactivate_txqsq(&c->sq[tc]); 2015 } 2016 2017 static void mlx5e_close_channel(struct mlx5e_channel *c) 2018 { 2019 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2020 mlx5e_close_xsk(c); 2021 mlx5e_close_queues(c); 2022 netif_napi_del(&c->napi); 2023 2024 kvfree(c); 2025 } 2026 2027 #define DEFAULT_FRAG_SIZE (2048) 2028 2029 static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, 2030 struct mlx5e_params *params, 2031 struct mlx5e_xsk_param *xsk, 2032 struct mlx5e_rq_frags_info *info) 2033 { 2034 u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu); 2035 int frag_size_max = DEFAULT_FRAG_SIZE; 2036 u32 buf_size = 0; 2037 int i; 2038 2039 #ifdef CONFIG_MLX5_EN_IPSEC 2040 if (MLX5_IPSEC_DEV(mdev)) 2041 byte_count += MLX5E_METADATA_ETHER_LEN; 2042 #endif 2043 2044 if (mlx5e_rx_is_linear_skb(params, xsk)) { 2045 int frag_stride; 2046 2047 frag_stride = mlx5e_rx_get_linear_frag_sz(params, xsk); 2048 frag_stride = roundup_pow_of_two(frag_stride); 2049 2050 info->arr[0].frag_size = byte_count; 2051 info->arr[0].frag_stride = frag_stride; 2052 info->num_frags = 1; 2053 info->wqe_bulk = PAGE_SIZE / frag_stride; 2054 goto out; 2055 } 2056 2057 if (byte_count > PAGE_SIZE + 2058 (MLX5E_MAX_RX_FRAGS - 1) * frag_size_max) 2059 frag_size_max = PAGE_SIZE; 2060 2061 i = 0; 2062 while (buf_size < byte_count) { 2063 int frag_size = byte_count - buf_size; 2064 2065 if (i < MLX5E_MAX_RX_FRAGS - 1) 2066 frag_size = min(frag_size, frag_size_max); 2067 2068 info->arr[i].frag_size = frag_size; 2069 info->arr[i].frag_stride = roundup_pow_of_two(frag_size); 2070 2071 buf_size += frag_size; 2072 i++; 2073 } 2074 info->num_frags = i; 2075 /* number of different wqes sharing a page */ 2076 info->wqe_bulk = 1 + (info->num_frags % 2); 2077 2078 out: 2079 info->wqe_bulk = max_t(u8, info->wqe_bulk, 8); 2080 info->log_num_frags = order_base_2(info->num_frags); 2081 } 2082 2083 static inline u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs) 2084 { 2085 int sz = sizeof(struct mlx5_wqe_data_seg) * ndsegs; 2086 2087 switch (wq_type) { 2088 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 2089 sz += sizeof(struct mlx5e_rx_wqe_ll); 2090 break; 2091 default: /* MLX5_WQ_TYPE_CYCLIC */ 2092 sz += sizeof(struct mlx5e_rx_wqe_cyc); 2093 } 2094 2095 return order_base_2(sz); 2096 } 2097 2098 static u8 mlx5e_get_rq_log_wq_sz(void *rqc) 2099 { 2100 void *wq = MLX5_ADDR_OF(rqc, rqc, wq); 2101 2102 return MLX5_GET(wq, wq, log_wq_sz); 2103 } 2104 2105 void mlx5e_build_rq_param(struct mlx5e_priv *priv, 2106 struct mlx5e_params *params, 2107 struct mlx5e_xsk_param *xsk, 2108 struct mlx5e_rq_param *param) 2109 { 2110 struct mlx5_core_dev *mdev = priv->mdev; 2111 void *rqc = param->rqc; 2112 void *wq = MLX5_ADDR_OF(rqc, rqc, wq); 2113 int ndsegs = 1; 2114 2115 switch (params->rq_wq_type) { 2116 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 2117 MLX5_SET(wq, wq, log_wqe_num_of_strides, 2118 mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk) - 2119 MLX5_MPWQE_LOG_NUM_STRIDES_BASE); 2120 MLX5_SET(wq, wq, log_wqe_stride_size, 2121 mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk) - 2122 MLX5_MPWQE_LOG_STRIDE_SZ_BASE); 2123 MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params, xsk)); 2124 break; 2125 default: /* MLX5_WQ_TYPE_CYCLIC */ 2126 MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames); 2127 mlx5e_build_rq_frags_info(mdev, params, xsk, ¶m->frags_info); 2128 ndsegs = param->frags_info.num_frags; 2129 } 2130 2131 MLX5_SET(wq, wq, wq_type, params->rq_wq_type); 2132 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN); 2133 MLX5_SET(wq, wq, log_wq_stride, 2134 mlx5e_get_rqwq_log_stride(params->rq_wq_type, ndsegs)); 2135 MLX5_SET(wq, wq, pd, mdev->mlx5e_res.pdn); 2136 MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter); 2137 MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable); 2138 MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en); 2139 2140 param->wq.buf_numa_node = dev_to_node(mdev->device); 2141 } 2142 2143 static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv, 2144 struct mlx5e_rq_param *param) 2145 { 2146 struct mlx5_core_dev *mdev = priv->mdev; 2147 void *rqc = param->rqc; 2148 void *wq = MLX5_ADDR_OF(rqc, rqc, wq); 2149 2150 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 2151 MLX5_SET(wq, wq, log_wq_stride, 2152 mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1)); 2153 MLX5_SET(rqc, rqc, counter_set_id, priv->drop_rq_q_counter); 2154 2155 param->wq.buf_numa_node = dev_to_node(mdev->device); 2156 } 2157 2158 void mlx5e_build_sq_param_common(struct mlx5e_priv *priv, 2159 struct mlx5e_sq_param *param) 2160 { 2161 void *sqc = param->sqc; 2162 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 2163 2164 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB)); 2165 MLX5_SET(wq, wq, pd, priv->mdev->mlx5e_res.pdn); 2166 2167 param->wq.buf_numa_node = dev_to_node(priv->mdev->device); 2168 } 2169 2170 static void mlx5e_build_sq_param(struct mlx5e_priv *priv, 2171 struct mlx5e_params *params, 2172 struct mlx5e_sq_param *param) 2173 { 2174 void *sqc = param->sqc; 2175 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 2176 bool allow_swp; 2177 2178 allow_swp = mlx5_geneve_tx_allowed(priv->mdev) || 2179 !!MLX5_IPSEC_DEV(priv->mdev); 2180 mlx5e_build_sq_param_common(priv, param); 2181 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); 2182 MLX5_SET(sqc, sqc, allow_swp, allow_swp); 2183 } 2184 2185 static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv, 2186 struct mlx5e_cq_param *param) 2187 { 2188 void *cqc = param->cqc; 2189 2190 MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index); 2191 if (MLX5_CAP_GEN(priv->mdev, cqe_128_always) && cache_line_size() >= 128) 2192 MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD); 2193 } 2194 2195 void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv, 2196 struct mlx5e_params *params, 2197 struct mlx5e_xsk_param *xsk, 2198 struct mlx5e_cq_param *param) 2199 { 2200 struct mlx5_core_dev *mdev = priv->mdev; 2201 void *cqc = param->cqc; 2202 u8 log_cq_size; 2203 2204 switch (params->rq_wq_type) { 2205 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 2206 log_cq_size = mlx5e_mpwqe_get_log_rq_size(params, xsk) + 2207 mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); 2208 break; 2209 default: /* MLX5_WQ_TYPE_CYCLIC */ 2210 log_cq_size = params->log_rq_mtu_frames; 2211 } 2212 2213 MLX5_SET(cqc, cqc, log_cq_size, log_cq_size); 2214 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { 2215 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM); 2216 MLX5_SET(cqc, cqc, cqe_comp_en, 1); 2217 } 2218 2219 mlx5e_build_common_cq_param(priv, param); 2220 param->cq_period_mode = params->rx_cq_moderation.cq_period_mode; 2221 } 2222 2223 void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv, 2224 struct mlx5e_params *params, 2225 struct mlx5e_cq_param *param) 2226 { 2227 void *cqc = param->cqc; 2228 2229 MLX5_SET(cqc, cqc, log_cq_size, params->log_sq_size); 2230 2231 mlx5e_build_common_cq_param(priv, param); 2232 param->cq_period_mode = params->tx_cq_moderation.cq_period_mode; 2233 } 2234 2235 void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv, 2236 u8 log_wq_size, 2237 struct mlx5e_cq_param *param) 2238 { 2239 void *cqc = param->cqc; 2240 2241 MLX5_SET(cqc, cqc, log_cq_size, log_wq_size); 2242 2243 mlx5e_build_common_cq_param(priv, param); 2244 2245 param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 2246 } 2247 2248 void mlx5e_build_icosq_param(struct mlx5e_priv *priv, 2249 u8 log_wq_size, 2250 struct mlx5e_sq_param *param) 2251 { 2252 void *sqc = param->sqc; 2253 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 2254 2255 mlx5e_build_sq_param_common(priv, param); 2256 2257 MLX5_SET(wq, wq, log_wq_sz, log_wq_size); 2258 MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq)); 2259 } 2260 2261 void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv, 2262 struct mlx5e_params *params, 2263 struct mlx5e_sq_param *param) 2264 { 2265 void *sqc = param->sqc; 2266 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 2267 2268 mlx5e_build_sq_param_common(priv, param); 2269 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); 2270 param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE); 2271 } 2272 2273 static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params, 2274 struct mlx5e_rq_param *rqp) 2275 { 2276 switch (params->rq_wq_type) { 2277 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 2278 return order_base_2(MLX5E_UMR_WQEBBS) + 2279 mlx5e_get_rq_log_wq_sz(rqp->rqc); 2280 default: /* MLX5_WQ_TYPE_CYCLIC */ 2281 return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; 2282 } 2283 } 2284 2285 static void mlx5e_build_channel_param(struct mlx5e_priv *priv, 2286 struct mlx5e_params *params, 2287 struct mlx5e_channel_param *cparam) 2288 { 2289 u8 icosq_log_wq_sz; 2290 2291 mlx5e_build_rq_param(priv, params, NULL, &cparam->rq); 2292 2293 icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq); 2294 2295 mlx5e_build_sq_param(priv, params, &cparam->sq); 2296 mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq); 2297 mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq); 2298 mlx5e_build_rx_cq_param(priv, params, NULL, &cparam->rx_cq); 2299 mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq); 2300 mlx5e_build_ico_cq_param(priv, icosq_log_wq_sz, &cparam->icosq_cq); 2301 } 2302 2303 int mlx5e_open_channels(struct mlx5e_priv *priv, 2304 struct mlx5e_channels *chs) 2305 { 2306 struct mlx5e_channel_param *cparam; 2307 int err = -ENOMEM; 2308 int i; 2309 2310 chs->num = chs->params.num_channels; 2311 2312 chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL); 2313 cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL); 2314 if (!chs->c || !cparam) 2315 goto err_free; 2316 2317 mlx5e_build_channel_param(priv, &chs->params, cparam); 2318 for (i = 0; i < chs->num; i++) { 2319 struct xdp_umem *umem = NULL; 2320 2321 if (chs->params.xdp_prog) 2322 umem = mlx5e_xsk_get_umem(&chs->params, chs->params.xsk, i); 2323 2324 err = mlx5e_open_channel(priv, i, &chs->params, cparam, umem, &chs->c[i]); 2325 if (err) 2326 goto err_close_channels; 2327 } 2328 2329 mlx5e_health_channels_update(priv); 2330 kvfree(cparam); 2331 return 0; 2332 2333 err_close_channels: 2334 for (i--; i >= 0; i--) 2335 mlx5e_close_channel(chs->c[i]); 2336 2337 err_free: 2338 kfree(chs->c); 2339 kvfree(cparam); 2340 chs->num = 0; 2341 return err; 2342 } 2343 2344 static void mlx5e_activate_channels(struct mlx5e_channels *chs) 2345 { 2346 int i; 2347 2348 for (i = 0; i < chs->num; i++) 2349 mlx5e_activate_channel(chs->c[i]); 2350 } 2351 2352 #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */ 2353 2354 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs) 2355 { 2356 int err = 0; 2357 int i; 2358 2359 for (i = 0; i < chs->num; i++) { 2360 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT; 2361 2362 err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout); 2363 2364 /* Don't wait on the XSK RQ, because the newer xdpsock sample 2365 * doesn't provide any Fill Ring entries at the setup stage. 2366 */ 2367 } 2368 2369 return err ? -ETIMEDOUT : 0; 2370 } 2371 2372 static void mlx5e_deactivate_channels(struct mlx5e_channels *chs) 2373 { 2374 int i; 2375 2376 for (i = 0; i < chs->num; i++) 2377 mlx5e_deactivate_channel(chs->c[i]); 2378 } 2379 2380 void mlx5e_close_channels(struct mlx5e_channels *chs) 2381 { 2382 int i; 2383 2384 for (i = 0; i < chs->num; i++) 2385 mlx5e_close_channel(chs->c[i]); 2386 2387 kfree(chs->c); 2388 chs->num = 0; 2389 } 2390 2391 static int 2392 mlx5e_create_rqt(struct mlx5e_priv *priv, int sz, struct mlx5e_rqt *rqt) 2393 { 2394 struct mlx5_core_dev *mdev = priv->mdev; 2395 void *rqtc; 2396 int inlen; 2397 int err; 2398 u32 *in; 2399 int i; 2400 2401 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz; 2402 in = kvzalloc(inlen, GFP_KERNEL); 2403 if (!in) 2404 return -ENOMEM; 2405 2406 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 2407 2408 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz); 2409 MLX5_SET(rqtc, rqtc, rqt_max_size, sz); 2410 2411 for (i = 0; i < sz; i++) 2412 MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn); 2413 2414 err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn); 2415 if (!err) 2416 rqt->enabled = true; 2417 2418 kvfree(in); 2419 return err; 2420 } 2421 2422 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt) 2423 { 2424 rqt->enabled = false; 2425 mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn); 2426 } 2427 2428 int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv) 2429 { 2430 struct mlx5e_rqt *rqt = &priv->indir_rqt; 2431 int err; 2432 2433 err = mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, rqt); 2434 if (err) 2435 mlx5_core_warn(priv->mdev, "create indirect rqts failed, %d\n", err); 2436 return err; 2437 } 2438 2439 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs) 2440 { 2441 int err; 2442 int ix; 2443 2444 for (ix = 0; ix < priv->max_nch; ix++) { 2445 err = mlx5e_create_rqt(priv, 1 /*size */, &tirs[ix].rqt); 2446 if (unlikely(err)) 2447 goto err_destroy_rqts; 2448 } 2449 2450 return 0; 2451 2452 err_destroy_rqts: 2453 mlx5_core_warn(priv->mdev, "create rqts failed, %d\n", err); 2454 for (ix--; ix >= 0; ix--) 2455 mlx5e_destroy_rqt(priv, &tirs[ix].rqt); 2456 2457 return err; 2458 } 2459 2460 void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs) 2461 { 2462 int i; 2463 2464 for (i = 0; i < priv->max_nch; i++) 2465 mlx5e_destroy_rqt(priv, &tirs[i].rqt); 2466 } 2467 2468 static int mlx5e_rx_hash_fn(int hfunc) 2469 { 2470 return (hfunc == ETH_RSS_HASH_TOP) ? 2471 MLX5_RX_HASH_FN_TOEPLITZ : 2472 MLX5_RX_HASH_FN_INVERTED_XOR8; 2473 } 2474 2475 int mlx5e_bits_invert(unsigned long a, int size) 2476 { 2477 int inv = 0; 2478 int i; 2479 2480 for (i = 0; i < size; i++) 2481 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i; 2482 2483 return inv; 2484 } 2485 2486 static void mlx5e_fill_rqt_rqns(struct mlx5e_priv *priv, int sz, 2487 struct mlx5e_redirect_rqt_param rrp, void *rqtc) 2488 { 2489 int i; 2490 2491 for (i = 0; i < sz; i++) { 2492 u32 rqn; 2493 2494 if (rrp.is_rss) { 2495 int ix = i; 2496 2497 if (rrp.rss.hfunc == ETH_RSS_HASH_XOR) 2498 ix = mlx5e_bits_invert(i, ilog2(sz)); 2499 2500 ix = priv->rss_params.indirection_rqt[ix]; 2501 rqn = rrp.rss.channels->c[ix]->rq.rqn; 2502 } else { 2503 rqn = rrp.rqn; 2504 } 2505 MLX5_SET(rqtc, rqtc, rq_num[i], rqn); 2506 } 2507 } 2508 2509 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, 2510 struct mlx5e_redirect_rqt_param rrp) 2511 { 2512 struct mlx5_core_dev *mdev = priv->mdev; 2513 void *rqtc; 2514 int inlen; 2515 u32 *in; 2516 int err; 2517 2518 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz; 2519 in = kvzalloc(inlen, GFP_KERNEL); 2520 if (!in) 2521 return -ENOMEM; 2522 2523 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 2524 2525 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz); 2526 MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1); 2527 mlx5e_fill_rqt_rqns(priv, sz, rrp, rqtc); 2528 err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen); 2529 2530 kvfree(in); 2531 return err; 2532 } 2533 2534 static u32 mlx5e_get_direct_rqn(struct mlx5e_priv *priv, int ix, 2535 struct mlx5e_redirect_rqt_param rrp) 2536 { 2537 if (!rrp.is_rss) 2538 return rrp.rqn; 2539 2540 if (ix >= rrp.rss.channels->num) 2541 return priv->drop_rq.rqn; 2542 2543 return rrp.rss.channels->c[ix]->rq.rqn; 2544 } 2545 2546 static void mlx5e_redirect_rqts(struct mlx5e_priv *priv, 2547 struct mlx5e_redirect_rqt_param rrp) 2548 { 2549 u32 rqtn; 2550 int ix; 2551 2552 if (priv->indir_rqt.enabled) { 2553 /* RSS RQ table */ 2554 rqtn = priv->indir_rqt.rqtn; 2555 mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, rrp); 2556 } 2557 2558 for (ix = 0; ix < priv->max_nch; ix++) { 2559 struct mlx5e_redirect_rqt_param direct_rrp = { 2560 .is_rss = false, 2561 { 2562 .rqn = mlx5e_get_direct_rqn(priv, ix, rrp) 2563 }, 2564 }; 2565 2566 /* Direct RQ Tables */ 2567 if (!priv->direct_tir[ix].rqt.enabled) 2568 continue; 2569 2570 rqtn = priv->direct_tir[ix].rqt.rqtn; 2571 mlx5e_redirect_rqt(priv, rqtn, 1, direct_rrp); 2572 } 2573 } 2574 2575 static void mlx5e_redirect_rqts_to_channels(struct mlx5e_priv *priv, 2576 struct mlx5e_channels *chs) 2577 { 2578 struct mlx5e_redirect_rqt_param rrp = { 2579 .is_rss = true, 2580 { 2581 .rss = { 2582 .channels = chs, 2583 .hfunc = priv->rss_params.hfunc, 2584 } 2585 }, 2586 }; 2587 2588 mlx5e_redirect_rqts(priv, rrp); 2589 } 2590 2591 static void mlx5e_redirect_rqts_to_drop(struct mlx5e_priv *priv) 2592 { 2593 struct mlx5e_redirect_rqt_param drop_rrp = { 2594 .is_rss = false, 2595 { 2596 .rqn = priv->drop_rq.rqn, 2597 }, 2598 }; 2599 2600 mlx5e_redirect_rqts(priv, drop_rrp); 2601 } 2602 2603 static const struct mlx5e_tirc_config tirc_default_config[MLX5E_NUM_INDIR_TIRS] = { 2604 [MLX5E_TT_IPV4_TCP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2605 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP, 2606 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 2607 }, 2608 [MLX5E_TT_IPV6_TCP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2609 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP, 2610 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 2611 }, 2612 [MLX5E_TT_IPV4_UDP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2613 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP, 2614 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 2615 }, 2616 [MLX5E_TT_IPV6_UDP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2617 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP, 2618 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 2619 }, 2620 [MLX5E_TT_IPV4_IPSEC_AH] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2621 .l4_prot_type = 0, 2622 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 2623 }, 2624 [MLX5E_TT_IPV6_IPSEC_AH] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2625 .l4_prot_type = 0, 2626 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 2627 }, 2628 [MLX5E_TT_IPV4_IPSEC_ESP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2629 .l4_prot_type = 0, 2630 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 2631 }, 2632 [MLX5E_TT_IPV6_IPSEC_ESP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2633 .l4_prot_type = 0, 2634 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 2635 }, 2636 [MLX5E_TT_IPV4] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2637 .l4_prot_type = 0, 2638 .rx_hash_fields = MLX5_HASH_IP, 2639 }, 2640 [MLX5E_TT_IPV6] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2641 .l4_prot_type = 0, 2642 .rx_hash_fields = MLX5_HASH_IP, 2643 }, 2644 }; 2645 2646 struct mlx5e_tirc_config mlx5e_tirc_get_default_config(enum mlx5e_traffic_types tt) 2647 { 2648 return tirc_default_config[tt]; 2649 } 2650 2651 static void mlx5e_build_tir_ctx_lro(struct mlx5e_params *params, void *tirc) 2652 { 2653 if (!params->lro_en) 2654 return; 2655 2656 #define ROUGH_MAX_L2_L3_HDR_SZ 256 2657 2658 MLX5_SET(tirc, tirc, lro_enable_mask, 2659 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | 2660 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO); 2661 MLX5_SET(tirc, tirc, lro_max_ip_payload_size, 2662 (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ - ROUGH_MAX_L2_L3_HDR_SZ) >> 8); 2663 MLX5_SET(tirc, tirc, lro_timeout_period_usecs, params->lro_timeout); 2664 } 2665 2666 void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_rss_params *rss_params, 2667 const struct mlx5e_tirc_config *ttconfig, 2668 void *tirc, bool inner) 2669 { 2670 void *hfso = inner ? MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner) : 2671 MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 2672 2673 MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_rx_hash_fn(rss_params->hfunc)); 2674 if (rss_params->hfunc == ETH_RSS_HASH_TOP) { 2675 void *rss_key = MLX5_ADDR_OF(tirc, tirc, 2676 rx_hash_toeplitz_key); 2677 size_t len = MLX5_FLD_SZ_BYTES(tirc, 2678 rx_hash_toeplitz_key); 2679 2680 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 2681 memcpy(rss_key, rss_params->toeplitz_hash_key, len); 2682 } 2683 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, 2684 ttconfig->l3_prot_type); 2685 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, 2686 ttconfig->l4_prot_type); 2687 MLX5_SET(rx_hash_field_select, hfso, selected_fields, 2688 ttconfig->rx_hash_fields); 2689 } 2690 2691 static void mlx5e_update_rx_hash_fields(struct mlx5e_tirc_config *ttconfig, 2692 enum mlx5e_traffic_types tt, 2693 u32 rx_hash_fields) 2694 { 2695 *ttconfig = tirc_default_config[tt]; 2696 ttconfig->rx_hash_fields = rx_hash_fields; 2697 } 2698 2699 void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in) 2700 { 2701 void *tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx); 2702 struct mlx5e_rss_params *rss = &priv->rss_params; 2703 struct mlx5_core_dev *mdev = priv->mdev; 2704 int ctxlen = MLX5_ST_SZ_BYTES(tirc); 2705 struct mlx5e_tirc_config ttconfig; 2706 int tt; 2707 2708 MLX5_SET(modify_tir_in, in, bitmask.hash, 1); 2709 2710 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 2711 memset(tirc, 0, ctxlen); 2712 mlx5e_update_rx_hash_fields(&ttconfig, tt, 2713 rss->rx_hash_fields[tt]); 2714 mlx5e_build_indir_tir_ctx_hash(rss, &ttconfig, tirc, false); 2715 mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in); 2716 } 2717 2718 /* Verify inner tirs resources allocated */ 2719 if (!priv->inner_indir_tir[0].tirn) 2720 return; 2721 2722 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 2723 memset(tirc, 0, ctxlen); 2724 mlx5e_update_rx_hash_fields(&ttconfig, tt, 2725 rss->rx_hash_fields[tt]); 2726 mlx5e_build_indir_tir_ctx_hash(rss, &ttconfig, tirc, true); 2727 mlx5_core_modify_tir(mdev, priv->inner_indir_tir[tt].tirn, in); 2728 } 2729 } 2730 2731 static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv) 2732 { 2733 struct mlx5_core_dev *mdev = priv->mdev; 2734 2735 void *in; 2736 void *tirc; 2737 int inlen; 2738 int err; 2739 int tt; 2740 int ix; 2741 2742 inlen = MLX5_ST_SZ_BYTES(modify_tir_in); 2743 in = kvzalloc(inlen, GFP_KERNEL); 2744 if (!in) 2745 return -ENOMEM; 2746 2747 MLX5_SET(modify_tir_in, in, bitmask.lro, 1); 2748 tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx); 2749 2750 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc); 2751 2752 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 2753 err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in); 2754 if (err) 2755 goto free_in; 2756 } 2757 2758 for (ix = 0; ix < priv->max_nch; ix++) { 2759 err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn, in); 2760 if (err) 2761 goto free_in; 2762 } 2763 2764 free_in: 2765 kvfree(in); 2766 2767 return err; 2768 } 2769 2770 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_lro); 2771 2772 static int mlx5e_set_mtu(struct mlx5_core_dev *mdev, 2773 struct mlx5e_params *params, u16 mtu) 2774 { 2775 u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu); 2776 int err; 2777 2778 err = mlx5_set_port_mtu(mdev, hw_mtu, 1); 2779 if (err) 2780 return err; 2781 2782 /* Update vport context MTU */ 2783 mlx5_modify_nic_vport_mtu(mdev, hw_mtu); 2784 return 0; 2785 } 2786 2787 static void mlx5e_query_mtu(struct mlx5_core_dev *mdev, 2788 struct mlx5e_params *params, u16 *mtu) 2789 { 2790 u16 hw_mtu = 0; 2791 int err; 2792 2793 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2794 if (err || !hw_mtu) /* fallback to port oper mtu */ 2795 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1); 2796 2797 *mtu = MLX5E_HW2SW_MTU(params, hw_mtu); 2798 } 2799 2800 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv) 2801 { 2802 struct mlx5e_params *params = &priv->channels.params; 2803 struct net_device *netdev = priv->netdev; 2804 struct mlx5_core_dev *mdev = priv->mdev; 2805 u16 mtu; 2806 int err; 2807 2808 err = mlx5e_set_mtu(mdev, params, params->sw_mtu); 2809 if (err) 2810 return err; 2811 2812 mlx5e_query_mtu(mdev, params, &mtu); 2813 if (mtu != params->sw_mtu) 2814 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n", 2815 __func__, mtu, params->sw_mtu); 2816 2817 params->sw_mtu = mtu; 2818 return 0; 2819 } 2820 2821 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu); 2822 2823 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv) 2824 { 2825 struct mlx5e_params *params = &priv->channels.params; 2826 struct net_device *netdev = priv->netdev; 2827 struct mlx5_core_dev *mdev = priv->mdev; 2828 u16 max_mtu; 2829 2830 /* MTU range: 68 - hw-specific max */ 2831 netdev->min_mtu = ETH_MIN_MTU; 2832 2833 mlx5_query_port_max_mtu(mdev, &max_mtu, 1); 2834 netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu), 2835 ETH_MAX_MTU); 2836 } 2837 2838 static void mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc) 2839 { 2840 int tc; 2841 2842 netdev_reset_tc(netdev); 2843 2844 if (ntc == 1) 2845 return; 2846 2847 netdev_set_num_tc(netdev, ntc); 2848 2849 /* Map netdev TCs to offset 0 2850 * We have our own UP to TXQ mapping for QoS 2851 */ 2852 for (tc = 0; tc < ntc; tc++) 2853 netdev_set_tc_queue(netdev, tc, nch, 0); 2854 } 2855 2856 static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv) 2857 { 2858 struct net_device *netdev = priv->netdev; 2859 int num_txqs, num_rxqs, nch, ntc; 2860 int old_num_txqs, old_ntc; 2861 int err; 2862 2863 old_num_txqs = netdev->real_num_tx_queues; 2864 old_ntc = netdev->num_tc; 2865 2866 nch = priv->channels.params.num_channels; 2867 ntc = priv->channels.params.num_tc; 2868 num_txqs = nch * ntc; 2869 num_rxqs = nch * priv->profile->rq_groups; 2870 2871 mlx5e_netdev_set_tcs(netdev, nch, ntc); 2872 2873 err = netif_set_real_num_tx_queues(netdev, num_txqs); 2874 if (err) { 2875 netdev_warn(netdev, "netif_set_real_num_tx_queues failed, %d\n", err); 2876 goto err_tcs; 2877 } 2878 err = netif_set_real_num_rx_queues(netdev, num_rxqs); 2879 if (err) { 2880 netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err); 2881 goto err_txqs; 2882 } 2883 2884 return 0; 2885 2886 err_txqs: 2887 /* netif_set_real_num_rx_queues could fail only when nch increased. Only 2888 * one of nch and ntc is changed in this function. That means, the call 2889 * to netif_set_real_num_tx_queues below should not fail, because it 2890 * decreases the number of TX queues. 2891 */ 2892 WARN_ON_ONCE(netif_set_real_num_tx_queues(netdev, old_num_txqs)); 2893 2894 err_tcs: 2895 mlx5e_netdev_set_tcs(netdev, old_num_txqs / old_ntc, old_ntc); 2896 return err; 2897 } 2898 2899 static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv, 2900 struct mlx5e_params *params) 2901 { 2902 struct mlx5_core_dev *mdev = priv->mdev; 2903 int num_comp_vectors, ix, irq; 2904 2905 num_comp_vectors = mlx5_comp_vectors_count(mdev); 2906 2907 for (ix = 0; ix < params->num_channels; ix++) { 2908 cpumask_clear(priv->scratchpad.cpumask); 2909 2910 for (irq = ix; irq < num_comp_vectors; irq += params->num_channels) { 2911 int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(mdev, irq)); 2912 2913 cpumask_set_cpu(cpu, priv->scratchpad.cpumask); 2914 } 2915 2916 netif_set_xps_queue(priv->netdev, priv->scratchpad.cpumask, ix); 2917 } 2918 } 2919 2920 int mlx5e_num_channels_changed(struct mlx5e_priv *priv) 2921 { 2922 u16 count = priv->channels.params.num_channels; 2923 int err; 2924 2925 err = mlx5e_update_netdev_queues(priv); 2926 if (err) 2927 return err; 2928 2929 mlx5e_set_default_xps_cpumasks(priv, &priv->channels.params); 2930 2931 if (!netif_is_rxfh_configured(priv->netdev)) 2932 mlx5e_build_default_indir_rqt(priv->rss_params.indirection_rqt, 2933 MLX5E_INDIR_RQT_SIZE, count); 2934 2935 return 0; 2936 } 2937 2938 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed); 2939 2940 static void mlx5e_build_txq_maps(struct mlx5e_priv *priv) 2941 { 2942 int i, ch; 2943 2944 ch = priv->channels.num; 2945 2946 for (i = 0; i < ch; i++) { 2947 int tc; 2948 2949 for (tc = 0; tc < priv->channels.params.num_tc; tc++) { 2950 struct mlx5e_channel *c = priv->channels.c[i]; 2951 struct mlx5e_txqsq *sq = &c->sq[tc]; 2952 2953 priv->txq2sq[sq->txq_ix] = sq; 2954 priv->channel_tc2realtxq[i][tc] = i + tc * ch; 2955 } 2956 } 2957 } 2958 2959 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) 2960 { 2961 mlx5e_build_txq_maps(priv); 2962 mlx5e_activate_channels(&priv->channels); 2963 mlx5e_xdp_tx_enable(priv); 2964 netif_tx_start_all_queues(priv->netdev); 2965 2966 if (mlx5e_is_vport_rep(priv)) 2967 mlx5e_add_sqs_fwd_rules(priv); 2968 2969 mlx5e_wait_channels_min_rx_wqes(&priv->channels); 2970 mlx5e_redirect_rqts_to_channels(priv, &priv->channels); 2971 2972 mlx5e_xsk_redirect_rqts_to_channels(priv, &priv->channels); 2973 } 2974 2975 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv) 2976 { 2977 mlx5e_xsk_redirect_rqts_to_drop(priv, &priv->channels); 2978 2979 mlx5e_redirect_rqts_to_drop(priv); 2980 2981 if (mlx5e_is_vport_rep(priv)) 2982 mlx5e_remove_sqs_fwd_rules(priv); 2983 2984 /* FIXME: This is a W/A only for tx timeout watch dog false alarm when 2985 * polling for inactive tx queues. 2986 */ 2987 netif_tx_stop_all_queues(priv->netdev); 2988 netif_tx_disable(priv->netdev); 2989 mlx5e_xdp_tx_disable(priv); 2990 mlx5e_deactivate_channels(&priv->channels); 2991 } 2992 2993 static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv, 2994 struct mlx5e_channels *new_chs, 2995 mlx5e_fp_preactivate preactivate, 2996 void *context) 2997 { 2998 struct net_device *netdev = priv->netdev; 2999 struct mlx5e_channels old_chs; 3000 int carrier_ok; 3001 int err = 0; 3002 3003 carrier_ok = netif_carrier_ok(netdev); 3004 netif_carrier_off(netdev); 3005 3006 mlx5e_deactivate_priv_channels(priv); 3007 3008 old_chs = priv->channels; 3009 priv->channels = *new_chs; 3010 3011 /* New channels are ready to roll, call the preactivate hook if needed 3012 * to modify HW settings or update kernel parameters. 3013 */ 3014 if (preactivate) { 3015 err = preactivate(priv, context); 3016 if (err) { 3017 priv->channels = old_chs; 3018 goto out; 3019 } 3020 } 3021 3022 mlx5e_close_channels(&old_chs); 3023 priv->profile->update_rx(priv); 3024 3025 out: 3026 mlx5e_activate_priv_channels(priv); 3027 3028 /* return carrier back if needed */ 3029 if (carrier_ok) 3030 netif_carrier_on(netdev); 3031 3032 return err; 3033 } 3034 3035 int mlx5e_safe_switch_channels(struct mlx5e_priv *priv, 3036 struct mlx5e_channels *new_chs, 3037 mlx5e_fp_preactivate preactivate, 3038 void *context) 3039 { 3040 int err; 3041 3042 err = mlx5e_open_channels(priv, new_chs); 3043 if (err) 3044 return err; 3045 3046 err = mlx5e_switch_priv_channels(priv, new_chs, preactivate, context); 3047 if (err) 3048 goto err_close; 3049 3050 return 0; 3051 3052 err_close: 3053 mlx5e_close_channels(new_chs); 3054 3055 return err; 3056 } 3057 3058 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv) 3059 { 3060 struct mlx5e_channels new_channels = {}; 3061 3062 new_channels.params = priv->channels.params; 3063 return mlx5e_safe_switch_channels(priv, &new_channels, NULL, NULL); 3064 } 3065 3066 void mlx5e_timestamp_init(struct mlx5e_priv *priv) 3067 { 3068 priv->tstamp.tx_type = HWTSTAMP_TX_OFF; 3069 priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE; 3070 } 3071 3072 static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev, 3073 enum mlx5_port_status state) 3074 { 3075 struct mlx5_eswitch *esw = mdev->priv.eswitch; 3076 int vport_admin_state; 3077 3078 mlx5_set_port_admin_status(mdev, state); 3079 3080 if (!MLX5_ESWITCH_MANAGER(mdev) || mlx5_eswitch_mode(esw) == MLX5_ESWITCH_OFFLOADS) 3081 return; 3082 3083 if (state == MLX5_PORT_UP) 3084 vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO; 3085 else 3086 vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3087 3088 mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state); 3089 } 3090 3091 int mlx5e_open_locked(struct net_device *netdev) 3092 { 3093 struct mlx5e_priv *priv = netdev_priv(netdev); 3094 int err; 3095 3096 set_bit(MLX5E_STATE_OPENED, &priv->state); 3097 3098 err = mlx5e_open_channels(priv, &priv->channels); 3099 if (err) 3100 goto err_clear_state_opened_flag; 3101 3102 priv->profile->update_rx(priv); 3103 mlx5e_activate_priv_channels(priv); 3104 if (priv->profile->update_carrier) 3105 priv->profile->update_carrier(priv); 3106 3107 mlx5e_queue_update_stats(priv); 3108 return 0; 3109 3110 err_clear_state_opened_flag: 3111 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3112 return err; 3113 } 3114 3115 int mlx5e_open(struct net_device *netdev) 3116 { 3117 struct mlx5e_priv *priv = netdev_priv(netdev); 3118 int err; 3119 3120 mutex_lock(&priv->state_lock); 3121 err = mlx5e_open_locked(netdev); 3122 if (!err) 3123 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP); 3124 mutex_unlock(&priv->state_lock); 3125 3126 return err; 3127 } 3128 3129 int mlx5e_close_locked(struct net_device *netdev) 3130 { 3131 struct mlx5e_priv *priv = netdev_priv(netdev); 3132 3133 /* May already be CLOSED in case a previous configuration operation 3134 * (e.g RX/TX queue size change) that involves close&open failed. 3135 */ 3136 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 3137 return 0; 3138 3139 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3140 3141 netif_carrier_off(priv->netdev); 3142 mlx5e_deactivate_priv_channels(priv); 3143 mlx5e_close_channels(&priv->channels); 3144 3145 return 0; 3146 } 3147 3148 int mlx5e_close(struct net_device *netdev) 3149 { 3150 struct mlx5e_priv *priv = netdev_priv(netdev); 3151 int err; 3152 3153 if (!netif_device_present(netdev)) 3154 return -ENODEV; 3155 3156 mutex_lock(&priv->state_lock); 3157 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN); 3158 err = mlx5e_close_locked(netdev); 3159 mutex_unlock(&priv->state_lock); 3160 3161 return err; 3162 } 3163 3164 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev, 3165 struct mlx5e_rq *rq, 3166 struct mlx5e_rq_param *param) 3167 { 3168 void *rqc = param->rqc; 3169 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); 3170 int err; 3171 3172 param->wq.db_numa_node = param->wq.buf_numa_node; 3173 3174 err = mlx5_wq_cyc_create(mdev, ¶m->wq, rqc_wq, &rq->wqe.wq, 3175 &rq->wq_ctrl); 3176 if (err) 3177 return err; 3178 3179 /* Mark as unused given "Drop-RQ" packets never reach XDP */ 3180 xdp_rxq_info_unused(&rq->xdp_rxq); 3181 3182 rq->mdev = mdev; 3183 3184 return 0; 3185 } 3186 3187 static int mlx5e_alloc_drop_cq(struct mlx5_core_dev *mdev, 3188 struct mlx5e_cq *cq, 3189 struct mlx5e_cq_param *param) 3190 { 3191 param->wq.buf_numa_node = dev_to_node(mdev->device); 3192 param->wq.db_numa_node = dev_to_node(mdev->device); 3193 3194 return mlx5e_alloc_cq_common(mdev, param, cq); 3195 } 3196 3197 int mlx5e_open_drop_rq(struct mlx5e_priv *priv, 3198 struct mlx5e_rq *drop_rq) 3199 { 3200 struct mlx5_core_dev *mdev = priv->mdev; 3201 struct mlx5e_cq_param cq_param = {}; 3202 struct mlx5e_rq_param rq_param = {}; 3203 struct mlx5e_cq *cq = &drop_rq->cq; 3204 int err; 3205 3206 mlx5e_build_drop_rq_param(priv, &rq_param); 3207 3208 err = mlx5e_alloc_drop_cq(mdev, cq, &cq_param); 3209 if (err) 3210 return err; 3211 3212 err = mlx5e_create_cq(cq, &cq_param); 3213 if (err) 3214 goto err_free_cq; 3215 3216 err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param); 3217 if (err) 3218 goto err_destroy_cq; 3219 3220 err = mlx5e_create_rq(drop_rq, &rq_param); 3221 if (err) 3222 goto err_free_rq; 3223 3224 err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 3225 if (err) 3226 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err); 3227 3228 return 0; 3229 3230 err_free_rq: 3231 mlx5e_free_rq(drop_rq); 3232 3233 err_destroy_cq: 3234 mlx5e_destroy_cq(cq); 3235 3236 err_free_cq: 3237 mlx5e_free_cq(cq); 3238 3239 return err; 3240 } 3241 3242 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq) 3243 { 3244 mlx5e_destroy_rq(drop_rq); 3245 mlx5e_free_rq(drop_rq); 3246 mlx5e_destroy_cq(&drop_rq->cq); 3247 mlx5e_free_cq(&drop_rq->cq); 3248 } 3249 3250 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn) 3251 { 3252 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 3253 3254 MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn); 3255 3256 if (MLX5_GET(tisc, tisc, tls_en)) 3257 MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.pdn); 3258 3259 if (mlx5_lag_is_lacp_owner(mdev)) 3260 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1); 3261 3262 return mlx5_core_create_tis(mdev, in, tisn); 3263 } 3264 3265 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn) 3266 { 3267 mlx5_core_destroy_tis(mdev, tisn); 3268 } 3269 3270 void mlx5e_destroy_tises(struct mlx5e_priv *priv) 3271 { 3272 int tc, i; 3273 3274 for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) 3275 for (tc = 0; tc < priv->profile->max_tc; tc++) 3276 mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]); 3277 } 3278 3279 static bool mlx5e_lag_should_assign_affinity(struct mlx5_core_dev *mdev) 3280 { 3281 return MLX5_CAP_GEN(mdev, lag_tx_port_affinity) && mlx5e_get_num_lag_ports(mdev) > 1; 3282 } 3283 3284 int mlx5e_create_tises(struct mlx5e_priv *priv) 3285 { 3286 int tc, i; 3287 int err; 3288 3289 for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) { 3290 for (tc = 0; tc < priv->profile->max_tc; tc++) { 3291 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 3292 void *tisc; 3293 3294 tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 3295 3296 MLX5_SET(tisc, tisc, prio, tc << 1); 3297 3298 if (mlx5e_lag_should_assign_affinity(priv->mdev)) 3299 MLX5_SET(tisc, tisc, lag_tx_port_affinity, i + 1); 3300 3301 err = mlx5e_create_tis(priv->mdev, in, &priv->tisn[i][tc]); 3302 if (err) 3303 goto err_close_tises; 3304 } 3305 } 3306 3307 return 0; 3308 3309 err_close_tises: 3310 for (; i >= 0; i--) { 3311 for (tc--; tc >= 0; tc--) 3312 mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]); 3313 tc = priv->profile->max_tc; 3314 } 3315 3316 return err; 3317 } 3318 3319 static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) 3320 { 3321 mlx5e_destroy_tises(priv); 3322 } 3323 3324 static void mlx5e_build_indir_tir_ctx_common(struct mlx5e_priv *priv, 3325 u32 rqtn, u32 *tirc) 3326 { 3327 MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn); 3328 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 3329 MLX5_SET(tirc, tirc, indirect_table, rqtn); 3330 MLX5_SET(tirc, tirc, tunneled_offload_en, 3331 priv->channels.params.tunneled_offload_en); 3332 3333 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc); 3334 } 3335 3336 static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv, 3337 enum mlx5e_traffic_types tt, 3338 u32 *tirc) 3339 { 3340 mlx5e_build_indir_tir_ctx_common(priv, priv->indir_rqt.rqtn, tirc); 3341 mlx5e_build_indir_tir_ctx_hash(&priv->rss_params, 3342 &tirc_default_config[tt], tirc, false); 3343 } 3344 3345 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 rqtn, u32 *tirc) 3346 { 3347 mlx5e_build_indir_tir_ctx_common(priv, rqtn, tirc); 3348 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8); 3349 } 3350 3351 static void mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv *priv, 3352 enum mlx5e_traffic_types tt, 3353 u32 *tirc) 3354 { 3355 mlx5e_build_indir_tir_ctx_common(priv, priv->indir_rqt.rqtn, tirc); 3356 mlx5e_build_indir_tir_ctx_hash(&priv->rss_params, 3357 &tirc_default_config[tt], tirc, true); 3358 } 3359 3360 int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc) 3361 { 3362 struct mlx5e_tir *tir; 3363 void *tirc; 3364 int inlen; 3365 int i = 0; 3366 int err; 3367 u32 *in; 3368 int tt; 3369 3370 inlen = MLX5_ST_SZ_BYTES(create_tir_in); 3371 in = kvzalloc(inlen, GFP_KERNEL); 3372 if (!in) 3373 return -ENOMEM; 3374 3375 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 3376 memset(in, 0, inlen); 3377 tir = &priv->indir_tir[tt]; 3378 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 3379 mlx5e_build_indir_tir_ctx(priv, tt, tirc); 3380 err = mlx5e_create_tir(priv->mdev, tir, in); 3381 if (err) { 3382 mlx5_core_warn(priv->mdev, "create indirect tirs failed, %d\n", err); 3383 goto err_destroy_inner_tirs; 3384 } 3385 } 3386 3387 if (!inner_ttc || !mlx5e_tunnel_inner_ft_supported(priv->mdev)) 3388 goto out; 3389 3390 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) { 3391 memset(in, 0, inlen); 3392 tir = &priv->inner_indir_tir[i]; 3393 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 3394 mlx5e_build_inner_indir_tir_ctx(priv, i, tirc); 3395 err = mlx5e_create_tir(priv->mdev, tir, in); 3396 if (err) { 3397 mlx5_core_warn(priv->mdev, "create inner indirect tirs failed, %d\n", err); 3398 goto err_destroy_inner_tirs; 3399 } 3400 } 3401 3402 out: 3403 kvfree(in); 3404 3405 return 0; 3406 3407 err_destroy_inner_tirs: 3408 for (i--; i >= 0; i--) 3409 mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]); 3410 3411 for (tt--; tt >= 0; tt--) 3412 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]); 3413 3414 kvfree(in); 3415 3416 return err; 3417 } 3418 3419 int mlx5e_create_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs) 3420 { 3421 struct mlx5e_tir *tir; 3422 void *tirc; 3423 int inlen; 3424 int err = 0; 3425 u32 *in; 3426 int ix; 3427 3428 inlen = MLX5_ST_SZ_BYTES(create_tir_in); 3429 in = kvzalloc(inlen, GFP_KERNEL); 3430 if (!in) 3431 return -ENOMEM; 3432 3433 for (ix = 0; ix < priv->max_nch; ix++) { 3434 memset(in, 0, inlen); 3435 tir = &tirs[ix]; 3436 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 3437 mlx5e_build_direct_tir_ctx(priv, tir->rqt.rqtn, tirc); 3438 err = mlx5e_create_tir(priv->mdev, tir, in); 3439 if (unlikely(err)) 3440 goto err_destroy_ch_tirs; 3441 } 3442 3443 goto out; 3444 3445 err_destroy_ch_tirs: 3446 mlx5_core_warn(priv->mdev, "create tirs failed, %d\n", err); 3447 for (ix--; ix >= 0; ix--) 3448 mlx5e_destroy_tir(priv->mdev, &tirs[ix]); 3449 3450 out: 3451 kvfree(in); 3452 3453 return err; 3454 } 3455 3456 void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv) 3457 { 3458 int i; 3459 3460 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) 3461 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]); 3462 3463 /* Verify inner tirs resources allocated */ 3464 if (!priv->inner_indir_tir[0].tirn) 3465 return; 3466 3467 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) 3468 mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]); 3469 } 3470 3471 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs) 3472 { 3473 int i; 3474 3475 for (i = 0; i < priv->max_nch; i++) 3476 mlx5e_destroy_tir(priv->mdev, &tirs[i]); 3477 } 3478 3479 static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable) 3480 { 3481 int err = 0; 3482 int i; 3483 3484 for (i = 0; i < chs->num; i++) { 3485 err = mlx5e_modify_rq_scatter_fcs(&chs->c[i]->rq, enable); 3486 if (err) 3487 return err; 3488 } 3489 3490 return 0; 3491 } 3492 3493 static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd) 3494 { 3495 int err = 0; 3496 int i; 3497 3498 for (i = 0; i < chs->num; i++) { 3499 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd); 3500 if (err) 3501 return err; 3502 } 3503 3504 return 0; 3505 } 3506 3507 static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, 3508 struct tc_mqprio_qopt *mqprio) 3509 { 3510 struct mlx5e_channels new_channels = {}; 3511 u8 tc = mqprio->num_tc; 3512 int err = 0; 3513 3514 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 3515 3516 if (tc && tc != MLX5E_MAX_NUM_TC) 3517 return -EINVAL; 3518 3519 mutex_lock(&priv->state_lock); 3520 3521 new_channels.params = priv->channels.params; 3522 new_channels.params.num_tc = tc ? tc : 1; 3523 3524 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { 3525 priv->channels.params = new_channels.params; 3526 goto out; 3527 } 3528 3529 err = mlx5e_safe_switch_channels(priv, &new_channels, 3530 mlx5e_num_channels_changed_ctx, NULL); 3531 if (err) 3532 goto out; 3533 3534 priv->max_opened_tc = max_t(u8, priv->max_opened_tc, 3535 new_channels.params.num_tc); 3536 out: 3537 mutex_unlock(&priv->state_lock); 3538 return err; 3539 } 3540 3541 static LIST_HEAD(mlx5e_block_cb_list); 3542 3543 static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, 3544 void *type_data) 3545 { 3546 struct mlx5e_priv *priv = netdev_priv(dev); 3547 3548 switch (type) { 3549 case TC_SETUP_BLOCK: { 3550 struct flow_block_offload *f = type_data; 3551 3552 f->unlocked_driver_cb = true; 3553 return flow_block_cb_setup_simple(type_data, 3554 &mlx5e_block_cb_list, 3555 mlx5e_setup_tc_block_cb, 3556 priv, priv, true); 3557 } 3558 case TC_SETUP_QDISC_MQPRIO: 3559 return mlx5e_setup_tc_mqprio(priv, type_data); 3560 default: 3561 return -EOPNOTSUPP; 3562 } 3563 } 3564 3565 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s) 3566 { 3567 int i; 3568 3569 for (i = 0; i < priv->max_nch; i++) { 3570 struct mlx5e_channel_stats *channel_stats = &priv->channel_stats[i]; 3571 struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq; 3572 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq; 3573 int j; 3574 3575 s->rx_packets += rq_stats->packets + xskrq_stats->packets; 3576 s->rx_bytes += rq_stats->bytes + xskrq_stats->bytes; 3577 3578 for (j = 0; j < priv->max_opened_tc; j++) { 3579 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j]; 3580 3581 s->tx_packets += sq_stats->packets; 3582 s->tx_bytes += sq_stats->bytes; 3583 s->tx_dropped += sq_stats->dropped; 3584 } 3585 } 3586 } 3587 3588 void 3589 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) 3590 { 3591 struct mlx5e_priv *priv = netdev_priv(dev); 3592 struct mlx5e_vport_stats *vstats = &priv->stats.vport; 3593 struct mlx5e_pport_stats *pstats = &priv->stats.pport; 3594 3595 /* In switchdev mode, monitor counters doesn't monitor 3596 * rx/tx stats of 802_3. The update stats mechanism 3597 * should keep the 802_3 layout counters updated 3598 */ 3599 if (!mlx5e_monitor_counter_supported(priv) || 3600 mlx5e_is_uplink_rep(priv)) { 3601 /* update HW stats in background for next time */ 3602 mlx5e_queue_update_stats(priv); 3603 } 3604 3605 if (mlx5e_is_uplink_rep(priv)) { 3606 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok); 3607 stats->rx_bytes = PPORT_802_3_GET(pstats, a_octets_received_ok); 3608 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok); 3609 stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok); 3610 } else { 3611 mlx5e_fold_sw_stats64(priv, stats); 3612 } 3613 3614 stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer; 3615 3616 stats->rx_length_errors = 3617 PPORT_802_3_GET(pstats, a_in_range_length_errors) + 3618 PPORT_802_3_GET(pstats, a_out_of_range_length_field) + 3619 PPORT_802_3_GET(pstats, a_frame_too_long_errors); 3620 stats->rx_crc_errors = 3621 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors); 3622 stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors); 3623 stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards); 3624 stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors + 3625 stats->rx_frame_errors; 3626 stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors; 3627 3628 /* vport multicast also counts packets that are dropped due to steering 3629 * or rx out of buffer 3630 */ 3631 stats->multicast = 3632 VPORT_COUNTER_GET(vstats, received_eth_multicast.packets); 3633 } 3634 3635 static void mlx5e_set_rx_mode(struct net_device *dev) 3636 { 3637 struct mlx5e_priv *priv = netdev_priv(dev); 3638 3639 queue_work(priv->wq, &priv->set_rx_mode_work); 3640 } 3641 3642 static int mlx5e_set_mac(struct net_device *netdev, void *addr) 3643 { 3644 struct mlx5e_priv *priv = netdev_priv(netdev); 3645 struct sockaddr *saddr = addr; 3646 3647 if (!is_valid_ether_addr(saddr->sa_data)) 3648 return -EADDRNOTAVAIL; 3649 3650 netif_addr_lock_bh(netdev); 3651 ether_addr_copy(netdev->dev_addr, saddr->sa_data); 3652 netif_addr_unlock_bh(netdev); 3653 3654 queue_work(priv->wq, &priv->set_rx_mode_work); 3655 3656 return 0; 3657 } 3658 3659 #define MLX5E_SET_FEATURE(features, feature, enable) \ 3660 do { \ 3661 if (enable) \ 3662 *features |= feature; \ 3663 else \ 3664 *features &= ~feature; \ 3665 } while (0) 3666 3667 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable); 3668 3669 static int set_feature_lro(struct net_device *netdev, bool enable) 3670 { 3671 struct mlx5e_priv *priv = netdev_priv(netdev); 3672 struct mlx5_core_dev *mdev = priv->mdev; 3673 struct mlx5e_channels new_channels = {}; 3674 struct mlx5e_params *old_params; 3675 int err = 0; 3676 bool reset; 3677 3678 mutex_lock(&priv->state_lock); 3679 3680 if (enable && priv->xsk.refcnt) { 3681 netdev_warn(netdev, "LRO is incompatible with AF_XDP (%hu XSKs are active)\n", 3682 priv->xsk.refcnt); 3683 err = -EINVAL; 3684 goto out; 3685 } 3686 3687 old_params = &priv->channels.params; 3688 if (enable && !MLX5E_GET_PFLAG(old_params, MLX5E_PFLAG_RX_STRIDING_RQ)) { 3689 netdev_warn(netdev, "can't set LRO with legacy RQ\n"); 3690 err = -EINVAL; 3691 goto out; 3692 } 3693 3694 reset = test_bit(MLX5E_STATE_OPENED, &priv->state); 3695 3696 new_channels.params = *old_params; 3697 new_channels.params.lro_en = enable; 3698 3699 if (old_params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) { 3700 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, old_params, NULL) == 3701 mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_channels.params, NULL)) 3702 reset = false; 3703 } 3704 3705 if (!reset) { 3706 *old_params = new_channels.params; 3707 err = mlx5e_modify_tirs_lro(priv); 3708 goto out; 3709 } 3710 3711 err = mlx5e_safe_switch_channels(priv, &new_channels, 3712 mlx5e_modify_tirs_lro_ctx, NULL); 3713 out: 3714 mutex_unlock(&priv->state_lock); 3715 return err; 3716 } 3717 3718 static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) 3719 { 3720 struct mlx5e_priv *priv = netdev_priv(netdev); 3721 3722 if (enable) 3723 mlx5e_enable_cvlan_filter(priv); 3724 else 3725 mlx5e_disable_cvlan_filter(priv); 3726 3727 return 0; 3728 } 3729 3730 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 3731 static int set_feature_tc_num_filters(struct net_device *netdev, bool enable) 3732 { 3733 struct mlx5e_priv *priv = netdev_priv(netdev); 3734 3735 if (!enable && mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD))) { 3736 netdev_err(netdev, 3737 "Active offloaded tc filters, can't turn hw_tc_offload off\n"); 3738 return -EINVAL; 3739 } 3740 3741 return 0; 3742 } 3743 #endif 3744 3745 static int set_feature_rx_all(struct net_device *netdev, bool enable) 3746 { 3747 struct mlx5e_priv *priv = netdev_priv(netdev); 3748 struct mlx5_core_dev *mdev = priv->mdev; 3749 3750 return mlx5_set_port_fcs(mdev, !enable); 3751 } 3752 3753 static int set_feature_rx_fcs(struct net_device *netdev, bool enable) 3754 { 3755 struct mlx5e_priv *priv = netdev_priv(netdev); 3756 int err; 3757 3758 mutex_lock(&priv->state_lock); 3759 3760 priv->channels.params.scatter_fcs_en = enable; 3761 err = mlx5e_modify_channels_scatter_fcs(&priv->channels, enable); 3762 if (err) 3763 priv->channels.params.scatter_fcs_en = !enable; 3764 3765 mutex_unlock(&priv->state_lock); 3766 3767 return err; 3768 } 3769 3770 static int set_feature_rx_vlan(struct net_device *netdev, bool enable) 3771 { 3772 struct mlx5e_priv *priv = netdev_priv(netdev); 3773 int err = 0; 3774 3775 mutex_lock(&priv->state_lock); 3776 3777 priv->channels.params.vlan_strip_disable = !enable; 3778 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 3779 goto unlock; 3780 3781 err = mlx5e_modify_channels_vsd(&priv->channels, !enable); 3782 if (err) 3783 priv->channels.params.vlan_strip_disable = enable; 3784 3785 unlock: 3786 mutex_unlock(&priv->state_lock); 3787 3788 return err; 3789 } 3790 3791 #ifdef CONFIG_MLX5_EN_ARFS 3792 static int set_feature_arfs(struct net_device *netdev, bool enable) 3793 { 3794 struct mlx5e_priv *priv = netdev_priv(netdev); 3795 int err; 3796 3797 if (enable) 3798 err = mlx5e_arfs_enable(priv); 3799 else 3800 err = mlx5e_arfs_disable(priv); 3801 3802 return err; 3803 } 3804 #endif 3805 3806 static int mlx5e_handle_feature(struct net_device *netdev, 3807 netdev_features_t *features, 3808 netdev_features_t wanted_features, 3809 netdev_features_t feature, 3810 mlx5e_feature_handler feature_handler) 3811 { 3812 netdev_features_t changes = wanted_features ^ netdev->features; 3813 bool enable = !!(wanted_features & feature); 3814 int err; 3815 3816 if (!(changes & feature)) 3817 return 0; 3818 3819 err = feature_handler(netdev, enable); 3820 if (err) { 3821 netdev_err(netdev, "%s feature %pNF failed, err %d\n", 3822 enable ? "Enable" : "Disable", &feature, err); 3823 return err; 3824 } 3825 3826 MLX5E_SET_FEATURE(features, feature, enable); 3827 return 0; 3828 } 3829 3830 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features) 3831 { 3832 netdev_features_t oper_features = netdev->features; 3833 int err = 0; 3834 3835 #define MLX5E_HANDLE_FEATURE(feature, handler) \ 3836 mlx5e_handle_feature(netdev, &oper_features, features, feature, handler) 3837 3838 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 3839 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER, 3840 set_feature_cvlan_filter); 3841 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 3842 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_tc_num_filters); 3843 #endif 3844 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all); 3845 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 3846 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan); 3847 #ifdef CONFIG_MLX5_EN_ARFS 3848 err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs); 3849 #endif 3850 3851 if (err) { 3852 netdev->features = oper_features; 3853 return -EINVAL; 3854 } 3855 3856 return 0; 3857 } 3858 3859 static netdev_features_t mlx5e_fix_features(struct net_device *netdev, 3860 netdev_features_t features) 3861 { 3862 struct mlx5e_priv *priv = netdev_priv(netdev); 3863 struct mlx5e_params *params; 3864 3865 mutex_lock(&priv->state_lock); 3866 params = &priv->channels.params; 3867 if (!bitmap_empty(priv->fs.vlan.active_svlans, VLAN_N_VID)) { 3868 /* HW strips the outer C-tag header, this is a problem 3869 * for S-tag traffic. 3870 */ 3871 features &= ~NETIF_F_HW_VLAN_CTAG_RX; 3872 if (!params->vlan_strip_disable) 3873 netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n"); 3874 } 3875 if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) { 3876 if (features & NETIF_F_LRO) { 3877 netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n"); 3878 features &= ~NETIF_F_LRO; 3879 } 3880 } 3881 3882 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { 3883 features &= ~NETIF_F_RXHASH; 3884 if (netdev->features & NETIF_F_RXHASH) 3885 netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n"); 3886 } 3887 3888 mutex_unlock(&priv->state_lock); 3889 3890 return features; 3891 } 3892 3893 static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, 3894 struct mlx5e_channels *chs, 3895 struct mlx5e_params *new_params, 3896 struct mlx5_core_dev *mdev) 3897 { 3898 u16 ix; 3899 3900 for (ix = 0; ix < chs->params.num_channels; ix++) { 3901 struct xdp_umem *umem = mlx5e_xsk_get_umem(&chs->params, chs->params.xsk, ix); 3902 struct mlx5e_xsk_param xsk; 3903 3904 if (!umem) 3905 continue; 3906 3907 mlx5e_build_xsk_param(umem, &xsk); 3908 3909 if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev)) { 3910 u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk); 3911 int max_mtu_frame, max_mtu_page, max_mtu; 3912 3913 /* Two criteria must be met: 3914 * 1. HW MTU + all headrooms <= XSK frame size. 3915 * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE. 3916 */ 3917 max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr); 3918 max_mtu_page = mlx5e_xdp_max_mtu(new_params, &xsk); 3919 max_mtu = min(max_mtu_frame, max_mtu_page); 3920 3921 netdev_err(netdev, "MTU %d is too big for an XSK running on channel %hu. Try MTU <= %d\n", 3922 new_params->sw_mtu, ix, max_mtu); 3923 return false; 3924 } 3925 } 3926 3927 return true; 3928 } 3929 3930 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, 3931 mlx5e_fp_preactivate preactivate) 3932 { 3933 struct mlx5e_priv *priv = netdev_priv(netdev); 3934 struct mlx5e_channels new_channels = {}; 3935 struct mlx5e_params *params; 3936 int err = 0; 3937 bool reset; 3938 3939 mutex_lock(&priv->state_lock); 3940 3941 params = &priv->channels.params; 3942 3943 reset = !params->lro_en; 3944 reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state); 3945 3946 new_channels.params = *params; 3947 new_channels.params.sw_mtu = new_mtu; 3948 3949 if (params->xdp_prog && 3950 !mlx5e_rx_is_linear_skb(&new_channels.params, NULL)) { 3951 netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n", 3952 new_mtu, mlx5e_xdp_max_mtu(params, NULL)); 3953 err = -EINVAL; 3954 goto out; 3955 } 3956 3957 if (priv->xsk.refcnt && 3958 !mlx5e_xsk_validate_mtu(netdev, &priv->channels, 3959 &new_channels.params, priv->mdev)) { 3960 err = -EINVAL; 3961 goto out; 3962 } 3963 3964 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 3965 bool is_linear = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, 3966 &new_channels.params, 3967 NULL); 3968 u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params, NULL); 3969 u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_channels.params, NULL); 3970 3971 /* If XSK is active, XSK RQs are linear. */ 3972 is_linear |= priv->xsk.refcnt; 3973 3974 /* Always reset in linear mode - hw_mtu is used in data path. */ 3975 reset = reset && (is_linear || (ppw_old != ppw_new)); 3976 } 3977 3978 if (!reset) { 3979 params->sw_mtu = new_mtu; 3980 if (preactivate) 3981 preactivate(priv, NULL); 3982 netdev->mtu = params->sw_mtu; 3983 goto out; 3984 } 3985 3986 err = mlx5e_safe_switch_channels(priv, &new_channels, preactivate, NULL); 3987 if (err) 3988 goto out; 3989 3990 netdev->mtu = new_channels.params.sw_mtu; 3991 3992 out: 3993 mutex_unlock(&priv->state_lock); 3994 return err; 3995 } 3996 3997 static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu) 3998 { 3999 return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu_ctx); 4000 } 4001 4002 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr) 4003 { 4004 struct hwtstamp_config config; 4005 int err; 4006 4007 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) || 4008 (mlx5_clock_get_ptp_index(priv->mdev) == -1)) 4009 return -EOPNOTSUPP; 4010 4011 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 4012 return -EFAULT; 4013 4014 /* TX HW timestamp */ 4015 switch (config.tx_type) { 4016 case HWTSTAMP_TX_OFF: 4017 case HWTSTAMP_TX_ON: 4018 break; 4019 default: 4020 return -ERANGE; 4021 } 4022 4023 mutex_lock(&priv->state_lock); 4024 /* RX HW timestamp */ 4025 switch (config.rx_filter) { 4026 case HWTSTAMP_FILTER_NONE: 4027 /* Reset CQE compression to Admin default */ 4028 mlx5e_modify_rx_cqe_compression_locked(priv, priv->channels.params.rx_cqe_compress_def); 4029 break; 4030 case HWTSTAMP_FILTER_ALL: 4031 case HWTSTAMP_FILTER_SOME: 4032 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 4033 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 4034 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 4035 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 4036 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 4037 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 4038 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 4039 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 4040 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 4041 case HWTSTAMP_FILTER_PTP_V2_EVENT: 4042 case HWTSTAMP_FILTER_PTP_V2_SYNC: 4043 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 4044 case HWTSTAMP_FILTER_NTP_ALL: 4045 /* Disable CQE compression */ 4046 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS)) 4047 netdev_warn(priv->netdev, "Disabling RX cqe compression\n"); 4048 err = mlx5e_modify_rx_cqe_compression_locked(priv, false); 4049 if (err) { 4050 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err); 4051 mutex_unlock(&priv->state_lock); 4052 return err; 4053 } 4054 config.rx_filter = HWTSTAMP_FILTER_ALL; 4055 break; 4056 default: 4057 mutex_unlock(&priv->state_lock); 4058 return -ERANGE; 4059 } 4060 4061 memcpy(&priv->tstamp, &config, sizeof(config)); 4062 mutex_unlock(&priv->state_lock); 4063 4064 /* might need to fix some features */ 4065 netdev_update_features(priv->netdev); 4066 4067 return copy_to_user(ifr->ifr_data, &config, 4068 sizeof(config)) ? -EFAULT : 0; 4069 } 4070 4071 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr) 4072 { 4073 struct hwtstamp_config *cfg = &priv->tstamp; 4074 4075 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz)) 4076 return -EOPNOTSUPP; 4077 4078 return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0; 4079 } 4080 4081 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 4082 { 4083 struct mlx5e_priv *priv = netdev_priv(dev); 4084 4085 switch (cmd) { 4086 case SIOCSHWTSTAMP: 4087 return mlx5e_hwstamp_set(priv, ifr); 4088 case SIOCGHWTSTAMP: 4089 return mlx5e_hwstamp_get(priv, ifr); 4090 default: 4091 return -EOPNOTSUPP; 4092 } 4093 } 4094 4095 #ifdef CONFIG_MLX5_ESWITCH 4096 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 4097 { 4098 struct mlx5e_priv *priv = netdev_priv(dev); 4099 struct mlx5_core_dev *mdev = priv->mdev; 4100 4101 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac); 4102 } 4103 4104 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, 4105 __be16 vlan_proto) 4106 { 4107 struct mlx5e_priv *priv = netdev_priv(dev); 4108 struct mlx5_core_dev *mdev = priv->mdev; 4109 4110 if (vlan_proto != htons(ETH_P_8021Q)) 4111 return -EPROTONOSUPPORT; 4112 4113 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1, 4114 vlan, qos); 4115 } 4116 4117 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting) 4118 { 4119 struct mlx5e_priv *priv = netdev_priv(dev); 4120 struct mlx5_core_dev *mdev = priv->mdev; 4121 4122 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting); 4123 } 4124 4125 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting) 4126 { 4127 struct mlx5e_priv *priv = netdev_priv(dev); 4128 struct mlx5_core_dev *mdev = priv->mdev; 4129 4130 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting); 4131 } 4132 4133 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 4134 int max_tx_rate) 4135 { 4136 struct mlx5e_priv *priv = netdev_priv(dev); 4137 struct mlx5_core_dev *mdev = priv->mdev; 4138 4139 return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1, 4140 max_tx_rate, min_tx_rate); 4141 } 4142 4143 static int mlx5_vport_link2ifla(u8 esw_link) 4144 { 4145 switch (esw_link) { 4146 case MLX5_VPORT_ADMIN_STATE_DOWN: 4147 return IFLA_VF_LINK_STATE_DISABLE; 4148 case MLX5_VPORT_ADMIN_STATE_UP: 4149 return IFLA_VF_LINK_STATE_ENABLE; 4150 } 4151 return IFLA_VF_LINK_STATE_AUTO; 4152 } 4153 4154 static int mlx5_ifla_link2vport(u8 ifla_link) 4155 { 4156 switch (ifla_link) { 4157 case IFLA_VF_LINK_STATE_DISABLE: 4158 return MLX5_VPORT_ADMIN_STATE_DOWN; 4159 case IFLA_VF_LINK_STATE_ENABLE: 4160 return MLX5_VPORT_ADMIN_STATE_UP; 4161 } 4162 return MLX5_VPORT_ADMIN_STATE_AUTO; 4163 } 4164 4165 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf, 4166 int link_state) 4167 { 4168 struct mlx5e_priv *priv = netdev_priv(dev); 4169 struct mlx5_core_dev *mdev = priv->mdev; 4170 4171 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1, 4172 mlx5_ifla_link2vport(link_state)); 4173 } 4174 4175 int mlx5e_get_vf_config(struct net_device *dev, 4176 int vf, struct ifla_vf_info *ivi) 4177 { 4178 struct mlx5e_priv *priv = netdev_priv(dev); 4179 struct mlx5_core_dev *mdev = priv->mdev; 4180 int err; 4181 4182 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi); 4183 if (err) 4184 return err; 4185 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate); 4186 return 0; 4187 } 4188 4189 int mlx5e_get_vf_stats(struct net_device *dev, 4190 int vf, struct ifla_vf_stats *vf_stats) 4191 { 4192 struct mlx5e_priv *priv = netdev_priv(dev); 4193 struct mlx5_core_dev *mdev = priv->mdev; 4194 4195 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1, 4196 vf_stats); 4197 } 4198 #endif 4199 4200 struct mlx5e_vxlan_work { 4201 struct work_struct work; 4202 struct mlx5e_priv *priv; 4203 u16 port; 4204 }; 4205 4206 static void mlx5e_vxlan_add_work(struct work_struct *work) 4207 { 4208 struct mlx5e_vxlan_work *vxlan_work = 4209 container_of(work, struct mlx5e_vxlan_work, work); 4210 struct mlx5e_priv *priv = vxlan_work->priv; 4211 u16 port = vxlan_work->port; 4212 4213 mutex_lock(&priv->state_lock); 4214 mlx5_vxlan_add_port(priv->mdev->vxlan, port); 4215 mutex_unlock(&priv->state_lock); 4216 4217 kfree(vxlan_work); 4218 } 4219 4220 static void mlx5e_vxlan_del_work(struct work_struct *work) 4221 { 4222 struct mlx5e_vxlan_work *vxlan_work = 4223 container_of(work, struct mlx5e_vxlan_work, work); 4224 struct mlx5e_priv *priv = vxlan_work->priv; 4225 u16 port = vxlan_work->port; 4226 4227 mutex_lock(&priv->state_lock); 4228 mlx5_vxlan_del_port(priv->mdev->vxlan, port); 4229 mutex_unlock(&priv->state_lock); 4230 kfree(vxlan_work); 4231 } 4232 4233 static void mlx5e_vxlan_queue_work(struct mlx5e_priv *priv, u16 port, int add) 4234 { 4235 struct mlx5e_vxlan_work *vxlan_work; 4236 4237 vxlan_work = kmalloc(sizeof(*vxlan_work), GFP_ATOMIC); 4238 if (!vxlan_work) 4239 return; 4240 4241 if (add) 4242 INIT_WORK(&vxlan_work->work, mlx5e_vxlan_add_work); 4243 else 4244 INIT_WORK(&vxlan_work->work, mlx5e_vxlan_del_work); 4245 4246 vxlan_work->priv = priv; 4247 vxlan_work->port = port; 4248 queue_work(priv->wq, &vxlan_work->work); 4249 } 4250 4251 void mlx5e_add_vxlan_port(struct net_device *netdev, struct udp_tunnel_info *ti) 4252 { 4253 struct mlx5e_priv *priv = netdev_priv(netdev); 4254 4255 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 4256 return; 4257 4258 if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) 4259 return; 4260 4261 mlx5e_vxlan_queue_work(priv, be16_to_cpu(ti->port), 1); 4262 } 4263 4264 void mlx5e_del_vxlan_port(struct net_device *netdev, struct udp_tunnel_info *ti) 4265 { 4266 struct mlx5e_priv *priv = netdev_priv(netdev); 4267 4268 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 4269 return; 4270 4271 if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) 4272 return; 4273 4274 mlx5e_vxlan_queue_work(priv, be16_to_cpu(ti->port), 0); 4275 } 4276 4277 static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv, 4278 struct sk_buff *skb, 4279 netdev_features_t features) 4280 { 4281 unsigned int offset = 0; 4282 struct udphdr *udph; 4283 u8 proto; 4284 u16 port; 4285 4286 switch (vlan_get_protocol(skb)) { 4287 case htons(ETH_P_IP): 4288 proto = ip_hdr(skb)->protocol; 4289 break; 4290 case htons(ETH_P_IPV6): 4291 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL); 4292 break; 4293 default: 4294 goto out; 4295 } 4296 4297 switch (proto) { 4298 case IPPROTO_GRE: 4299 return features; 4300 case IPPROTO_IPIP: 4301 case IPPROTO_IPV6: 4302 if (mlx5e_tunnel_proto_supported(priv->mdev, IPPROTO_IPIP)) 4303 return features; 4304 break; 4305 case IPPROTO_UDP: 4306 udph = udp_hdr(skb); 4307 port = be16_to_cpu(udph->dest); 4308 4309 /* Verify if UDP port is being offloaded by HW */ 4310 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port)) 4311 return features; 4312 4313 #if IS_ENABLED(CONFIG_GENEVE) 4314 /* Support Geneve offload for default UDP port */ 4315 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev)) 4316 return features; 4317 #endif 4318 } 4319 4320 out: 4321 /* Disable CSUM and GSO if the udp dport is not offloaded by HW */ 4322 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 4323 } 4324 4325 netdev_features_t mlx5e_features_check(struct sk_buff *skb, 4326 struct net_device *netdev, 4327 netdev_features_t features) 4328 { 4329 struct mlx5e_priv *priv = netdev_priv(netdev); 4330 4331 features = vlan_features_check(skb, features); 4332 features = vxlan_features_check(skb, features); 4333 4334 #ifdef CONFIG_MLX5_EN_IPSEC 4335 if (mlx5e_ipsec_feature_check(skb, netdev, features)) 4336 return features; 4337 #endif 4338 4339 /* Validate if the tunneled packet is being offloaded by HW */ 4340 if (skb->encapsulation && 4341 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK)) 4342 return mlx5e_tunnel_features_check(priv, skb, features); 4343 4344 return features; 4345 } 4346 4347 static void mlx5e_tx_timeout_work(struct work_struct *work) 4348 { 4349 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 4350 tx_timeout_work); 4351 bool report_failed = false; 4352 int err; 4353 int i; 4354 4355 rtnl_lock(); 4356 mutex_lock(&priv->state_lock); 4357 4358 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 4359 goto unlock; 4360 4361 for (i = 0; i < priv->channels.num * priv->channels.params.num_tc; i++) { 4362 struct netdev_queue *dev_queue = 4363 netdev_get_tx_queue(priv->netdev, i); 4364 struct mlx5e_txqsq *sq = priv->txq2sq[i]; 4365 4366 if (!netif_xmit_stopped(dev_queue)) 4367 continue; 4368 4369 if (mlx5e_reporter_tx_timeout(sq)) 4370 report_failed = true; 4371 } 4372 4373 if (!report_failed) 4374 goto unlock; 4375 4376 err = mlx5e_safe_reopen_channels(priv); 4377 if (err) 4378 netdev_err(priv->netdev, 4379 "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n", 4380 err); 4381 4382 unlock: 4383 mutex_unlock(&priv->state_lock); 4384 rtnl_unlock(); 4385 } 4386 4387 static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue) 4388 { 4389 struct mlx5e_priv *priv = netdev_priv(dev); 4390 4391 netdev_err(dev, "TX timeout detected\n"); 4392 queue_work(priv->wq, &priv->tx_timeout_work); 4393 } 4394 4395 static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog) 4396 { 4397 struct net_device *netdev = priv->netdev; 4398 struct mlx5e_channels new_channels = {}; 4399 4400 if (priv->channels.params.lro_en) { 4401 netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n"); 4402 return -EINVAL; 4403 } 4404 4405 if (MLX5_IPSEC_DEV(priv->mdev)) { 4406 netdev_warn(netdev, "can't set XDP with IPSec offload\n"); 4407 return -EINVAL; 4408 } 4409 4410 new_channels.params = priv->channels.params; 4411 new_channels.params.xdp_prog = prog; 4412 4413 /* No XSK params: AF_XDP can't be enabled yet at the point of setting 4414 * the XDP program. 4415 */ 4416 if (!mlx5e_rx_is_linear_skb(&new_channels.params, NULL)) { 4417 netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n", 4418 new_channels.params.sw_mtu, 4419 mlx5e_xdp_max_mtu(&new_channels.params, NULL)); 4420 return -EINVAL; 4421 } 4422 4423 return 0; 4424 } 4425 4426 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) 4427 { 4428 struct mlx5e_priv *priv = netdev_priv(netdev); 4429 struct bpf_prog *old_prog; 4430 bool reset, was_opened; 4431 int err = 0; 4432 int i; 4433 4434 mutex_lock(&priv->state_lock); 4435 4436 if (prog) { 4437 err = mlx5e_xdp_allowed(priv, prog); 4438 if (err) 4439 goto unlock; 4440 } 4441 4442 was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state); 4443 /* no need for full reset when exchanging programs */ 4444 reset = (!priv->channels.params.xdp_prog || !prog); 4445 4446 if (was_opened && !reset) 4447 /* num_channels is invariant here, so we can take the 4448 * batched reference right upfront. 4449 */ 4450 bpf_prog_add(prog, priv->channels.num); 4451 4452 if (was_opened && reset) { 4453 struct mlx5e_channels new_channels = {}; 4454 4455 new_channels.params = priv->channels.params; 4456 new_channels.params.xdp_prog = prog; 4457 mlx5e_set_rq_type(priv->mdev, &new_channels.params); 4458 old_prog = priv->channels.params.xdp_prog; 4459 4460 err = mlx5e_safe_switch_channels(priv, &new_channels, NULL, NULL); 4461 if (err) 4462 goto unlock; 4463 } else { 4464 /* exchange programs, extra prog reference we got from caller 4465 * as long as we don't fail from this point onwards. 4466 */ 4467 old_prog = xchg(&priv->channels.params.xdp_prog, prog); 4468 } 4469 4470 if (old_prog) 4471 bpf_prog_put(old_prog); 4472 4473 if (!was_opened && reset) /* change RQ type according to priv->xdp_prog */ 4474 mlx5e_set_rq_type(priv->mdev, &priv->channels.params); 4475 4476 if (!was_opened || reset) 4477 goto unlock; 4478 4479 /* exchanging programs w/o reset, we update ref counts on behalf 4480 * of the channels RQs here. 4481 */ 4482 for (i = 0; i < priv->channels.num; i++) { 4483 struct mlx5e_channel *c = priv->channels.c[i]; 4484 bool xsk_open = test_bit(MLX5E_CHANNEL_STATE_XSK, c->state); 4485 4486 clear_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state); 4487 if (xsk_open) 4488 clear_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state); 4489 napi_synchronize(&c->napi); 4490 /* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */ 4491 4492 old_prog = xchg(&c->rq.xdp_prog, prog); 4493 if (old_prog) 4494 bpf_prog_put(old_prog); 4495 4496 if (xsk_open) { 4497 old_prog = xchg(&c->xskrq.xdp_prog, prog); 4498 if (old_prog) 4499 bpf_prog_put(old_prog); 4500 } 4501 4502 set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state); 4503 if (xsk_open) 4504 set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state); 4505 /* napi_schedule in case we have missed anything */ 4506 napi_schedule(&c->napi); 4507 } 4508 4509 unlock: 4510 mutex_unlock(&priv->state_lock); 4511 return err; 4512 } 4513 4514 static u32 mlx5e_xdp_query(struct net_device *dev) 4515 { 4516 struct mlx5e_priv *priv = netdev_priv(dev); 4517 const struct bpf_prog *xdp_prog; 4518 u32 prog_id = 0; 4519 4520 mutex_lock(&priv->state_lock); 4521 xdp_prog = priv->channels.params.xdp_prog; 4522 if (xdp_prog) 4523 prog_id = xdp_prog->aux->id; 4524 mutex_unlock(&priv->state_lock); 4525 4526 return prog_id; 4527 } 4528 4529 static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp) 4530 { 4531 switch (xdp->command) { 4532 case XDP_SETUP_PROG: 4533 return mlx5e_xdp_set(dev, xdp->prog); 4534 case XDP_QUERY_PROG: 4535 xdp->prog_id = mlx5e_xdp_query(dev); 4536 return 0; 4537 case XDP_SETUP_XSK_UMEM: 4538 return mlx5e_xsk_setup_umem(dev, xdp->xsk.umem, 4539 xdp->xsk.queue_id); 4540 default: 4541 return -EINVAL; 4542 } 4543 } 4544 4545 #ifdef CONFIG_MLX5_ESWITCH 4546 static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 4547 struct net_device *dev, u32 filter_mask, 4548 int nlflags) 4549 { 4550 struct mlx5e_priv *priv = netdev_priv(dev); 4551 struct mlx5_core_dev *mdev = priv->mdev; 4552 u8 mode, setting; 4553 int err; 4554 4555 err = mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting); 4556 if (err) 4557 return err; 4558 mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB; 4559 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 4560 mode, 4561 0, 0, nlflags, filter_mask, NULL); 4562 } 4563 4564 static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 4565 u16 flags, struct netlink_ext_ack *extack) 4566 { 4567 struct mlx5e_priv *priv = netdev_priv(dev); 4568 struct mlx5_core_dev *mdev = priv->mdev; 4569 struct nlattr *attr, *br_spec; 4570 u16 mode = BRIDGE_MODE_UNDEF; 4571 u8 setting; 4572 int rem; 4573 4574 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 4575 if (!br_spec) 4576 return -EINVAL; 4577 4578 nla_for_each_nested(attr, br_spec, rem) { 4579 if (nla_type(attr) != IFLA_BRIDGE_MODE) 4580 continue; 4581 4582 if (nla_len(attr) < sizeof(mode)) 4583 return -EINVAL; 4584 4585 mode = nla_get_u16(attr); 4586 if (mode > BRIDGE_MODE_VEPA) 4587 return -EINVAL; 4588 4589 break; 4590 } 4591 4592 if (mode == BRIDGE_MODE_UNDEF) 4593 return -EINVAL; 4594 4595 setting = (mode == BRIDGE_MODE_VEPA) ? 1 : 0; 4596 return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting); 4597 } 4598 #endif 4599 4600 const struct net_device_ops mlx5e_netdev_ops = { 4601 .ndo_open = mlx5e_open, 4602 .ndo_stop = mlx5e_close, 4603 .ndo_start_xmit = mlx5e_xmit, 4604 .ndo_setup_tc = mlx5e_setup_tc, 4605 .ndo_select_queue = mlx5e_select_queue, 4606 .ndo_get_stats64 = mlx5e_get_stats, 4607 .ndo_set_rx_mode = mlx5e_set_rx_mode, 4608 .ndo_set_mac_address = mlx5e_set_mac, 4609 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid, 4610 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid, 4611 .ndo_set_features = mlx5e_set_features, 4612 .ndo_fix_features = mlx5e_fix_features, 4613 .ndo_change_mtu = mlx5e_change_nic_mtu, 4614 .ndo_do_ioctl = mlx5e_ioctl, 4615 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate, 4616 .ndo_udp_tunnel_add = mlx5e_add_vxlan_port, 4617 .ndo_udp_tunnel_del = mlx5e_del_vxlan_port, 4618 .ndo_features_check = mlx5e_features_check, 4619 .ndo_tx_timeout = mlx5e_tx_timeout, 4620 .ndo_bpf = mlx5e_xdp, 4621 .ndo_xdp_xmit = mlx5e_xdp_xmit, 4622 .ndo_xsk_wakeup = mlx5e_xsk_wakeup, 4623 #ifdef CONFIG_MLX5_EN_ARFS 4624 .ndo_rx_flow_steer = mlx5e_rx_flow_steer, 4625 #endif 4626 #ifdef CONFIG_MLX5_ESWITCH 4627 .ndo_bridge_setlink = mlx5e_bridge_setlink, 4628 .ndo_bridge_getlink = mlx5e_bridge_getlink, 4629 4630 /* SRIOV E-Switch NDOs */ 4631 .ndo_set_vf_mac = mlx5e_set_vf_mac, 4632 .ndo_set_vf_vlan = mlx5e_set_vf_vlan, 4633 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk, 4634 .ndo_set_vf_trust = mlx5e_set_vf_trust, 4635 .ndo_set_vf_rate = mlx5e_set_vf_rate, 4636 .ndo_get_vf_config = mlx5e_get_vf_config, 4637 .ndo_set_vf_link_state = mlx5e_set_vf_link_state, 4638 .ndo_get_vf_stats = mlx5e_get_vf_stats, 4639 #endif 4640 .ndo_get_devlink_port = mlx5e_get_devlink_port, 4641 }; 4642 4643 static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev) 4644 { 4645 if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) 4646 return -EOPNOTSUPP; 4647 if (!MLX5_CAP_GEN(mdev, eth_net_offloads) || 4648 !MLX5_CAP_GEN(mdev, nic_flow_table) || 4649 !MLX5_CAP_ETH(mdev, csum_cap) || 4650 !MLX5_CAP_ETH(mdev, max_lso_cap) || 4651 !MLX5_CAP_ETH(mdev, vlan_cap) || 4652 !MLX5_CAP_ETH(mdev, rss_ind_tbl_cap) || 4653 MLX5_CAP_FLOWTABLE(mdev, 4654 flow_table_properties_nic_receive.max_ft_level) 4655 < 3) { 4656 mlx5_core_warn(mdev, 4657 "Not creating net device, some required device capabilities are missing\n"); 4658 return -EOPNOTSUPP; 4659 } 4660 if (!MLX5_CAP_ETH(mdev, self_lb_en_modifiable)) 4661 mlx5_core_warn(mdev, "Self loop back prevention is not supported\n"); 4662 if (!MLX5_CAP_GEN(mdev, cq_moderation)) 4663 mlx5_core_warn(mdev, "CQ moderation is not supported\n"); 4664 4665 return 0; 4666 } 4667 4668 void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len, 4669 int num_channels) 4670 { 4671 int i; 4672 4673 for (i = 0; i < len; i++) 4674 indirection_rqt[i] = i % num_channels; 4675 } 4676 4677 static bool slow_pci_heuristic(struct mlx5_core_dev *mdev) 4678 { 4679 u32 link_speed = 0; 4680 u32 pci_bw = 0; 4681 4682 mlx5e_port_max_linkspeed(mdev, &link_speed); 4683 pci_bw = pcie_bandwidth_available(mdev->pdev, NULL, NULL, NULL); 4684 mlx5_core_dbg_once(mdev, "Max link speed = %d, PCI BW = %d\n", 4685 link_speed, pci_bw); 4686 4687 #define MLX5E_SLOW_PCI_RATIO (2) 4688 4689 return link_speed && pci_bw && 4690 link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw; 4691 } 4692 4693 static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) 4694 { 4695 struct dim_cq_moder moder; 4696 4697 moder.cq_period_mode = cq_period_mode; 4698 moder.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS; 4699 moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC; 4700 if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE) 4701 moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE; 4702 4703 return moder; 4704 } 4705 4706 static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode) 4707 { 4708 struct dim_cq_moder moder; 4709 4710 moder.cq_period_mode = cq_period_mode; 4711 moder.pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS; 4712 moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC; 4713 if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE) 4714 moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE; 4715 4716 return moder; 4717 } 4718 4719 static u8 mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode) 4720 { 4721 return cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE ? 4722 DIM_CQ_PERIOD_MODE_START_FROM_CQE : 4723 DIM_CQ_PERIOD_MODE_START_FROM_EQE; 4724 } 4725 4726 void mlx5e_reset_tx_moderation(struct mlx5e_params *params, u8 cq_period_mode) 4727 { 4728 if (params->tx_dim_enabled) { 4729 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode); 4730 4731 params->tx_cq_moderation = net_dim_get_def_tx_moderation(dim_period_mode); 4732 } else { 4733 params->tx_cq_moderation = mlx5e_get_def_tx_moderation(cq_period_mode); 4734 } 4735 } 4736 4737 void mlx5e_reset_rx_moderation(struct mlx5e_params *params, u8 cq_period_mode) 4738 { 4739 if (params->rx_dim_enabled) { 4740 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode); 4741 4742 params->rx_cq_moderation = net_dim_get_def_rx_moderation(dim_period_mode); 4743 } else { 4744 params->rx_cq_moderation = mlx5e_get_def_rx_moderation(cq_period_mode); 4745 } 4746 } 4747 4748 void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode) 4749 { 4750 mlx5e_reset_tx_moderation(params, cq_period_mode); 4751 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_TX_CQE_BASED_MODER, 4752 params->tx_cq_moderation.cq_period_mode == 4753 MLX5_CQ_PERIOD_MODE_START_FROM_CQE); 4754 } 4755 4756 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode) 4757 { 4758 mlx5e_reset_rx_moderation(params, cq_period_mode); 4759 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER, 4760 params->rx_cq_moderation.cq_period_mode == 4761 MLX5_CQ_PERIOD_MODE_START_FROM_CQE); 4762 } 4763 4764 static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout) 4765 { 4766 int i; 4767 4768 /* The supported periods are organized in ascending order */ 4769 for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++) 4770 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout) 4771 break; 4772 4773 return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]); 4774 } 4775 4776 void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, 4777 struct mlx5e_params *params) 4778 { 4779 /* Prefer Striding RQ, unless any of the following holds: 4780 * - Striding RQ configuration is not possible/supported. 4781 * - Slow PCI heuristic. 4782 * - Legacy RQ would use linear SKB while Striding RQ would use non-linear. 4783 * 4784 * No XSK params: checking the availability of striding RQ in general. 4785 */ 4786 if (!slow_pci_heuristic(mdev) && 4787 mlx5e_striding_rq_possible(mdev, params) && 4788 (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) || 4789 !mlx5e_rx_is_linear_skb(params, NULL))) 4790 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true); 4791 mlx5e_set_rq_type(mdev, params); 4792 mlx5e_init_rq_type_params(mdev, params); 4793 } 4794 4795 void mlx5e_build_rss_params(struct mlx5e_rss_params *rss_params, 4796 u16 num_channels) 4797 { 4798 enum mlx5e_traffic_types tt; 4799 4800 rss_params->hfunc = ETH_RSS_HASH_TOP; 4801 netdev_rss_key_fill(rss_params->toeplitz_hash_key, 4802 sizeof(rss_params->toeplitz_hash_key)); 4803 mlx5e_build_default_indir_rqt(rss_params->indirection_rqt, 4804 MLX5E_INDIR_RQT_SIZE, num_channels); 4805 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) 4806 rss_params->rx_hash_fields[tt] = 4807 tirc_default_config[tt].rx_hash_fields; 4808 } 4809 4810 void mlx5e_build_nic_params(struct mlx5e_priv *priv, 4811 struct mlx5e_xsk *xsk, 4812 struct mlx5e_rss_params *rss_params, 4813 struct mlx5e_params *params, 4814 u16 mtu) 4815 { 4816 struct mlx5_core_dev *mdev = priv->mdev; 4817 u8 rx_cq_period_mode; 4818 4819 params->sw_mtu = mtu; 4820 params->hard_mtu = MLX5E_ETH_HARD_MTU; 4821 params->num_channels = min_t(unsigned int, MLX5E_MAX_NUM_CHANNELS / 2, 4822 priv->max_nch); 4823 params->num_tc = 1; 4824 4825 /* SQ */ 4826 params->log_sq_size = is_kdump_kernel() ? 4827 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE : 4828 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; 4829 4830 /* XDP SQ */ 4831 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, 4832 MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe)); 4833 4834 /* set CQE compression */ 4835 params->rx_cqe_compress_def = false; 4836 if (MLX5_CAP_GEN(mdev, cqe_compression) && 4837 MLX5_CAP_GEN(mdev, vport_group_manager)) 4838 params->rx_cqe_compress_def = slow_pci_heuristic(mdev); 4839 4840 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def); 4841 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false); 4842 4843 /* RQ */ 4844 mlx5e_build_rq_params(mdev, params); 4845 4846 /* HW LRO */ 4847 if (MLX5_CAP_ETH(mdev, lro_cap) && 4848 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 4849 /* No XSK params: checking the availability of striding RQ in general. */ 4850 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL)) 4851 params->lro_en = !slow_pci_heuristic(mdev); 4852 } 4853 params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT); 4854 4855 /* CQ moderation params */ 4856 rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 4857 MLX5_CQ_PERIOD_MODE_START_FROM_CQE : 4858 MLX5_CQ_PERIOD_MODE_START_FROM_EQE; 4859 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 4860 params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 4861 mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode); 4862 mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE); 4863 4864 /* TX inline */ 4865 mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); 4866 4867 /* RSS */ 4868 mlx5e_build_rss_params(rss_params, params->num_channels); 4869 params->tunneled_offload_en = 4870 mlx5e_tunnel_inner_ft_supported(mdev); 4871 4872 /* AF_XDP */ 4873 params->xsk = xsk; 4874 } 4875 4876 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev) 4877 { 4878 struct mlx5e_priv *priv = netdev_priv(netdev); 4879 4880 mlx5_query_mac_address(priv->mdev, netdev->dev_addr); 4881 if (is_zero_ether_addr(netdev->dev_addr) && 4882 !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) { 4883 eth_hw_addr_random(netdev); 4884 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr); 4885 } 4886 } 4887 4888 static void mlx5e_build_nic_netdev(struct net_device *netdev) 4889 { 4890 struct mlx5e_priv *priv = netdev_priv(netdev); 4891 struct mlx5_core_dev *mdev = priv->mdev; 4892 bool fcs_supported; 4893 bool fcs_enabled; 4894 4895 SET_NETDEV_DEV(netdev, mdev->device); 4896 4897 netdev->netdev_ops = &mlx5e_netdev_ops; 4898 4899 mlx5e_dcbnl_build_netdev(netdev); 4900 4901 netdev->watchdog_timeo = 15 * HZ; 4902 4903 netdev->ethtool_ops = &mlx5e_ethtool_ops; 4904 4905 netdev->vlan_features |= NETIF_F_SG; 4906 netdev->vlan_features |= NETIF_F_HW_CSUM; 4907 netdev->vlan_features |= NETIF_F_GRO; 4908 netdev->vlan_features |= NETIF_F_TSO; 4909 netdev->vlan_features |= NETIF_F_TSO6; 4910 netdev->vlan_features |= NETIF_F_RXCSUM; 4911 netdev->vlan_features |= NETIF_F_RXHASH; 4912 4913 netdev->mpls_features |= NETIF_F_SG; 4914 netdev->mpls_features |= NETIF_F_HW_CSUM; 4915 netdev->mpls_features |= NETIF_F_TSO; 4916 netdev->mpls_features |= NETIF_F_TSO6; 4917 4918 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_TX; 4919 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_RX; 4920 4921 if (!!MLX5_CAP_ETH(mdev, lro_cap) && 4922 mlx5e_check_fragmented_striding_rq_cap(mdev)) 4923 netdev->vlan_features |= NETIF_F_LRO; 4924 4925 netdev->hw_features = netdev->vlan_features; 4926 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 4927 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 4928 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; 4929 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 4930 4931 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev) || 4932 mlx5e_any_tunnel_proto_supported(mdev)) { 4933 netdev->hw_enc_features |= NETIF_F_HW_CSUM; 4934 netdev->hw_enc_features |= NETIF_F_TSO; 4935 netdev->hw_enc_features |= NETIF_F_TSO6; 4936 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL; 4937 } 4938 4939 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) { 4940 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 4941 NETIF_F_GSO_UDP_TUNNEL_CSUM; 4942 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 4943 NETIF_F_GSO_UDP_TUNNEL_CSUM; 4944 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; 4945 netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL | 4946 NETIF_F_GSO_UDP_TUNNEL_CSUM; 4947 } 4948 4949 if (mlx5e_tunnel_proto_supported(mdev, IPPROTO_GRE)) { 4950 netdev->hw_features |= NETIF_F_GSO_GRE | 4951 NETIF_F_GSO_GRE_CSUM; 4952 netdev->hw_enc_features |= NETIF_F_GSO_GRE | 4953 NETIF_F_GSO_GRE_CSUM; 4954 netdev->gso_partial_features |= NETIF_F_GSO_GRE | 4955 NETIF_F_GSO_GRE_CSUM; 4956 } 4957 4958 if (mlx5e_tunnel_proto_supported(mdev, IPPROTO_IPIP)) { 4959 netdev->hw_features |= NETIF_F_GSO_IPXIP4 | 4960 NETIF_F_GSO_IPXIP6; 4961 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4 | 4962 NETIF_F_GSO_IPXIP6; 4963 netdev->gso_partial_features |= NETIF_F_GSO_IPXIP4 | 4964 NETIF_F_GSO_IPXIP6; 4965 } 4966 4967 netdev->hw_features |= NETIF_F_GSO_PARTIAL; 4968 netdev->gso_partial_features |= NETIF_F_GSO_UDP_L4; 4969 netdev->hw_features |= NETIF_F_GSO_UDP_L4; 4970 netdev->features |= NETIF_F_GSO_UDP_L4; 4971 4972 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled); 4973 4974 if (fcs_supported) 4975 netdev->hw_features |= NETIF_F_RXALL; 4976 4977 if (MLX5_CAP_ETH(mdev, scatter_fcs)) 4978 netdev->hw_features |= NETIF_F_RXFCS; 4979 4980 netdev->features = netdev->hw_features; 4981 if (!priv->channels.params.lro_en) 4982 netdev->features &= ~NETIF_F_LRO; 4983 4984 if (fcs_enabled) 4985 netdev->features &= ~NETIF_F_RXALL; 4986 4987 if (!priv->channels.params.scatter_fcs_en) 4988 netdev->features &= ~NETIF_F_RXFCS; 4989 4990 /* prefere CQE compression over rxhash */ 4991 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS)) 4992 netdev->features &= ~NETIF_F_RXHASH; 4993 4994 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f) 4995 if (FT_CAP(flow_modify_en) && 4996 FT_CAP(modify_root) && 4997 FT_CAP(identified_miss_table_mode) && 4998 FT_CAP(flow_table_modify)) { 4999 #ifdef CONFIG_MLX5_ESWITCH 5000 netdev->hw_features |= NETIF_F_HW_TC; 5001 #endif 5002 #ifdef CONFIG_MLX5_EN_ARFS 5003 netdev->hw_features |= NETIF_F_NTUPLE; 5004 #endif 5005 } 5006 5007 netdev->features |= NETIF_F_HIGHDMA; 5008 netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 5009 5010 netdev->priv_flags |= IFF_UNICAST_FLT; 5011 5012 mlx5e_set_netdev_dev_addr(netdev); 5013 mlx5e_ipsec_build_netdev(priv); 5014 mlx5e_tls_build_netdev(priv); 5015 } 5016 5017 void mlx5e_create_q_counters(struct mlx5e_priv *priv) 5018 { 5019 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 5020 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 5021 struct mlx5_core_dev *mdev = priv->mdev; 5022 int err; 5023 5024 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); 5025 err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out); 5026 if (!err) 5027 priv->q_counter = 5028 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 5029 5030 err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out); 5031 if (!err) 5032 priv->drop_rq_q_counter = 5033 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 5034 } 5035 5036 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv) 5037 { 5038 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 5039 5040 MLX5_SET(dealloc_q_counter_in, in, opcode, 5041 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 5042 if (priv->q_counter) { 5043 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 5044 priv->q_counter); 5045 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in); 5046 } 5047 5048 if (priv->drop_rq_q_counter) { 5049 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 5050 priv->drop_rq_q_counter); 5051 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in); 5052 } 5053 } 5054 5055 static int mlx5e_nic_init(struct mlx5_core_dev *mdev, 5056 struct net_device *netdev, 5057 const struct mlx5e_profile *profile, 5058 void *ppriv) 5059 { 5060 struct mlx5e_priv *priv = netdev_priv(netdev); 5061 struct mlx5e_rss_params *rss = &priv->rss_params; 5062 int err; 5063 5064 err = mlx5e_netdev_init(netdev, priv, mdev, profile, ppriv); 5065 if (err) 5066 return err; 5067 5068 mlx5e_build_nic_params(priv, &priv->xsk, rss, &priv->channels.params, 5069 netdev->mtu); 5070 5071 mlx5e_timestamp_init(priv); 5072 5073 err = mlx5e_ipsec_init(priv); 5074 if (err) 5075 mlx5_core_err(mdev, "IPSec initialization failed, %d\n", err); 5076 err = mlx5e_tls_init(priv); 5077 if (err) 5078 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err); 5079 mlx5e_build_nic_netdev(netdev); 5080 mlx5e_health_create_reporters(priv); 5081 5082 return 0; 5083 } 5084 5085 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) 5086 { 5087 mlx5e_health_destroy_reporters(priv); 5088 mlx5e_tls_cleanup(priv); 5089 mlx5e_ipsec_cleanup(priv); 5090 mlx5e_netdev_cleanup(priv->netdev, priv); 5091 } 5092 5093 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) 5094 { 5095 struct mlx5_core_dev *mdev = priv->mdev; 5096 int err; 5097 5098 mlx5e_create_q_counters(priv); 5099 5100 err = mlx5e_open_drop_rq(priv, &priv->drop_rq); 5101 if (err) { 5102 mlx5_core_err(mdev, "open drop rq failed, %d\n", err); 5103 goto err_destroy_q_counters; 5104 } 5105 5106 err = mlx5e_create_indirect_rqt(priv); 5107 if (err) 5108 goto err_close_drop_rq; 5109 5110 err = mlx5e_create_direct_rqts(priv, priv->direct_tir); 5111 if (err) 5112 goto err_destroy_indirect_rqts; 5113 5114 err = mlx5e_create_indirect_tirs(priv, true); 5115 if (err) 5116 goto err_destroy_direct_rqts; 5117 5118 err = mlx5e_create_direct_tirs(priv, priv->direct_tir); 5119 if (err) 5120 goto err_destroy_indirect_tirs; 5121 5122 err = mlx5e_create_direct_rqts(priv, priv->xsk_tir); 5123 if (unlikely(err)) 5124 goto err_destroy_direct_tirs; 5125 5126 err = mlx5e_create_direct_tirs(priv, priv->xsk_tir); 5127 if (unlikely(err)) 5128 goto err_destroy_xsk_rqts; 5129 5130 err = mlx5e_create_flow_steering(priv); 5131 if (err) { 5132 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err); 5133 goto err_destroy_xsk_tirs; 5134 } 5135 5136 err = mlx5e_tc_nic_init(priv); 5137 if (err) 5138 goto err_destroy_flow_steering; 5139 5140 #ifdef CONFIG_MLX5_EN_ARFS 5141 priv->netdev->rx_cpu_rmap = mlx5_eq_table_get_rmap(priv->mdev); 5142 #endif 5143 5144 return 0; 5145 5146 err_destroy_flow_steering: 5147 mlx5e_destroy_flow_steering(priv); 5148 err_destroy_xsk_tirs: 5149 mlx5e_destroy_direct_tirs(priv, priv->xsk_tir); 5150 err_destroy_xsk_rqts: 5151 mlx5e_destroy_direct_rqts(priv, priv->xsk_tir); 5152 err_destroy_direct_tirs: 5153 mlx5e_destroy_direct_tirs(priv, priv->direct_tir); 5154 err_destroy_indirect_tirs: 5155 mlx5e_destroy_indirect_tirs(priv); 5156 err_destroy_direct_rqts: 5157 mlx5e_destroy_direct_rqts(priv, priv->direct_tir); 5158 err_destroy_indirect_rqts: 5159 mlx5e_destroy_rqt(priv, &priv->indir_rqt); 5160 err_close_drop_rq: 5161 mlx5e_close_drop_rq(&priv->drop_rq); 5162 err_destroy_q_counters: 5163 mlx5e_destroy_q_counters(priv); 5164 return err; 5165 } 5166 5167 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) 5168 { 5169 mlx5e_tc_nic_cleanup(priv); 5170 mlx5e_destroy_flow_steering(priv); 5171 mlx5e_destroy_direct_tirs(priv, priv->xsk_tir); 5172 mlx5e_destroy_direct_rqts(priv, priv->xsk_tir); 5173 mlx5e_destroy_direct_tirs(priv, priv->direct_tir); 5174 mlx5e_destroy_indirect_tirs(priv); 5175 mlx5e_destroy_direct_rqts(priv, priv->direct_tir); 5176 mlx5e_destroy_rqt(priv, &priv->indir_rqt); 5177 mlx5e_close_drop_rq(&priv->drop_rq); 5178 mlx5e_destroy_q_counters(priv); 5179 } 5180 5181 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) 5182 { 5183 int err; 5184 5185 err = mlx5e_create_tises(priv); 5186 if (err) { 5187 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err); 5188 return err; 5189 } 5190 5191 mlx5e_dcbnl_initialize(priv); 5192 return 0; 5193 } 5194 5195 static void mlx5e_nic_enable(struct mlx5e_priv *priv) 5196 { 5197 struct net_device *netdev = priv->netdev; 5198 struct mlx5_core_dev *mdev = priv->mdev; 5199 5200 mlx5e_init_l2_addr(priv); 5201 5202 /* Marking the link as currently not needed by the Driver */ 5203 if (!netif_running(netdev)) 5204 mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN); 5205 5206 mlx5e_set_netdev_mtu_boundaries(priv); 5207 mlx5e_set_dev_port_mtu(priv); 5208 5209 mlx5_lag_add(mdev, netdev); 5210 5211 mlx5e_enable_async_events(priv); 5212 if (mlx5e_monitor_counter_supported(priv)) 5213 mlx5e_monitor_counter_init(priv); 5214 5215 mlx5e_hv_vhca_stats_create(priv); 5216 if (netdev->reg_state != NETREG_REGISTERED) 5217 return; 5218 mlx5e_dcbnl_init_app(priv); 5219 5220 queue_work(priv->wq, &priv->set_rx_mode_work); 5221 5222 rtnl_lock(); 5223 if (netif_running(netdev)) 5224 mlx5e_open(netdev); 5225 if (mlx5_vxlan_allowed(priv->mdev->vxlan)) 5226 udp_tunnel_get_rx_info(netdev); 5227 netif_device_attach(netdev); 5228 rtnl_unlock(); 5229 } 5230 5231 static void mlx5e_nic_disable(struct mlx5e_priv *priv) 5232 { 5233 struct mlx5_core_dev *mdev = priv->mdev; 5234 5235 if (priv->netdev->reg_state == NETREG_REGISTERED) 5236 mlx5e_dcbnl_delete_app(priv); 5237 5238 rtnl_lock(); 5239 if (netif_running(priv->netdev)) 5240 mlx5e_close(priv->netdev); 5241 if (mlx5_vxlan_allowed(priv->mdev->vxlan)) 5242 udp_tunnel_drop_rx_info(priv->netdev); 5243 netif_device_detach(priv->netdev); 5244 rtnl_unlock(); 5245 5246 queue_work(priv->wq, &priv->set_rx_mode_work); 5247 5248 mlx5e_hv_vhca_stats_destroy(priv); 5249 if (mlx5e_monitor_counter_supported(priv)) 5250 mlx5e_monitor_counter_cleanup(priv); 5251 5252 mlx5e_disable_async_events(priv); 5253 mlx5_lag_remove(mdev); 5254 } 5255 5256 int mlx5e_update_nic_rx(struct mlx5e_priv *priv) 5257 { 5258 return mlx5e_refresh_tirs(priv, false, false); 5259 } 5260 5261 static const struct mlx5e_profile mlx5e_nic_profile = { 5262 .init = mlx5e_nic_init, 5263 .cleanup = mlx5e_nic_cleanup, 5264 .init_rx = mlx5e_init_nic_rx, 5265 .cleanup_rx = mlx5e_cleanup_nic_rx, 5266 .init_tx = mlx5e_init_nic_tx, 5267 .cleanup_tx = mlx5e_cleanup_nic_tx, 5268 .enable = mlx5e_nic_enable, 5269 .disable = mlx5e_nic_disable, 5270 .update_rx = mlx5e_update_nic_rx, 5271 .update_stats = mlx5e_update_ndo_stats, 5272 .update_carrier = mlx5e_update_carrier, 5273 .rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe, 5274 .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq, 5275 .max_tc = MLX5E_MAX_NUM_TC, 5276 .rq_groups = MLX5E_NUM_RQ_GROUPS(XSK), 5277 .stats_grps = mlx5e_nic_stats_grps, 5278 .stats_grps_num = mlx5e_nic_stats_grps_num, 5279 }; 5280 5281 /* mlx5e generic netdev management API (move to en_common.c) */ 5282 5283 /* mlx5e_netdev_init/cleanup must be called from profile->init/cleanup callbacks */ 5284 int mlx5e_netdev_init(struct net_device *netdev, 5285 struct mlx5e_priv *priv, 5286 struct mlx5_core_dev *mdev, 5287 const struct mlx5e_profile *profile, 5288 void *ppriv) 5289 { 5290 /* priv init */ 5291 priv->mdev = mdev; 5292 priv->netdev = netdev; 5293 priv->profile = profile; 5294 priv->ppriv = ppriv; 5295 priv->msglevel = MLX5E_MSG_LEVEL; 5296 priv->max_nch = netdev->num_rx_queues / max_t(u8, profile->rq_groups, 1); 5297 priv->max_opened_tc = 1; 5298 5299 if (!alloc_cpumask_var(&priv->scratchpad.cpumask, GFP_KERNEL)) 5300 return -ENOMEM; 5301 5302 mutex_init(&priv->state_lock); 5303 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); 5304 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); 5305 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work); 5306 INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work); 5307 5308 priv->wq = create_singlethread_workqueue("mlx5e"); 5309 if (!priv->wq) 5310 goto err_free_cpumask; 5311 5312 /* netdev init */ 5313 netif_carrier_off(netdev); 5314 5315 return 0; 5316 5317 err_free_cpumask: 5318 free_cpumask_var(priv->scratchpad.cpumask); 5319 5320 return -ENOMEM; 5321 } 5322 5323 void mlx5e_netdev_cleanup(struct net_device *netdev, struct mlx5e_priv *priv) 5324 { 5325 destroy_workqueue(priv->wq); 5326 free_cpumask_var(priv->scratchpad.cpumask); 5327 } 5328 5329 struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev, 5330 const struct mlx5e_profile *profile, 5331 int nch, 5332 void *ppriv) 5333 { 5334 struct net_device *netdev; 5335 int err; 5336 5337 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), 5338 nch * profile->max_tc, 5339 nch * profile->rq_groups); 5340 if (!netdev) { 5341 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n"); 5342 return NULL; 5343 } 5344 5345 err = profile->init(mdev, netdev, profile, ppriv); 5346 if (err) { 5347 mlx5_core_err(mdev, "failed to init mlx5e profile %d\n", err); 5348 goto err_free_netdev; 5349 } 5350 5351 return netdev; 5352 5353 err_free_netdev: 5354 free_netdev(netdev); 5355 5356 return NULL; 5357 } 5358 5359 int mlx5e_attach_netdev(struct mlx5e_priv *priv) 5360 { 5361 const bool take_rtnl = priv->netdev->reg_state == NETREG_REGISTERED; 5362 const struct mlx5e_profile *profile; 5363 int max_nch; 5364 int err; 5365 5366 profile = priv->profile; 5367 clear_bit(MLX5E_STATE_DESTROYING, &priv->state); 5368 5369 /* max number of channels may have changed */ 5370 max_nch = mlx5e_get_max_num_channels(priv->mdev); 5371 if (priv->channels.params.num_channels > max_nch) { 5372 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch); 5373 /* Reducing the number of channels - RXFH has to be reset, and 5374 * mlx5e_num_channels_changed below will build the RQT. 5375 */ 5376 priv->netdev->priv_flags &= ~IFF_RXFH_CONFIGURED; 5377 priv->channels.params.num_channels = max_nch; 5378 } 5379 /* 1. Set the real number of queues in the kernel the first time. 5380 * 2. Set our default XPS cpumask. 5381 * 3. Build the RQT. 5382 * 5383 * rtnl_lock is required by netif_set_real_num_*_queues in case the 5384 * netdev has been registered by this point (if this function was called 5385 * in the reload or resume flow). 5386 */ 5387 if (take_rtnl) 5388 rtnl_lock(); 5389 err = mlx5e_num_channels_changed(priv); 5390 if (take_rtnl) 5391 rtnl_unlock(); 5392 if (err) 5393 goto out; 5394 5395 err = profile->init_tx(priv); 5396 if (err) 5397 goto out; 5398 5399 err = profile->init_rx(priv); 5400 if (err) 5401 goto err_cleanup_tx; 5402 5403 if (profile->enable) 5404 profile->enable(priv); 5405 5406 return 0; 5407 5408 err_cleanup_tx: 5409 profile->cleanup_tx(priv); 5410 5411 out: 5412 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 5413 cancel_work_sync(&priv->update_stats_work); 5414 return err; 5415 } 5416 5417 void mlx5e_detach_netdev(struct mlx5e_priv *priv) 5418 { 5419 const struct mlx5e_profile *profile = priv->profile; 5420 5421 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 5422 5423 if (profile->disable) 5424 profile->disable(priv); 5425 flush_workqueue(priv->wq); 5426 5427 profile->cleanup_rx(priv); 5428 profile->cleanup_tx(priv); 5429 cancel_work_sync(&priv->update_stats_work); 5430 } 5431 5432 void mlx5e_destroy_netdev(struct mlx5e_priv *priv) 5433 { 5434 const struct mlx5e_profile *profile = priv->profile; 5435 struct net_device *netdev = priv->netdev; 5436 5437 if (profile->cleanup) 5438 profile->cleanup(priv); 5439 free_netdev(netdev); 5440 } 5441 5442 /* mlx5e_attach and mlx5e_detach scope should be only creating/destroying 5443 * hardware contexts and to connect it to the current netdev. 5444 */ 5445 static int mlx5e_attach(struct mlx5_core_dev *mdev, void *vpriv) 5446 { 5447 struct mlx5e_priv *priv = vpriv; 5448 struct net_device *netdev = priv->netdev; 5449 int err; 5450 5451 if (netif_device_present(netdev)) 5452 return 0; 5453 5454 err = mlx5e_create_mdev_resources(mdev); 5455 if (err) 5456 return err; 5457 5458 err = mlx5e_attach_netdev(priv); 5459 if (err) { 5460 mlx5e_destroy_mdev_resources(mdev); 5461 return err; 5462 } 5463 5464 return 0; 5465 } 5466 5467 static void mlx5e_detach(struct mlx5_core_dev *mdev, void *vpriv) 5468 { 5469 struct mlx5e_priv *priv = vpriv; 5470 struct net_device *netdev = priv->netdev; 5471 5472 #ifdef CONFIG_MLX5_ESWITCH 5473 if (MLX5_ESWITCH_MANAGER(mdev) && vpriv == mdev) 5474 return; 5475 #endif 5476 5477 if (!netif_device_present(netdev)) 5478 return; 5479 5480 mlx5e_detach_netdev(priv); 5481 mlx5e_destroy_mdev_resources(mdev); 5482 } 5483 5484 static void *mlx5e_add(struct mlx5_core_dev *mdev) 5485 { 5486 struct net_device *netdev; 5487 void *priv; 5488 int err; 5489 int nch; 5490 5491 err = mlx5e_check_required_hca_cap(mdev); 5492 if (err) 5493 return NULL; 5494 5495 #ifdef CONFIG_MLX5_ESWITCH 5496 if (MLX5_ESWITCH_MANAGER(mdev) && 5497 mlx5_eswitch_mode(mdev->priv.eswitch) == MLX5_ESWITCH_OFFLOADS) { 5498 mlx5e_rep_register_vport_reps(mdev); 5499 return mdev; 5500 } 5501 #endif 5502 5503 nch = mlx5e_get_max_num_channels(mdev); 5504 netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, nch, NULL); 5505 if (!netdev) { 5506 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n"); 5507 return NULL; 5508 } 5509 5510 dev_net_set(netdev, mlx5_core_net(mdev)); 5511 priv = netdev_priv(netdev); 5512 5513 err = mlx5e_attach(mdev, priv); 5514 if (err) { 5515 mlx5_core_err(mdev, "mlx5e_attach failed, %d\n", err); 5516 goto err_destroy_netdev; 5517 } 5518 5519 err = mlx5e_devlink_port_register(priv); 5520 if (err) { 5521 mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err); 5522 goto err_detach; 5523 } 5524 5525 err = register_netdev(netdev); 5526 if (err) { 5527 mlx5_core_err(mdev, "register_netdev failed, %d\n", err); 5528 goto err_devlink_port_unregister; 5529 } 5530 5531 mlx5e_devlink_port_type_eth_set(priv); 5532 5533 mlx5e_dcbnl_init_app(priv); 5534 return priv; 5535 5536 err_devlink_port_unregister: 5537 mlx5e_devlink_port_unregister(priv); 5538 err_detach: 5539 mlx5e_detach(mdev, priv); 5540 err_destroy_netdev: 5541 mlx5e_destroy_netdev(priv); 5542 return NULL; 5543 } 5544 5545 static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv) 5546 { 5547 struct mlx5e_priv *priv; 5548 5549 #ifdef CONFIG_MLX5_ESWITCH 5550 if (MLX5_ESWITCH_MANAGER(mdev) && vpriv == mdev) { 5551 mlx5e_rep_unregister_vport_reps(mdev); 5552 return; 5553 } 5554 #endif 5555 priv = vpriv; 5556 mlx5e_dcbnl_delete_app(priv); 5557 unregister_netdev(priv->netdev); 5558 mlx5e_devlink_port_unregister(priv); 5559 mlx5e_detach(mdev, vpriv); 5560 mlx5e_destroy_netdev(priv); 5561 } 5562 5563 static struct mlx5_interface mlx5e_interface = { 5564 .add = mlx5e_add, 5565 .remove = mlx5e_remove, 5566 .attach = mlx5e_attach, 5567 .detach = mlx5e_detach, 5568 .protocol = MLX5_INTERFACE_PROTOCOL_ETH, 5569 }; 5570 5571 void mlx5e_init(void) 5572 { 5573 mlx5e_ipsec_build_inverse_table(); 5574 mlx5e_build_ptys2ethtool_map(); 5575 mlx5_register_interface(&mlx5e_interface); 5576 } 5577 5578 void mlx5e_cleanup(void) 5579 { 5580 mlx5_unregister_interface(&mlx5e_interface); 5581 } 5582