1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2021, The University of Queensland 14 * Copyright (c) 2018, Joyent, Inc. 15 * Copyright 2023 RackTop Systems, Inc. 16 * Copyright 2023 MNX Cloud, Inc. 17 */ 18 19 /* 20 * Mellanox Connect-X 4/5/6 driver. 21 * 22 * More details in mlxcx.c 23 */ 24 25 #ifndef _MLXCX_H 26 #define _MLXCX_H 27 28 /* 29 * mlxcx(4D) defintions 30 */ 31 32 #include <sys/ddi.h> 33 #include <sys/sunddi.h> 34 #include <sys/ddifm.h> 35 #include <sys/id_space.h> 36 #include <sys/list.h> 37 #include <sys/taskq_impl.h> 38 #include <sys/stddef.h> 39 #include <sys/stream.h> 40 #include <sys/strsun.h> 41 #include <sys/mac_provider.h> 42 #include <sys/mac_ether.h> 43 #include <sys/cpuvar.h> 44 #include <sys/ethernet.h> 45 46 #include <inet/ip.h> 47 #include <inet/ip6.h> 48 49 #include <sys/ddifm.h> 50 #include <sys/fm/protocol.h> 51 #include <sys/fm/util.h> 52 #include <sys/fm/io/ddi.h> 53 54 #include <mlxcx_reg.h> 55 56 #ifdef __cplusplus 57 extern "C" { 58 #endif 59 60 #define MLXCX_VENDOR_ID 0x15b3 61 62 /* 63 * The PCI device ids for the cards we support. The device IDs correspond to 64 * the device ids in the driver manifest, and the names were obtained from 65 * the PCI id database in /usr/share/hwdata/pci.ids 66 */ 67 #define MLXCX_CX4_DEVID 0x1013 68 #define MLXCX_CX4_VF_DEVID 0x1014 69 #define MLXCX_CX4_LX_DEVID 0x1015 70 #define MLXCX_CX4_LX_VF_DEVID 0x1016 71 #define MLXCX_CX5_DEVID 0x1017 72 #define MLXCX_CX5_VF_DEVID 0x1018 73 #define MLXCX_CX5_EX_DEVID 0x1019 74 #define MLXCX_CX5_EX_VF_DEVID 0x101a 75 #define MLXCX_CX6_DEVID 0x101b 76 #define MLXCX_CX6_VF_DEVID 0x101c 77 #define MLXCX_CX6_DF_DEVID 0x101d 78 #define MLXCX_CX5_GEN_VF_DEVID 0x101e 79 #define MLXCX_CX6_LX_DEVID 0x101f 80 81 /* 82 * Get access to the first PCI BAR. 83 */ 84 #define MLXCX_REG_NUMBER 1 85 86 /* 87 * The command queue is supposed to be a page, which is 4k. 88 */ 89 #define MLXCX_CMD_DMA_PAGE_SIZE 4096 90 91 /* 92 * Queues can allocate in units of this much memory. 93 */ 94 #define MLXCX_QUEUE_DMA_PAGE_SIZE 4096 95 96 /* 97 * We advertise two sizes of groups to MAC -- a certain number of "large" 98 * groups (including the default group, which is sized to at least ncpus) 99 * followed by a certain number of "small" groups. 100 * 101 * This allows us to have a larger amount of classification resources available 102 * for zones/VMs without resorting to software classification. 103 */ 104 #define MLXCX_RX_NGROUPS_LARGE_DFLT 2 105 #define MLXCX_RX_NRINGS_PER_LARGE_GROUP_DFLT 16 106 #define MLXCX_RX_NGROUPS_SMALL_DFLT 256 107 #define MLXCX_RX_NRINGS_PER_SMALL_GROUP_DFLT 4 108 109 #define MLXCX_TX_NGROUPS_DFLT 1 110 #define MLXCX_TX_NRINGS_PER_GROUP_DFLT 64 111 112 /* 113 * Queues will be sized to (1 << *Q_SIZE_SHIFT) entries long. 114 */ 115 #define MLXCX_EQ_SIZE_SHIFT_DFLT 9 116 117 /* 118 * The CQ, SQ and RQ sizes can effect throughput on higher speed interfaces. 119 * EQ less so, as it only takes a single EQ entry to indicate there are 120 * multiple completions on the CQ. 121 * 122 * Particularly on the Rx side, the RQ (and corresponding CQ) would run 123 * low on available entries. A symptom of this is the refill taskq running 124 * frequently. A larger RQ (and CQ) alleviates this, and as there is a 125 * close relationship between SQ and CQ size, the SQ is increased too. 126 */ 127 #define MLXCX_CQ_SIZE_SHIFT_DFLT 10 128 #define MLXCX_CQ_SIZE_SHIFT_25G 12 129 130 /* 131 * Default to making SQs bigger than RQs for 9k MTU, since most packets will 132 * spill over into more than one slot. RQ WQEs are always 1 slot. 133 */ 134 #define MLXCX_SQ_SIZE_SHIFT_DFLT 11 135 #define MLXCX_SQ_SIZE_SHIFT_25G 13 136 137 #define MLXCX_RQ_SIZE_SHIFT_DFLT 10 138 #define MLXCX_RQ_SIZE_SHIFT_25G 12 139 140 #define MLXCX_CQ_HWM_GAP 16 141 #define MLXCX_CQ_LWM_GAP 24 142 143 #define MLXCX_WQ_HWM_GAP MLXCX_CQ_HWM_GAP 144 #define MLXCX_WQ_LWM_GAP MLXCX_CQ_LWM_GAP 145 146 #define MLXCX_RQ_REFILL_STEP 64 147 148 /* 149 * CQ event moderation 150 */ 151 #define MLXCX_CQEMOD_PERIOD_USEC_DFLT 50 152 #define MLXCX_CQEMOD_COUNT_DFLT \ 153 (8 * ((1 << MLXCX_CQ_SIZE_SHIFT_DFLT) / 10)) 154 155 /* 156 * EQ interrupt moderation 157 */ 158 #define MLXCX_INTRMOD_PERIOD_USEC_DFLT 10 159 160 /* Size of root flow tables */ 161 #define MLXCX_FTBL_ROOT_SIZE_SHIFT_DFLT 12 162 163 /* Size of 2nd level flow tables for VLAN filtering */ 164 #define MLXCX_FTBL_VLAN_SIZE_SHIFT_DFLT 4 165 166 /* 167 * How big does an mblk have to be before we dma_bind() it instead of 168 * bcopying? 169 */ 170 #define MLXCX_TX_BIND_THRESHOLD_DFLT 2048 171 172 /* 173 * How often to check the status of completion queues for overflow and 174 * other problems. 175 */ 176 #define MLXCX_WQ_CHECK_INTERVAL_SEC_DFLT 300 177 #define MLXCX_CQ_CHECK_INTERVAL_SEC_DFLT 300 178 #define MLXCX_EQ_CHECK_INTERVAL_SEC_DFLT 30 179 180 /* 181 * After this many packets, the packets received so far are passed to 182 * the mac layer. 183 */ 184 #define MLXCX_RX_PER_CQ_DEFAULT 256 185 #define MLXCX_RX_PER_CQ_MIN 16 186 #define MLXCX_RX_PER_CQ_MAX 4096 187 188 #define MLXCX_DOORBELL_TRIES_DFLT 3 189 extern uint_t mlxcx_doorbell_tries; 190 191 #define MLXCX_STUCK_INTR_COUNT_DFLT 128 192 extern uint_t mlxcx_stuck_intr_count; 193 194 #define MLXCX_BUF_BIND_MAX_ATTEMTPS 50 195 196 #define MLXCX_MTU_OFFSET \ 197 (sizeof (struct ether_vlan_header) + ETHERFCSL) 198 199 /* 200 * This is the current version of the command structure that the driver expects 201 * to be found in the ISS. 202 */ 203 #define MLXCX_CMD_REVISION 5 204 205 #ifdef DEBUG 206 #define MLXCX_DMA_SYNC(dma, flag) VERIFY0(ddi_dma_sync( \ 207 (dma).mxdb_dma_handle, 0, 0, \ 208 (flag))) 209 #else 210 #define MLXCX_DMA_SYNC(dma, flag) (void) ddi_dma_sync( \ 211 (dma).mxdb_dma_handle, 0, 0, \ 212 (flag)) 213 #endif 214 215 #define MLXCX_FM_SERVICE_MLXCX "mlxcx" 216 217 /* 218 * This macro defines the expected value of the 'Interface Step Sequence ID' 219 * (issi) which represents the version of the start up and tear down sequence. 220 * We must check that hardware supports this and tell it which version we're 221 * using as well. 222 */ 223 #define MLXCX_CURRENT_ISSI 1 224 225 /* 226 * This is the size of a page that the hardware expects from us when 227 * manipulating pages. 228 */ 229 #define MLXCX_HW_PAGE_SIZE 4096 230 231 /* 232 * This is a special lkey value used to terminate a list of scatter pointers. 233 */ 234 #define MLXCX_NULL_LKEY 0x100 235 236 /* 237 * The max function id we support in manage pages requests. 238 * At the moment we only support/expect func 0 from manage pages, but 239 * structures and code are in place to support any number. 240 */ 241 #define MLXCX_FUNC_ID_MAX 0 242 243 /* 244 * Forwards 245 */ 246 struct mlxcx; 247 typedef struct mlxcx mlxcx_t; 248 typedef struct mlxcx_cmd mlxcx_cmd_t; 249 typedef struct mlxcx_port mlxcx_port_t; 250 251 typedef struct { 252 mlxcx_t *mlp_mlx; 253 int32_t mlp_npages; 254 uint16_t mlp_func; 255 } mlxcx_pages_request_t; 256 257 typedef struct mlxcx_async_param { 258 mlxcx_t *mla_mlx; 259 taskq_ent_t mla_tqe; 260 boolean_t mla_pending; 261 kmutex_t mla_mtx; 262 263 /* 264 * Parameters specific to the function dispatched. 265 */ 266 union { 267 void *mla_arg; 268 mlxcx_pages_request_t mla_pages; 269 mlxcx_port_t *mla_port; 270 }; 271 } mlxcx_async_param_t; 272 273 typedef enum { 274 MLXCX_DMABUF_HDL_ALLOC = 1 << 0, 275 MLXCX_DMABUF_MEM_ALLOC = 1 << 1, 276 MLXCX_DMABUF_BOUND = 1 << 2, 277 MLXCX_DMABUF_FOREIGN = 1 << 3, 278 } mlxcx_dma_buffer_flags_t; 279 280 typedef struct mlxcx_dma_buffer { 281 mlxcx_dma_buffer_flags_t mxdb_flags; 282 caddr_t mxdb_va; /* Buffer VA */ 283 size_t mxdb_len; /* Buffer logical len */ 284 ddi_acc_handle_t mxdb_acc_handle; 285 ddi_dma_handle_t mxdb_dma_handle; 286 uint_t mxdb_ncookies; 287 } mlxcx_dma_buffer_t; 288 289 typedef struct mlxcx_dev_page { 290 list_node_t mxdp_list; 291 avl_node_t mxdp_tree; 292 uintptr_t mxdp_pa; 293 mlxcx_dma_buffer_t mxdp_dma; 294 } mlxcx_dev_page_t; 295 296 /* 297 * Data structure to keep track of all information related to the command queue. 298 */ 299 typedef enum { 300 MLXCX_CMD_QUEUE_S_IDLE = 1, 301 MLXCX_CMD_QUEUE_S_BUSY, 302 MLXCX_CMD_QUEUE_S_BROKEN 303 } mlxcx_cmd_queue_status_t; 304 305 typedef struct mlxcx_cmd_queue { 306 kmutex_t mcmd_lock; 307 kcondvar_t mcmd_cv; 308 mlxcx_dma_buffer_t mcmd_dma; 309 310 boolean_t mcmd_polled; 311 312 uint8_t mcmd_size_l2; 313 uint8_t mcmd_stride_l2; 314 uint_t mcmd_size; 315 /* 316 * The mask has a bit for each command slot, there are a maximum 317 * of 32 slots. When the bit is set in the mask, it indicates 318 * the slot is available. 319 */ 320 uint32_t mcmd_mask; 321 322 mlxcx_cmd_t *mcmd_active[MLXCX_CMD_MAX]; 323 324 ddi_taskq_t *mcmd_taskq; 325 id_space_t *mcmd_tokens; 326 } mlxcx_cmd_queue_t; 327 328 typedef struct mlxcd_cmd_mbox { 329 list_node_t mlbox_node; 330 mlxcx_dma_buffer_t mlbox_dma; 331 mlxcx_cmd_mailbox_t *mlbox_data; 332 } mlxcx_cmd_mbox_t; 333 334 typedef enum { 335 MLXCX_EQ_ALLOC = 1 << 0, /* dma mem alloc'd, size set */ 336 MLXCX_EQ_CREATED = 1 << 1, /* CREATE_EQ sent to hw */ 337 MLXCX_EQ_DESTROYED = 1 << 2, /* DESTROY_EQ sent to hw */ 338 MLXCX_EQ_ARMED = 1 << 3, /* Armed through the UAR */ 339 MLXCX_EQ_POLLING = 1 << 4, /* Currently being polled */ 340 MLXCX_EQ_INTR_ENABLED = 1 << 5, /* ddi_intr_enable()'d */ 341 MLXCX_EQ_INTR_ACTIVE = 1 << 6, /* 'rupt handler running */ 342 MLXCX_EQ_INTR_QUIESCE = 1 << 7, /* 'rupt handler to quiesce */ 343 MLXCX_EQ_ATTACHING = 1 << 8, /* mlxcx_attach still running */ 344 } mlxcx_eventq_state_t; 345 346 typedef struct mlxcx_bf { 347 kmutex_t mbf_mtx; 348 uint_t mbf_cnt; 349 uint_t mbf_even; 350 uint_t mbf_odd; 351 } mlxcx_bf_t; 352 353 typedef struct mlxcx_uar { 354 boolean_t mlu_allocated; 355 uint_t mlu_num; 356 uint_t mlu_base; 357 358 volatile uint_t mlu_bfcnt; 359 mlxcx_bf_t mlu_bf[MLXCX_BF_PER_UAR]; 360 } mlxcx_uar_t; 361 362 typedef struct mlxcx_pd { 363 boolean_t mlpd_allocated; 364 uint32_t mlpd_num; 365 } mlxcx_pd_t; 366 367 typedef struct mlxcx_tdom { 368 boolean_t mltd_allocated; 369 uint32_t mltd_num; 370 } mlxcx_tdom_t; 371 372 typedef enum { 373 MLXCX_PORT_VPORT_PROMISC = 1 << 0, 374 } mlxcx_port_flags_t; 375 376 typedef struct mlxcx_flow_table mlxcx_flow_table_t; 377 typedef struct mlxcx_flow_group mlxcx_flow_group_t; 378 379 typedef struct { 380 uint64_t mlps_rx_drops; 381 } mlxcx_port_stats_t; 382 383 typedef enum { 384 MLXCX_PORT_INIT = 1 << 0 385 } mlxcx_port_init_t; 386 387 struct mlxcx_port { 388 kmutex_t mlp_mtx; 389 mlxcx_port_init_t mlp_init; 390 mlxcx_t *mlp_mlx; 391 /* 392 * The mlp_num we have here starts at zero (it's an index), but the 393 * numbering we have to use for register access starts at 1. We 394 * currently write mlp_num into the other_vport fields in mlxcx_cmd.c 395 * (where 0 is a magic number meaning "my vport") so if we ever add 396 * support for virtualisation features and deal with more than one 397 * vport, we will probably have to change this. 398 */ 399 uint_t mlp_num; 400 mlxcx_port_flags_t mlp_flags; 401 uint64_t mlp_guid; 402 uint8_t mlp_mac_address[ETHERADDRL]; 403 404 uint_t mlp_mtu; 405 uint_t mlp_max_mtu; 406 407 mlxcx_port_status_t mlp_admin_status; 408 mlxcx_port_status_t mlp_oper_status; 409 410 boolean_t mlp_autoneg; 411 mlxcx_eth_proto_t mlp_max_proto; 412 mlxcx_eth_proto_t mlp_admin_proto; 413 mlxcx_eth_proto_t mlp_oper_proto; 414 mlxcx_ext_eth_proto_t mlp_ext_max_proto; 415 mlxcx_ext_eth_proto_t mlp_ext_admin_proto; 416 mlxcx_ext_eth_proto_t mlp_ext_oper_proto; 417 mlxcx_pplm_fec_active_t mlp_fec_active; 418 link_fec_t mlp_fec_requested; 419 420 mlxcx_eth_inline_mode_t mlp_wqe_min_inline; 421 422 /* Root flow tables */ 423 mlxcx_flow_table_t *mlp_rx_flow; 424 mlxcx_flow_table_t *mlp_tx_flow; 425 426 mlxcx_flow_group_t *mlp_promisc; 427 mlxcx_flow_group_t *mlp_bcast; 428 mlxcx_flow_group_t *mlp_umcast; 429 430 avl_tree_t mlp_dmac_fe; 431 432 mlxcx_port_stats_t mlp_stats; 433 434 mlxcx_module_status_t mlp_last_modstate; 435 mlxcx_module_error_type_t mlp_last_moderr; 436 437 mlxcx_async_param_t mlx_port_event; 438 }; 439 440 typedef enum { 441 MLXCX_EQ_TYPE_ANY, 442 MLXCX_EQ_TYPE_RX, 443 MLXCX_EQ_TYPE_TX 444 } mlxcx_eventq_type_t; 445 446 /* 447 * mlxcx_event_queue_t is a representation of an event queue (EQ). 448 * There is a 1-1 tie in between an EQ and an interrupt vector, and 449 * knowledge of that effects how some members of the struct are used 450 * and modified. 451 * 452 * Most of the struct members are immmutable except for during set up and 453 * teardown, for those it is safe to access them without a mutex once 454 * the driver is initialized. 455 * 456 * Members which are not immutable and are protected by mleq_mtx are: 457 * * mleq_state - EQ state. Changes during transitions between 458 * polling modes. 459 * * mleq_cq - an AVL tree of completions queues using this EQ. 460 * 461 * Another member which is not immutable is mleq_cc. This is the EQ 462 * consumer counter, it *must* only be incremented in the EQ's interrupt 463 * context. It is also fed back to the hardware during re-arming of 464 * the EQ, again this *must* only happen in the EQ's interrupt context. 465 * 466 * There are a couple of struct members (mleq_check_disarm_cc and 467 * mleq_check_disarm_cnt) which are used to help monitor the health 468 * and consistency of the EQ. They are only used and modified during health 469 * monitoring, which is both infrequent and single threaded, consequently 470 * no mutex guards are needed. 471 * 472 * Care is taken not to use the mleq_mtx when possible, both to avoid 473 * contention in what is "hot" code and avoid breaking requirements 474 * of mac(9E). 475 */ 476 typedef struct mlxcx_event_queue { 477 kmutex_t mleq_mtx; 478 kcondvar_t mleq_cv; 479 mlxcx_t *mleq_mlx; 480 mlxcx_eventq_state_t mleq_state; 481 mlxcx_eventq_type_t mleq_type; 482 483 mlxcx_dma_buffer_t mleq_dma; 484 485 size_t mleq_entshift; 486 size_t mleq_nents; 487 mlxcx_eventq_ent_t *mleq_ent; 488 uint32_t mleq_cc; /* consumer counter */ 489 uint32_t mleq_cc_armed; 490 491 uint32_t mleq_events; 492 493 uint32_t mleq_badintrs; 494 495 /* Hardware eq number */ 496 uint_t mleq_num; 497 /* Index into the mlxcx_t's interrupts array */ 498 uint_t mleq_intr_index; 499 500 /* UAR region that has this EQ's doorbell in it */ 501 mlxcx_uar_t *mleq_uar; 502 503 /* Tree of CQn => mlxcx_completion_queue_t */ 504 avl_tree_t mleq_cqs; 505 506 uint32_t mleq_check_disarm_cc; 507 uint_t mleq_check_disarm_cnt; 508 } mlxcx_event_queue_t; 509 510 typedef enum { 511 MLXCX_TIS_CREATED = 1 << 0, 512 MLXCX_TIS_DESTROYED = 1 << 1, 513 } mlxcx_tis_state_t; 514 515 typedef struct mlxcx_tis { 516 mlxcx_tis_state_t mltis_state; 517 list_node_t mltis_entry; 518 uint_t mltis_num; 519 mlxcx_tdom_t *mltis_tdom; 520 } mlxcx_tis_t; 521 522 typedef enum { 523 MLXCX_BUFFER_INIT, 524 MLXCX_BUFFER_FREE, 525 MLXCX_BUFFER_ON_WQ, 526 MLXCX_BUFFER_ON_LOAN, 527 MLXCX_BUFFER_ON_CHAIN, 528 } mlxcx_buffer_state_t; 529 530 typedef enum { 531 MLXCX_SHARD_READY, 532 MLXCX_SHARD_DRAINING, 533 } mlxcx_shard_state_t; 534 535 typedef struct mlxcx_buf_shard { 536 mlxcx_shard_state_t mlbs_state; 537 list_node_t mlbs_entry; 538 kmutex_t mlbs_mtx; 539 list_t mlbs_busy; 540 list_t mlbs_free; 541 list_t mlbs_loaned; 542 kcondvar_t mlbs_free_nonempty; 543 } mlxcx_buf_shard_t; 544 545 typedef struct mlxcx_buffer { 546 mlxcx_buf_shard_t *mlb_shard; 547 list_node_t mlb_entry; 548 list_node_t mlb_cq_entry; 549 550 struct mlxcx_buffer *mlb_tx_head; /* head of tx chain */ 551 list_t mlb_tx_chain; 552 list_node_t mlb_tx_chain_entry; 553 554 boolean_t mlb_foreign; 555 size_t mlb_used; 556 mblk_t *mlb_tx_mp; 557 558 /* 559 * The number of work queue basic blocks this buf uses. 560 */ 561 uint_t mlb_wqebbs; 562 563 mlxcx_t *mlb_mlx; 564 mlxcx_buffer_state_t mlb_state; 565 uint_t mlb_wqe_index; 566 mlxcx_dma_buffer_t mlb_dma; 567 mblk_t *mlb_mp; 568 frtn_t mlb_frtn; 569 } mlxcx_buffer_t; 570 571 typedef enum { 572 MLXCX_CQ_ALLOC = 1 << 0, 573 MLXCX_CQ_CREATED = 1 << 1, 574 MLXCX_CQ_DESTROYED = 1 << 2, 575 MLXCX_CQ_EQAVL = 1 << 3, 576 MLXCX_CQ_BLOCKED_MAC = 1 << 4, 577 MLXCX_CQ_TEARDOWN = 1 << 5, 578 MLXCX_CQ_POLLING = 1 << 6, 579 MLXCX_CQ_ARMED = 1 << 7, 580 } mlxcx_completionq_state_t; 581 582 typedef struct mlxcx_work_queue mlxcx_work_queue_t; 583 584 typedef struct mlxcx_completion_queue { 585 kmutex_t mlcq_mtx; 586 kmutex_t mlcq_arm_mtx; 587 mlxcx_t *mlcq_mlx; 588 mlxcx_completionq_state_t mlcq_state; 589 590 mlxcx_port_stats_t *mlcq_stats; 591 592 list_node_t mlcq_entry; 593 avl_node_t mlcq_eq_entry; 594 595 uint_t mlcq_num; 596 597 mlxcx_work_queue_t *mlcq_wq; 598 mlxcx_event_queue_t *mlcq_eq; 599 600 /* UAR region that has this CQ's UAR doorbell in it */ 601 mlxcx_uar_t *mlcq_uar; 602 603 mlxcx_dma_buffer_t mlcq_dma; 604 605 size_t mlcq_entshift; 606 size_t mlcq_nents; 607 mlxcx_completionq_ent_t *mlcq_ent; 608 uint32_t mlcq_cc; /* consumer counter */ 609 uint32_t mlcq_cc_armed; /* cc at last arm */ 610 uint32_t mlcq_ec; /* event counter */ 611 uint32_t mlcq_ec_armed; /* ec at last arm */ 612 613 mlxcx_dma_buffer_t mlcq_doorbell_dma; 614 mlxcx_completionq_doorbell_t *mlcq_doorbell; 615 616 uint64_t mlcq_bufcnt; 617 size_t mlcq_bufhwm; 618 size_t mlcq_buflwm; 619 list_t mlcq_buffers; 620 kmutex_t mlcq_bufbmtx; 621 list_t mlcq_buffers_b; 622 623 uint_t mlcq_check_disarm_cnt; 624 uint64_t mlcq_check_disarm_cc; 625 626 uint_t mlcq_cqemod_period_usec; 627 uint_t mlcq_cqemod_count; 628 629 mac_ring_handle_t mlcq_mac_hdl; 630 uint64_t mlcq_mac_gen; 631 632 boolean_t mlcq_fm_repd_qstate; 633 } mlxcx_completion_queue_t; 634 635 typedef enum { 636 MLXCX_WQ_ALLOC = 1 << 0, 637 MLXCX_WQ_CREATED = 1 << 1, 638 MLXCX_WQ_STARTED = 1 << 2, 639 MLXCX_WQ_DESTROYED = 1 << 3, 640 MLXCX_WQ_TEARDOWN = 1 << 4, 641 MLXCX_WQ_BUFFERS = 1 << 5, 642 MLXCX_WQ_REFILLING = 1 << 6, 643 MLXCX_WQ_BLOCKED_MAC = 1 << 7 644 } mlxcx_workq_state_t; 645 646 typedef enum { 647 MLXCX_WQ_TYPE_SENDQ = 1, 648 MLXCX_WQ_TYPE_RECVQ 649 } mlxcx_workq_type_t; 650 651 typedef struct mlxcx_ring_group mlxcx_ring_group_t; 652 653 struct mlxcx_work_queue { 654 kmutex_t mlwq_mtx; 655 mlxcx_t *mlwq_mlx; 656 mlxcx_workq_type_t mlwq_type; 657 mlxcx_workq_state_t mlwq_state; 658 659 list_node_t mlwq_entry; 660 list_node_t mlwq_group_entry; 661 662 mlxcx_ring_group_t *mlwq_group; 663 664 uint_t mlwq_num; 665 666 mlxcx_completion_queue_t *mlwq_cq; 667 mlxcx_pd_t *mlwq_pd; 668 669 /* Required for send queues */ 670 mlxcx_tis_t *mlwq_tis; 671 672 /* UAR region that has this WQ's blueflame buffers in it */ 673 mlxcx_uar_t *mlwq_uar; 674 675 mlxcx_dma_buffer_t mlwq_dma; 676 677 mlxcx_eth_inline_mode_t mlwq_inline_mode; 678 size_t mlwq_entshift; 679 size_t mlwq_nents; 680 /* Discriminate based on mwq_type */ 681 union { 682 mlxcx_sendq_ent_t *mlwq_send_ent; 683 mlxcx_sendq_extra_ent_t *mlwq_send_extra_ent; 684 mlxcx_recvq_ent_t *mlwq_recv_ent; 685 mlxcx_sendq_bf_t *mlwq_bf_ent; 686 }; 687 uint64_t mlwq_pc; /* producer counter */ 688 689 uint64_t mlwq_wqebb_used; 690 size_t mlwq_bufhwm; 691 size_t mlwq_buflwm; 692 693 mlxcx_dma_buffer_t mlwq_doorbell_dma; 694 mlxcx_workq_doorbell_t *mlwq_doorbell; 695 696 mlxcx_buf_shard_t *mlwq_bufs; 697 mlxcx_buf_shard_t *mlwq_foreign_bufs; 698 699 taskq_ent_t mlwq_tqe; 700 701 boolean_t mlwq_fm_repd_qstate; 702 }; 703 704 #define MLXCX_RQT_MAX_SIZE 64 705 706 typedef enum { 707 MLXCX_RQT_CREATED = 1 << 0, 708 MLXCX_RQT_DESTROYED = 1 << 1, 709 MLXCX_RQT_DIRTY = 1 << 2, 710 } mlxcx_rqtable_state_t; 711 712 typedef struct mlxcx_rqtable { 713 mlxcx_rqtable_state_t mlrqt_state; 714 list_node_t mlrqt_entry; 715 uint_t mlrqt_num; 716 717 size_t mlrqt_max; 718 size_t mlrqt_used; 719 720 size_t mlrqt_rq_size; 721 mlxcx_work_queue_t **mlrqt_rq; 722 } mlxcx_rqtable_t; 723 724 typedef enum { 725 MLXCX_TIR_CREATED = 1 << 0, 726 MLXCX_TIR_DESTROYED = 1 << 1, 727 } mlxcx_tir_state_t; 728 729 typedef struct mlxcx_tir { 730 mlxcx_tir_state_t mltir_state; 731 list_node_t mltir_entry; 732 uint_t mltir_num; 733 mlxcx_tdom_t *mltir_tdom; 734 mlxcx_tir_type_t mltir_type; 735 union { 736 mlxcx_rqtable_t *mltir_rqtable; 737 mlxcx_work_queue_t *mltir_rq; 738 }; 739 mlxcx_tir_hash_fn_t mltir_hash_fn; 740 uint8_t mltir_toeplitz_key[40]; 741 mlxcx_tir_rx_hash_l3_type_t mltir_l3_type; 742 mlxcx_tir_rx_hash_l4_type_t mltir_l4_type; 743 mlxcx_tir_rx_hash_fields_t mltir_hash_fields; 744 } mlxcx_tir_t; 745 746 typedef enum { 747 MLXCX_FLOW_GROUP_CREATED = 1 << 0, 748 MLXCX_FLOW_GROUP_BUSY = 1 << 1, 749 MLXCX_FLOW_GROUP_DESTROYED = 1 << 2, 750 } mlxcx_flow_group_state_t; 751 752 typedef enum { 753 MLXCX_FLOW_MATCH_SMAC = 1 << 0, 754 MLXCX_FLOW_MATCH_DMAC = 1 << 1, 755 MLXCX_FLOW_MATCH_VLAN = 1 << 2, 756 MLXCX_FLOW_MATCH_VID = 1 << 3, 757 MLXCX_FLOW_MATCH_IP_VER = 1 << 4, 758 MLXCX_FLOW_MATCH_SRCIP = 1 << 5, 759 MLXCX_FLOW_MATCH_DSTIP = 1 << 6, 760 MLXCX_FLOW_MATCH_IP_PROTO = 1 << 7, 761 MLXCX_FLOW_MATCH_SQN = 1 << 8, 762 MLXCX_FLOW_MATCH_VXLAN = 1 << 9, 763 } mlxcx_flow_mask_t; 764 765 struct mlxcx_flow_group { 766 list_node_t mlfg_entry; 767 list_node_t mlfg_role_entry; 768 mlxcx_flow_group_state_t mlfg_state; 769 mlxcx_flow_table_t *mlfg_table; 770 uint_t mlfg_num; 771 size_t mlfg_start_idx; 772 size_t mlfg_size; 773 size_t mlfg_avail; 774 list_t mlfg_entries; 775 mlxcx_flow_mask_t mlfg_mask; 776 }; 777 778 typedef enum { 779 MLXCX_FLOW_ENTRY_RESERVED = 1 << 0, 780 MLXCX_FLOW_ENTRY_CREATED = 1 << 1, 781 MLXCX_FLOW_ENTRY_DELETED = 1 << 2, 782 MLXCX_FLOW_ENTRY_DIRTY = 1 << 3, 783 } mlxcx_flow_entry_state_t; 784 785 typedef struct { 786 mlxcx_tir_t *mlfed_tir; 787 mlxcx_flow_table_t *mlfed_flow; 788 } mlxcx_flow_entry_dest_t; 789 790 typedef struct mlxcx_flow_entry { 791 list_node_t mlfe_group_entry; 792 avl_node_t mlfe_dmac_entry; 793 mlxcx_flow_entry_state_t mlfe_state; 794 mlxcx_flow_table_t *mlfe_table; 795 mlxcx_flow_group_t *mlfe_group; 796 uint_t mlfe_index; 797 798 mlxcx_flow_action_t mlfe_action; 799 800 /* Criteria for match */ 801 uint8_t mlfe_smac[ETHERADDRL]; 802 uint8_t mlfe_dmac[ETHERADDRL]; 803 804 mlxcx_vlan_type_t mlfe_vlan_type; 805 uint16_t mlfe_vid; 806 807 uint_t mlfe_ip_version; 808 uint8_t mlfe_srcip[IPV6_ADDR_LEN]; 809 uint8_t mlfe_dstip[IPV6_ADDR_LEN]; 810 811 uint_t mlfe_ip_proto; 812 uint16_t mlfe_sport; 813 uint16_t mlfe_dport; 814 815 uint32_t mlfe_sqn; 816 uint32_t mlfe_vxlan_vni; 817 818 /* Destinations */ 819 size_t mlfe_ndest; 820 mlxcx_flow_entry_dest_t mlfe_dest[MLXCX_FLOW_MAX_DESTINATIONS]; 821 822 /* 823 * mlxcx_group_mac_ts joining this entry to N ring groups 824 * only used by FEs on the root rx flow table 825 */ 826 list_t mlfe_ring_groups; 827 } mlxcx_flow_entry_t; 828 829 typedef enum { 830 MLXCX_FLOW_TABLE_CREATED = 1 << 0, 831 MLXCX_FLOW_TABLE_DESTROYED = 1 << 1, 832 MLXCX_FLOW_TABLE_ROOT = 1 << 2 833 } mlxcx_flow_table_state_t; 834 835 struct mlxcx_flow_table { 836 kmutex_t mlft_mtx; 837 mlxcx_flow_table_state_t mlft_state; 838 uint_t mlft_level; 839 uint_t mlft_num; 840 mlxcx_flow_table_type_t mlft_type; 841 842 mlxcx_port_t *mlft_port; 843 844 size_t mlft_entshift; 845 size_t mlft_nents; 846 847 size_t mlft_entsize; 848 mlxcx_flow_entry_t *mlft_ent; 849 850 /* First entry not yet claimed by a group */ 851 size_t mlft_next_ent; 852 853 list_t mlft_groups; 854 }; 855 856 typedef enum { 857 MLXCX_GROUP_RX, 858 MLXCX_GROUP_TX 859 } mlxcx_group_type_t; 860 861 typedef enum { 862 MLXCX_GROUP_INIT = 1 << 0, 863 MLXCX_GROUP_WQS = 1 << 1, 864 MLXCX_GROUP_TIRTIS = 1 << 2, 865 MLXCX_GROUP_FLOWS = 1 << 3, 866 MLXCX_GROUP_RUNNING = 1 << 4, 867 MLXCX_GROUP_RQT = 1 << 5, 868 } mlxcx_group_state_t; 869 870 #define MLXCX_RX_HASH_FT_SIZE_SHIFT 4 871 872 typedef enum { 873 MLXCX_TIR_ROLE_IPv4 = 0, 874 MLXCX_TIR_ROLE_IPv6, 875 MLXCX_TIR_ROLE_TCPv4, 876 MLXCX_TIR_ROLE_TCPv6, 877 MLXCX_TIR_ROLE_UDPv4, 878 MLXCX_TIR_ROLE_UDPv6, 879 MLXCX_TIR_ROLE_OTHER, 880 881 MLXCX_TIRS_PER_GROUP 882 } mlxcx_tir_role_t; 883 884 typedef struct { 885 avl_node_t mlgm_group_entry; 886 list_node_t mlgm_fe_entry; 887 mlxcx_ring_group_t *mlgm_group; 888 uint8_t mlgm_mac[6]; 889 mlxcx_flow_entry_t *mlgm_fe; 890 } mlxcx_group_mac_t; 891 892 typedef struct { 893 list_node_t mlgv_entry; 894 boolean_t mlgv_tagged; 895 uint16_t mlgv_vid; 896 mlxcx_flow_entry_t *mlgv_fe; 897 } mlxcx_group_vlan_t; 898 899 struct mlxcx_ring_group { 900 kmutex_t mlg_mtx; 901 mlxcx_t *mlg_mlx; 902 mlxcx_group_state_t mlg_state; 903 mlxcx_group_type_t mlg_type; 904 905 mac_group_handle_t mlg_mac_hdl; 906 907 union { 908 mlxcx_tis_t mlg_tis; 909 mlxcx_tir_t mlg_tir[MLXCX_TIRS_PER_GROUP]; 910 }; 911 mlxcx_port_t *mlg_port; 912 913 size_t mlg_nwqs; 914 size_t mlg_wqs_size; 915 mlxcx_work_queue_t *mlg_wqs; 916 917 mlxcx_rqtable_t *mlg_rqt; 918 919 /* 920 * Flow table for matching VLAN IDs 921 */ 922 mlxcx_flow_table_t *mlg_rx_vlan_ft; 923 mlxcx_flow_group_t *mlg_rx_vlan_fg; 924 mlxcx_flow_group_t *mlg_rx_vlan_def_fg; 925 mlxcx_flow_group_t *mlg_rx_vlan_promisc_fg; 926 list_t mlg_rx_vlans; 927 928 taskq_t *mlg_refill_tq; 929 930 /* 931 * Flow table for separating out by protocol before hashing 932 */ 933 mlxcx_flow_table_t *mlg_rx_hash_ft; 934 935 /* 936 * Links to flow entries on the root flow table which are pointing to 937 * our rx_vlan_ft. 938 */ 939 avl_tree_t mlg_rx_macs; 940 }; 941 942 typedef enum mlxcx_cmd_state { 943 MLXCX_CMD_S_DONE = 1 << 0, 944 MLXCX_CMD_S_ERROR = 1 << 1 945 } mlxcx_cmd_state_t; 946 947 struct mlxcx_cmd { 948 struct mlxcx *mlcmd_mlxp; 949 kmutex_t mlcmd_lock; 950 kcondvar_t mlcmd_cv; 951 952 boolean_t mlcmd_poll; 953 uint8_t mlcmd_token; 954 mlxcx_cmd_op_t mlcmd_op; 955 956 /* 957 * Command data and extended mailboxes for responses. 958 */ 959 const void *mlcmd_in; 960 uint32_t mlcmd_inlen; 961 void *mlcmd_out; 962 uint32_t mlcmd_outlen; 963 list_t mlcmd_mbox_in; 964 uint8_t mlcmd_nboxes_in; 965 list_t mlcmd_mbox_out; 966 uint8_t mlcmd_nboxes_out; 967 /* 968 * Status information. 969 */ 970 mlxcx_cmd_state_t mlcmd_state; 971 uint8_t mlcmd_status; 972 }; 973 974 /* 975 * Our view of capabilities. 976 */ 977 typedef struct mlxcx_hca_cap { 978 mlxcx_hca_cap_mode_t mhc_mode; 979 mlxcx_hca_cap_type_t mhc_type; 980 union { 981 uint8_t mhc_bulk[MLXCX_HCA_CAP_SIZE]; 982 mlxcx_hca_cap_general_caps_t mhc_general; 983 mlxcx_hca_cap_eth_caps_t mhc_eth; 984 mlxcx_hca_cap_flow_caps_t mhc_flow; 985 }; 986 } mlxcx_hca_cap_t; 987 988 typedef struct { 989 /* Cooked values */ 990 boolean_t mlc_checksum; 991 boolean_t mlc_lso; 992 boolean_t mlc_vxlan; 993 boolean_t mlc_pcam; 994 boolean_t mlc_ext_ptys; 995 size_t mlc_max_lso_size; 996 size_t mlc_max_rqt_size; 997 998 size_t mlc_max_rx_ft_shift; 999 size_t mlc_max_rx_fe_dest; 1000 size_t mlc_max_rx_flows; 1001 size_t mlc_max_rx_ft; 1002 1003 size_t mlc_max_tir; 1004 1005 /* Raw caps data */ 1006 mlxcx_hca_cap_t mlc_hca_cur; 1007 mlxcx_hca_cap_t mlc_hca_max; 1008 mlxcx_hca_cap_t mlc_ether_cur; 1009 mlxcx_hca_cap_t mlc_ether_max; 1010 mlxcx_hca_cap_t mlc_nic_flow_cur; 1011 mlxcx_hca_cap_t mlc_nic_flow_max; 1012 } mlxcx_caps_t; 1013 1014 typedef struct { 1015 uint_t mldp_eq_size_shift; 1016 uint_t mldp_cq_size_shift; 1017 uint_t mldp_cq_size_shift_default; 1018 uint_t mldp_rq_size_shift; 1019 uint_t mldp_rq_size_shift_default; 1020 uint_t mldp_sq_size_shift; 1021 uint_t mldp_sq_size_shift_default; 1022 uint_t mldp_cqemod_period_usec; 1023 uint_t mldp_cqemod_count; 1024 uint_t mldp_intrmod_period_usec; 1025 uint_t mldp_rx_ngroups_large; 1026 uint_t mldp_rx_ngroups_small; 1027 uint_t mldp_rx_nrings_per_large_group; 1028 uint_t mldp_rx_nrings_per_small_group; 1029 uint_t mldp_rx_per_cq; 1030 uint_t mldp_tx_ngroups; 1031 uint_t mldp_tx_nrings_per_group; 1032 uint_t mldp_ftbl_root_size_shift; 1033 size_t mldp_tx_bind_threshold; 1034 uint_t mldp_ftbl_vlan_size_shift; 1035 uint64_t mldp_eq_check_interval_sec; 1036 uint64_t mldp_cq_check_interval_sec; 1037 uint64_t mldp_wq_check_interval_sec; 1038 } mlxcx_drv_props_t; 1039 1040 typedef struct { 1041 mlxcx_t *mlts_mlx; 1042 uint8_t mlts_index; 1043 id_t mlts_ksensor; 1044 int16_t mlts_value; 1045 int16_t mlts_max_value; 1046 uint8_t mlts_name[MLXCX_MTMP_NAMELEN]; 1047 } mlxcx_temp_sensor_t; 1048 1049 /* 1050 * The oldest card supported by this driver is ConnectX-4. So far (at least), 1051 * newer models tend to just add features vs. replacing them, so it seems 1052 * reasonable to assume an unknown model likely supports everything the 1053 * ConnectX-6 cards do. 1054 */ 1055 typedef enum { 1056 MLXCX_DEV_CX4 = 0, 1057 MLXCX_DEV_CX5 = 1, 1058 MLXCX_DEV_CX6 = 2, 1059 MLXCX_DEV_UNKNOWN = 3, 1060 } mlxcx_dev_type_t; 1061 1062 typedef enum { 1063 MLXCX_ATTACH_FM = 1 << 0, 1064 MLXCX_ATTACH_PCI_CONFIG = 1 << 1, 1065 MLXCX_ATTACH_REGS = 1 << 2, 1066 MLXCX_ATTACH_CMD = 1 << 3, 1067 MLXCX_ATTACH_ENABLE_HCA = 1 << 4, 1068 MLXCX_ATTACH_PAGE_LIST = 1 << 5, 1069 MLXCX_ATTACH_INIT_HCA = 1 << 6, 1070 MLXCX_ATTACH_UAR_PD_TD = 1 << 7, 1071 MLXCX_ATTACH_INTRS = 1 << 8, 1072 MLXCX_ATTACH_PORTS = 1 << 9, 1073 MLXCX_ATTACH_MAC_HDL = 1 << 10, 1074 MLXCX_ATTACH_CQS = 1 << 11, 1075 MLXCX_ATTACH_WQS = 1 << 12, 1076 MLXCX_ATTACH_GROUPS = 1 << 13, 1077 MLXCX_ATTACH_BUFS = 1 << 14, 1078 MLXCX_ATTACH_CAPS = 1 << 15, 1079 MLXCX_ATTACH_CHKTIMERS = 1 << 16, 1080 MLXCX_ATTACH_ASYNC_TQ = 1 << 17, 1081 MLXCX_ATTACH_SENSORS = 1 << 18 1082 } mlxcx_attach_progress_t; 1083 1084 struct mlxcx { 1085 /* entry on the mlxcx_glist */ 1086 list_node_t mlx_gentry; 1087 1088 dev_info_t *mlx_dip; 1089 int mlx_inst; 1090 mlxcx_attach_progress_t mlx_attach; 1091 1092 mlxcx_dev_type_t mlx_type; 1093 mlxcx_drv_props_t mlx_props; 1094 1095 /* 1096 * Misc. data 1097 */ 1098 uint16_t mlx_fw_maj; 1099 uint16_t mlx_fw_min; 1100 uint16_t mlx_fw_rev; 1101 uint16_t mlx_cmd_rev; 1102 1103 /* 1104 * Various capabilities of hardware. 1105 */ 1106 mlxcx_caps_t *mlx_caps; 1107 1108 uint_t mlx_max_sdu; 1109 uint_t mlx_sdu; 1110 1111 /* 1112 * FM State 1113 */ 1114 int mlx_fm_caps; 1115 1116 /* 1117 * PCI Data 1118 */ 1119 ddi_acc_handle_t mlx_cfg_handle; 1120 ddi_acc_handle_t mlx_regs_handle; 1121 caddr_t mlx_regs_base; 1122 1123 /* 1124 * MAC handle 1125 */ 1126 mac_handle_t mlx_mac_hdl; 1127 1128 /* 1129 * Main command queue for issuing general FW control commands. 1130 */ 1131 mlxcx_cmd_queue_t mlx_cmd; 1132 1133 /* 1134 * Interrupts 1135 */ 1136 uint_t mlx_intr_pri; 1137 uint_t mlx_async_intr_pri; 1138 uint_t mlx_intr_type; /* always MSI-X */ 1139 int mlx_intr_count; 1140 size_t mlx_intr_size; /* allocation size */ 1141 int mlx_intr_cq0; 1142 ddi_intr_handle_t *mlx_intr_handles; 1143 1144 /* 1145 * Basic firmware resources which we use for a variety of things. 1146 * The UAR is a reference to a page where CQ and EQ doorbells are 1147 * located. It also holds all the BlueFlame stuff (which we don't 1148 * use). 1149 */ 1150 mlxcx_uar_t mlx_uar; 1151 /* 1152 * The PD (Protection Domain) and TDOM (Transport Domain) are opaque 1153 * entities to us (they're Infiniband constructs we don't actually care 1154 * about) -- we just allocate them and shove their ID numbers in 1155 * whenever we're asked for one. 1156 * 1157 * The "reserved" LKEY is what we should put in queue entries that 1158 * have references to memory to indicate that they're using linear 1159 * addresses (comes from the QUERY_SPECIAL_CONTEXTS cmd). 1160 */ 1161 mlxcx_pd_t mlx_pd; 1162 mlxcx_tdom_t mlx_tdom; 1163 uint_t mlx_rsvd_lkey; 1164 1165 /* 1166 * Our event queues. These are 1:1 with interrupts. 1167 */ 1168 size_t mlx_eqs_size; /* allocation size */ 1169 mlxcx_event_queue_t *mlx_eqs; 1170 1171 /* 1172 * Page list. These represent the set of 4k pages we've given to 1173 * hardware. 1174 * 1175 * We can add to this list at the request of hardware from interrupt 1176 * context (the PAGE_REQUEST event), so it's protected by pagemtx. 1177 */ 1178 kmutex_t mlx_pagemtx; 1179 uint_t mlx_npages; 1180 avl_tree_t mlx_pages; 1181 1182 mlxcx_async_param_t mlx_npages_req[MLXCX_FUNC_ID_MAX + 1]; 1183 1184 /* 1185 * Taskq for processing asynchronous events which may issue 1186 * commands to the HCA. 1187 */ 1188 taskq_t *mlx_async_tq; 1189 1190 /* 1191 * Port state 1192 */ 1193 uint_t mlx_nports; 1194 size_t mlx_ports_size; 1195 mlxcx_port_t *mlx_ports; 1196 1197 /* 1198 * Completion queues (CQs). These are also indexed off the 1199 * event_queue_ts that they each report to. 1200 */ 1201 list_t mlx_cqs; 1202 1203 uint_t mlx_next_eq; 1204 1205 /* 1206 * Work queues (WQs). 1207 */ 1208 list_t mlx_wqs; 1209 1210 /* 1211 * Ring groups 1212 */ 1213 size_t mlx_rx_ngroups; 1214 size_t mlx_rx_groups_size; 1215 mlxcx_ring_group_t *mlx_rx_groups; 1216 1217 size_t mlx_tx_ngroups; 1218 size_t mlx_tx_groups_size; 1219 mlxcx_ring_group_t *mlx_tx_groups; 1220 1221 kmem_cache_t *mlx_bufs_cache; 1222 list_t mlx_buf_shards; 1223 1224 ddi_periodic_t mlx_eq_checktimer; 1225 ddi_periodic_t mlx_cq_checktimer; 1226 ddi_periodic_t mlx_wq_checktimer; 1227 1228 /* 1229 * Sensors 1230 */ 1231 uint8_t mlx_temp_nsensors; 1232 mlxcx_temp_sensor_t *mlx_temp_sensors; 1233 }; 1234 1235 /* 1236 * Register access 1237 */ 1238 extern uint16_t mlxcx_get16(mlxcx_t *, uintptr_t); 1239 extern uint32_t mlxcx_get32(mlxcx_t *, uintptr_t); 1240 extern uint64_t mlxcx_get64(mlxcx_t *, uintptr_t); 1241 1242 extern void mlxcx_put32(mlxcx_t *, uintptr_t, uint32_t); 1243 extern void mlxcx_put64(mlxcx_t *, uintptr_t, uint64_t); 1244 1245 extern void mlxcx_uar_put32(mlxcx_t *, mlxcx_uar_t *, uintptr_t, uint32_t); 1246 extern void mlxcx_uar_put64(mlxcx_t *, mlxcx_uar_t *, uintptr_t, uint64_t); 1247 1248 /* 1249 * Logging functions. 1250 */ 1251 extern void mlxcx_warn(mlxcx_t *, const char *, ...); 1252 extern void mlxcx_note(mlxcx_t *, const char *, ...); 1253 extern void mlxcx_panic(mlxcx_t *, const char *, ...); 1254 1255 extern void mlxcx_fm_ereport(mlxcx_t *, const char *); 1256 1257 extern void mlxcx_check_sq(mlxcx_t *, mlxcx_work_queue_t *); 1258 extern void mlxcx_check_rq(mlxcx_t *, mlxcx_work_queue_t *); 1259 1260 /* 1261 * DMA Functions 1262 */ 1263 extern void mlxcx_dma_free(mlxcx_dma_buffer_t *); 1264 extern boolean_t mlxcx_dma_alloc(mlxcx_t *, mlxcx_dma_buffer_t *, 1265 ddi_dma_attr_t *, ddi_device_acc_attr_t *, boolean_t, size_t, boolean_t); 1266 extern boolean_t mlxcx_dma_init(mlxcx_t *, mlxcx_dma_buffer_t *, 1267 ddi_dma_attr_t *, boolean_t); 1268 extern boolean_t mlxcx_dma_bind_mblk(mlxcx_t *, mlxcx_dma_buffer_t *, 1269 const mblk_t *, size_t, boolean_t); 1270 extern boolean_t mlxcx_dma_alloc_offset(mlxcx_t *, mlxcx_dma_buffer_t *, 1271 ddi_dma_attr_t *, ddi_device_acc_attr_t *, boolean_t, 1272 size_t, size_t, boolean_t); 1273 extern void mlxcx_dma_unbind(mlxcx_t *, mlxcx_dma_buffer_t *); 1274 extern void mlxcx_dma_acc_attr(mlxcx_t *, ddi_device_acc_attr_t *); 1275 extern void mlxcx_dma_page_attr(mlxcx_t *, ddi_dma_attr_t *); 1276 extern void mlxcx_dma_queue_attr(mlxcx_t *, ddi_dma_attr_t *); 1277 extern void mlxcx_dma_qdbell_attr(mlxcx_t *, ddi_dma_attr_t *); 1278 extern void mlxcx_dma_buf_attr(mlxcx_t *, ddi_dma_attr_t *); 1279 1280 extern boolean_t mlxcx_give_pages(mlxcx_t *, int32_t, int32_t *); 1281 1282 static inline const ddi_dma_cookie_t * 1283 mlxcx_dma_cookie_iter(const mlxcx_dma_buffer_t *db, 1284 const ddi_dma_cookie_t *prev) 1285 { 1286 ASSERT(db->mxdb_flags & MLXCX_DMABUF_BOUND); 1287 return (ddi_dma_cookie_iter(db->mxdb_dma_handle, prev)); 1288 } 1289 1290 static inline const ddi_dma_cookie_t * 1291 mlxcx_dma_cookie_one(const mlxcx_dma_buffer_t *db) 1292 { 1293 ASSERT(db->mxdb_flags & MLXCX_DMABUF_BOUND); 1294 return (ddi_dma_cookie_one(db->mxdb_dma_handle)); 1295 } 1296 1297 /* 1298 * From mlxcx_intr.c 1299 */ 1300 extern boolean_t mlxcx_intr_setup(mlxcx_t *); 1301 extern void mlxcx_intr_disable(mlxcx_t *); 1302 extern void mlxcx_intr_teardown(mlxcx_t *); 1303 extern void mlxcx_arm_eq(mlxcx_t *, mlxcx_event_queue_t *); 1304 extern void mlxcx_arm_cq(mlxcx_t *, mlxcx_completion_queue_t *); 1305 extern void mlxcx_update_cqci(mlxcx_t *, mlxcx_completion_queue_t *); 1306 1307 extern mblk_t *mlxcx_rx_poll(mlxcx_t *, mlxcx_completion_queue_t *, size_t); 1308 1309 /* 1310 * From mlxcx_gld.c 1311 */ 1312 extern boolean_t mlxcx_register_mac(mlxcx_t *); 1313 1314 /* 1315 * From mlxcx_ring.c 1316 */ 1317 extern boolean_t mlxcx_wq_alloc_dma(mlxcx_t *, mlxcx_work_queue_t *); 1318 extern void mlxcx_wq_rele_dma(mlxcx_t *, mlxcx_work_queue_t *); 1319 1320 extern boolean_t mlxcx_buf_create(mlxcx_t *, mlxcx_buf_shard_t *, 1321 mlxcx_buffer_t **); 1322 extern boolean_t mlxcx_buf_create_foreign(mlxcx_t *, mlxcx_buf_shard_t *, 1323 mlxcx_buffer_t **); 1324 extern mlxcx_buffer_t *mlxcx_buf_take(mlxcx_t *, mlxcx_work_queue_t *); 1325 extern size_t mlxcx_buf_take_n(mlxcx_t *, mlxcx_work_queue_t *, 1326 mlxcx_buffer_t **, size_t); 1327 extern boolean_t mlxcx_buf_loan(mlxcx_t *, mlxcx_buffer_t *); 1328 extern void mlxcx_buf_return(mlxcx_t *, mlxcx_buffer_t *); 1329 extern void mlxcx_buf_return_chain(mlxcx_t *, mlxcx_buffer_t *, boolean_t); 1330 extern void mlxcx_buf_destroy(mlxcx_t *, mlxcx_buffer_t *); 1331 extern void mlxcx_shard_ready(mlxcx_buf_shard_t *); 1332 extern void mlxcx_shard_draining(mlxcx_buf_shard_t *); 1333 1334 extern uint_t mlxcx_buf_bind_or_copy(mlxcx_t *, mlxcx_work_queue_t *, 1335 mblk_t *, size_t, mlxcx_buffer_t **); 1336 1337 extern boolean_t mlxcx_rx_group_setup(mlxcx_t *, mlxcx_ring_group_t *); 1338 extern boolean_t mlxcx_tx_group_setup(mlxcx_t *, mlxcx_ring_group_t *); 1339 1340 extern boolean_t mlxcx_rx_group_start(mlxcx_t *, mlxcx_ring_group_t *); 1341 extern boolean_t mlxcx_tx_ring_start(mlxcx_t *, mlxcx_ring_group_t *, 1342 mlxcx_work_queue_t *); 1343 extern boolean_t mlxcx_rx_ring_start(mlxcx_t *, mlxcx_ring_group_t *, 1344 mlxcx_work_queue_t *); 1345 1346 extern boolean_t mlxcx_rq_add_buffer(mlxcx_t *, mlxcx_work_queue_t *, 1347 mlxcx_buffer_t *); 1348 extern boolean_t mlxcx_rq_add_buffers(mlxcx_t *, mlxcx_work_queue_t *, 1349 mlxcx_buffer_t **, size_t); 1350 extern boolean_t mlxcx_sq_add_buffer(mlxcx_t *, mlxcx_work_queue_t *, 1351 uint8_t *, size_t, uint32_t, mlxcx_buffer_t *); 1352 extern boolean_t mlxcx_sq_add_nop(mlxcx_t *, mlxcx_work_queue_t *); 1353 extern void mlxcx_rq_refill(mlxcx_t *, mlxcx_work_queue_t *); 1354 1355 extern void mlxcx_teardown_groups(mlxcx_t *); 1356 extern void mlxcx_wq_teardown(mlxcx_t *, mlxcx_work_queue_t *); 1357 extern void mlxcx_cq_teardown(mlxcx_t *, mlxcx_completion_queue_t *); 1358 extern void mlxcx_teardown_rx_group(mlxcx_t *, mlxcx_ring_group_t *); 1359 extern void mlxcx_teardown_tx_group(mlxcx_t *, mlxcx_ring_group_t *); 1360 1361 extern void mlxcx_tx_completion(mlxcx_t *, mlxcx_completion_queue_t *, 1362 mlxcx_completionq_ent_t *, mlxcx_buffer_t *); 1363 extern mblk_t *mlxcx_rx_completion(mlxcx_t *, mlxcx_completion_queue_t *, 1364 mlxcx_completionq_ent_t *, mlxcx_buffer_t *); 1365 1366 extern mlxcx_buf_shard_t *mlxcx_mlbs_create(mlxcx_t *); 1367 1368 /* 1369 * Flow mgmt 1370 */ 1371 extern boolean_t mlxcx_add_umcast_entry(mlxcx_t *, mlxcx_port_t *, 1372 mlxcx_ring_group_t *, const uint8_t *); 1373 extern boolean_t mlxcx_remove_umcast_entry(mlxcx_t *, mlxcx_port_t *, 1374 mlxcx_ring_group_t *, const uint8_t *); 1375 extern void mlxcx_remove_all_umcast_entries(mlxcx_t *, mlxcx_port_t *, 1376 mlxcx_ring_group_t *); 1377 extern boolean_t mlxcx_setup_flow_group(mlxcx_t *, mlxcx_flow_table_t *, 1378 mlxcx_flow_group_t *); 1379 extern void mlxcx_teardown_flow_table(mlxcx_t *, mlxcx_flow_table_t *); 1380 1381 extern void mlxcx_remove_all_vlan_entries(mlxcx_t *, mlxcx_ring_group_t *); 1382 extern boolean_t mlxcx_remove_vlan_entry(mlxcx_t *, mlxcx_ring_group_t *, 1383 boolean_t, uint16_t); 1384 extern boolean_t mlxcx_add_vlan_entry(mlxcx_t *, mlxcx_ring_group_t *, 1385 boolean_t, uint16_t); 1386 1387 /* 1388 * Command functions 1389 */ 1390 extern boolean_t mlxcx_cmd_queue_init(mlxcx_t *); 1391 extern void mlxcx_cmd_queue_fini(mlxcx_t *); 1392 1393 extern void mlxcx_cmd_completion(mlxcx_t *, mlxcx_eventq_ent_t *); 1394 extern void mlxcx_cmd_eq_enable(mlxcx_t *); 1395 extern void mlxcx_cmd_eq_disable(mlxcx_t *); 1396 1397 extern boolean_t mlxcx_cmd_enable_hca(mlxcx_t *); 1398 extern boolean_t mlxcx_cmd_disable_hca(mlxcx_t *); 1399 1400 extern boolean_t mlxcx_cmd_query_issi(mlxcx_t *, uint_t *); 1401 extern boolean_t mlxcx_cmd_set_issi(mlxcx_t *, uint16_t); 1402 1403 extern boolean_t mlxcx_cmd_query_pages(mlxcx_t *, uint_t, int32_t *); 1404 extern boolean_t mlxcx_cmd_give_pages(mlxcx_t *, uint_t, int32_t, 1405 mlxcx_dev_page_t **); 1406 extern boolean_t mlxcx_cmd_return_pages(mlxcx_t *, int32_t, uint64_t *, 1407 int32_t *); 1408 1409 extern boolean_t mlxcx_cmd_query_hca_cap(mlxcx_t *, mlxcx_hca_cap_type_t, 1410 mlxcx_hca_cap_mode_t, mlxcx_hca_cap_t *); 1411 1412 extern boolean_t mlxcx_cmd_set_driver_version(mlxcx_t *, const char *); 1413 1414 extern boolean_t mlxcx_cmd_init_hca(mlxcx_t *); 1415 extern boolean_t mlxcx_cmd_teardown_hca(mlxcx_t *); 1416 1417 extern boolean_t mlxcx_cmd_alloc_uar(mlxcx_t *, mlxcx_uar_t *); 1418 extern boolean_t mlxcx_cmd_dealloc_uar(mlxcx_t *, mlxcx_uar_t *); 1419 1420 extern boolean_t mlxcx_cmd_alloc_pd(mlxcx_t *, mlxcx_pd_t *); 1421 extern boolean_t mlxcx_cmd_dealloc_pd(mlxcx_t *, mlxcx_pd_t *); 1422 1423 extern boolean_t mlxcx_cmd_alloc_tdom(mlxcx_t *, mlxcx_tdom_t *); 1424 extern boolean_t mlxcx_cmd_dealloc_tdom(mlxcx_t *, mlxcx_tdom_t *); 1425 1426 extern boolean_t mlxcx_cmd_create_eq(mlxcx_t *, mlxcx_event_queue_t *); 1427 extern boolean_t mlxcx_cmd_destroy_eq(mlxcx_t *, mlxcx_event_queue_t *); 1428 extern boolean_t mlxcx_cmd_query_eq(mlxcx_t *, mlxcx_event_queue_t *, 1429 mlxcx_eventq_ctx_t *); 1430 1431 extern boolean_t mlxcx_cmd_create_cq(mlxcx_t *, mlxcx_completion_queue_t *); 1432 extern boolean_t mlxcx_cmd_destroy_cq(mlxcx_t *, mlxcx_completion_queue_t *); 1433 extern boolean_t mlxcx_cmd_query_cq(mlxcx_t *, mlxcx_completion_queue_t *, 1434 mlxcx_completionq_ctx_t *); 1435 1436 extern boolean_t mlxcx_cmd_create_rq(mlxcx_t *, mlxcx_work_queue_t *); 1437 extern boolean_t mlxcx_cmd_start_rq(mlxcx_t *, mlxcx_work_queue_t *); 1438 extern boolean_t mlxcx_cmd_stop_rq(mlxcx_t *, mlxcx_work_queue_t *); 1439 extern boolean_t mlxcx_cmd_destroy_rq(mlxcx_t *, mlxcx_work_queue_t *); 1440 extern boolean_t mlxcx_cmd_query_rq(mlxcx_t *, mlxcx_work_queue_t *, 1441 mlxcx_rq_ctx_t *); 1442 1443 extern boolean_t mlxcx_cmd_create_tir(mlxcx_t *, mlxcx_tir_t *); 1444 extern boolean_t mlxcx_cmd_destroy_tir(mlxcx_t *, mlxcx_tir_t *); 1445 1446 extern boolean_t mlxcx_cmd_create_sq(mlxcx_t *, mlxcx_work_queue_t *); 1447 extern boolean_t mlxcx_cmd_start_sq(mlxcx_t *, mlxcx_work_queue_t *); 1448 extern boolean_t mlxcx_cmd_stop_sq(mlxcx_t *, mlxcx_work_queue_t *); 1449 extern boolean_t mlxcx_cmd_destroy_sq(mlxcx_t *, mlxcx_work_queue_t *); 1450 extern boolean_t mlxcx_cmd_query_sq(mlxcx_t *, mlxcx_work_queue_t *, 1451 mlxcx_sq_ctx_t *); 1452 1453 extern boolean_t mlxcx_cmd_create_tis(mlxcx_t *, mlxcx_tis_t *); 1454 extern boolean_t mlxcx_cmd_destroy_tis(mlxcx_t *, mlxcx_tis_t *); 1455 1456 extern boolean_t mlxcx_cmd_query_nic_vport_ctx(mlxcx_t *, mlxcx_port_t *); 1457 extern boolean_t mlxcx_cmd_query_special_ctxs(mlxcx_t *); 1458 1459 extern boolean_t mlxcx_cmd_modify_nic_vport_ctx(mlxcx_t *, mlxcx_port_t *, 1460 mlxcx_modify_nic_vport_ctx_fields_t); 1461 1462 extern boolean_t mlxcx_cmd_create_flow_table(mlxcx_t *, mlxcx_flow_table_t *); 1463 extern boolean_t mlxcx_cmd_destroy_flow_table(mlxcx_t *, mlxcx_flow_table_t *); 1464 extern boolean_t mlxcx_cmd_set_flow_table_root(mlxcx_t *, mlxcx_flow_table_t *); 1465 1466 extern boolean_t mlxcx_cmd_create_flow_group(mlxcx_t *, mlxcx_flow_group_t *); 1467 extern boolean_t mlxcx_cmd_set_flow_table_entry(mlxcx_t *, 1468 mlxcx_flow_entry_t *); 1469 extern boolean_t mlxcx_cmd_delete_flow_table_entry(mlxcx_t *, 1470 mlxcx_flow_entry_t *); 1471 extern boolean_t mlxcx_cmd_destroy_flow_group(mlxcx_t *, mlxcx_flow_group_t *); 1472 1473 extern boolean_t mlxcx_cmd_access_register(mlxcx_t *, mlxcx_cmd_reg_opmod_t, 1474 mlxcx_register_id_t, mlxcx_register_data_t *); 1475 extern boolean_t mlxcx_cmd_query_port_mtu(mlxcx_t *, mlxcx_port_t *); 1476 extern boolean_t mlxcx_cmd_query_port_status(mlxcx_t *, mlxcx_port_t *); 1477 extern boolean_t mlxcx_cmd_modify_port_status(mlxcx_t *, mlxcx_port_t *, 1478 mlxcx_port_status_t); 1479 extern boolean_t mlxcx_cmd_query_port_speed(mlxcx_t *, mlxcx_port_t *); 1480 extern boolean_t mlxcx_cmd_query_port_fec(mlxcx_t *, mlxcx_port_t *); 1481 extern boolean_t mlxcx_cmd_modify_port_fec(mlxcx_t *, mlxcx_port_t *, 1482 mlxcx_pplm_fec_caps_t); 1483 1484 extern boolean_t mlxcx_cmd_set_port_mtu(mlxcx_t *, mlxcx_port_t *); 1485 1486 extern boolean_t mlxcx_cmd_create_rqt(mlxcx_t *, mlxcx_rqtable_t *); 1487 extern boolean_t mlxcx_cmd_destroy_rqt(mlxcx_t *, mlxcx_rqtable_t *); 1488 1489 extern boolean_t mlxcx_cmd_set_int_mod(mlxcx_t *, uint_t, uint_t); 1490 1491 extern boolean_t mlxcx_cmd_query_module_status(mlxcx_t *, uint_t, 1492 mlxcx_module_status_t *, mlxcx_module_error_type_t *); 1493 extern boolean_t mlxcx_cmd_set_port_led(mlxcx_t *, mlxcx_port_t *, uint16_t); 1494 1495 /* Comparator for avl_ts */ 1496 extern int mlxcx_cq_compare(const void *, const void *); 1497 extern int mlxcx_dmac_fe_compare(const void *, const void *); 1498 extern int mlxcx_grmac_compare(const void *, const void *); 1499 extern int mlxcx_page_compare(const void *, const void *); 1500 1501 extern void mlxcx_update_link_state(mlxcx_t *, mlxcx_port_t *); 1502 1503 extern void mlxcx_eth_proto_to_string(mlxcx_eth_proto_t, mlxcx_ext_eth_proto_t, 1504 char *, size_t); 1505 extern const char *mlxcx_port_status_string(mlxcx_port_status_t); 1506 1507 extern const char *mlxcx_event_name(mlxcx_event_t); 1508 1509 /* 1510 * Sensor Functions 1511 */ 1512 extern boolean_t mlxcx_setup_sensors(mlxcx_t *); 1513 extern void mlxcx_teardown_sensors(mlxcx_t *); 1514 1515 #ifdef __cplusplus 1516 } 1517 #endif 1518 1519 #endif /* _MLXCX_H */ 1520