1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2020, The University of Queensland 14 * Copyright (c) 2018, Joyent, Inc. 15 * Copyright 2020 RackTop Systems, Inc. 16 */ 17 18 /* 19 * Controls the management of commands that are issues to and from the HCA 20 * command queue. 21 */ 22 23 #include <mlxcx.h> 24 25 #include <sys/debug.h> 26 #include <sys/sysmacros.h> 27 28 /* 29 * When we start up the command queue, it will undergo some internal 30 * initialization after we set the command queue address. These values allow us 31 * to control how much time we should wait for that to occur. 32 */ 33 clock_t mlxcx_cmd_init_delay = 1000 * 10; /* 10 ms in us */ 34 uint_t mlxcx_cmd_init_trys = 100; /* Wait at most 1s */ 35 36 clock_t mlxcx_cmd_delay = 1000 * 1; /* 1 ms in us */ 37 uint_t mlxcx_cmd_tries = 5000; /* Wait at most 1s */ 38 39 /* 40 * This macro is used to identify that we care about our own function that we're 41 * communicating with. We always use this function. 42 */ 43 #define MLXCX_FUNCTION_SELF (to_be16(0)) 44 45 static const char * 46 mlxcx_cmd_response_string(mlxcx_cmd_ret_t ret) 47 { 48 switch (ret) { 49 case MLXCX_CMD_R_OK: 50 return ("MLXCX_CMD_R_OK"); 51 case MLXCX_CMD_R_INTERNAL_ERR: 52 return ("MLXCX_CMD_R_INTERNAL_ERR"); 53 case MLXCX_CMD_R_BAD_OP: 54 return ("MLXCX_CMD_R_BAD_OP"); 55 case MLXCX_CMD_R_BAD_PARAM: 56 return ("MLXCX_CMD_R_BAD_PARAM"); 57 case MLXCX_CMD_R_BAD_SYS_STATE: 58 return ("MLXCX_CMD_R_BAD_SYS_STATE"); 59 case MLXCX_CMD_R_BAD_RESOURCE: 60 return ("MLXCX_CMD_R_BAD_RESOURCE"); 61 case MLXCX_CMD_R_RESOURCE_BUSY: 62 return ("MLXCX_CMD_R_RESOURCE_BUSY"); 63 case MLXCX_CMD_R_EXCEED_LIM: 64 return ("MLXCX_CMD_R_EXCEED_LIM"); 65 case MLXCX_CMD_R_BAD_RES_STATE: 66 return ("MLXCX_CMD_R_BAD_RES_STATE"); 67 case MLXCX_CMD_R_BAD_INDEX: 68 return ("MLXCX_CMD_R_BAD_INDEX"); 69 case MLXCX_CMD_R_NO_RESOURCES: 70 return ("MLXCX_CMD_R_NO_RESOURCES"); 71 case MLXCX_CMD_R_BAD_INPUT_LEN: 72 return ("MLXCX_CMD_R_BAD_INPUT_LEN"); 73 case MLXCX_CMD_R_BAD_OUTPUT_LEN: 74 return ("MLXCX_CMD_R_BAD_OUTPUT_LEN"); 75 case MLXCX_CMD_R_BAD_RESOURCE_STATE: 76 return ("MLXCX_CMD_R_BAD_RESOURCE_STATE"); 77 case MLXCX_CMD_R_BAD_PKT: 78 return ("MLXCX_CMD_R_BAD_PKT"); 79 case MLXCX_CMD_R_BAD_SIZE: 80 return ("MLXCX_CMD_R_BAD_SIZE"); 81 default: 82 return ("Unknown command"); 83 } 84 } 85 86 static const char * 87 mlxcx_cmd_opcode_string(mlxcx_cmd_op_t op) 88 { 89 switch (op) { 90 case MLXCX_OP_QUERY_HCA_CAP: 91 return ("MLXCX_OP_QUERY_HCA_CAP"); 92 case MLXCX_OP_QUERY_ADAPTER: 93 return ("MLXCX_OP_QUERY_ADAPTER"); 94 case MLXCX_OP_INIT_HCA: 95 return ("MLXCX_OP_INIT_HCA"); 96 case MLXCX_OP_TEARDOWN_HCA: 97 return ("MLXCX_OP_TEARDOWN_HCA"); 98 case MLXCX_OP_ENABLE_HCA: 99 return ("MLXCX_OP_ENABLE_HCA"); 100 case MLXCX_OP_DISABLE_HCA: 101 return ("MLXCX_OP_DISABLE_HCA"); 102 case MLXCX_OP_QUERY_PAGES: 103 return ("MLXCX_OP_QUERY_PAGES"); 104 case MLXCX_OP_MANAGE_PAGES: 105 return ("MLXCX_OP_MANAGE_PAGES"); 106 case MLXCX_OP_SET_HCA_CAP: 107 return ("MLXCX_OP_SET_HCA_CAP"); 108 case MLXCX_OP_QUERY_ISSI: 109 return ("MLXCX_OP_QUERY_ISSI"); 110 case MLXCX_OP_SET_ISSI: 111 return ("MLXCX_OP_SET_ISSI"); 112 case MLXCX_OP_SET_DRIVER_VERSION: 113 return ("MLXCX_OP_SET_DRIVER_VERSION"); 114 case MLXCX_OP_QUERY_OTHER_HCA_CAP: 115 return ("MLXCX_OP_QUERY_OTHER_HCA_CAP"); 116 case MLXCX_OP_MODIFY_OTHER_HCA_CAP: 117 return ("MLXCX_OP_MODIFY_OTHER_HCA_CAP"); 118 case MLXCX_OP_SET_TUNNELED_OPERATIONS: 119 return ("MLXCX_OP_SET_TUNNELED_OPERATIONS"); 120 case MLXCX_OP_CREATE_MKEY: 121 return ("MLXCX_OP_CREATE_MKEY"); 122 case MLXCX_OP_QUERY_MKEY: 123 return ("MLXCX_OP_QUERY_MKEY"); 124 case MLXCX_OP_DESTROY_MKEY: 125 return ("MLXCX_OP_DESTROY_MKEY"); 126 case MLXCX_OP_QUERY_SPECIAL_CONTEXTS: 127 return ("MLXCX_OP_QUERY_SPECIAL_CONTEXTS"); 128 case MLXCX_OP_PAGE_FAULT_RESUME: 129 return ("MLXCX_OP_PAGE_FAULT_RESUME"); 130 case MLXCX_OP_CREATE_EQ: 131 return ("MLXCX_OP_CREATE_EQ"); 132 case MLXCX_OP_DESTROY_EQ: 133 return ("MLXCX_OP_DESTROY_EQ"); 134 case MLXCX_OP_QUERY_EQ: 135 return ("MLXCX_OP_QUERY_EQ"); 136 case MLXCX_OP_GEN_EQE: 137 return ("MLXCX_OP_GEN_EQE"); 138 case MLXCX_OP_CREATE_CQ: 139 return ("MLXCX_OP_CREATE_CQ"); 140 case MLXCX_OP_DESTROY_CQ: 141 return ("MLXCX_OP_DESTROY_CQ"); 142 case MLXCX_OP_QUERY_CQ: 143 return ("MLXCX_OP_QUERY_CQ"); 144 case MLXCX_OP_MODIFY_CQ: 145 return ("MLXCX_OP_MODIFY_CQ"); 146 case MLXCX_OP_CREATE_QP: 147 return ("MLXCX_OP_CREATE_QP"); 148 case MLXCX_OP_DESTROY_QP: 149 return ("MLXCX_OP_DESTROY_QP"); 150 case MLXCX_OP_RST2INIT_QP: 151 return ("MLXCX_OP_RST2INIT_QP"); 152 case MLXCX_OP_INIT2RTR_QP: 153 return ("MLXCX_OP_INIT2RTR_QP"); 154 case MLXCX_OP_RTR2RTS_QP: 155 return ("MLXCX_OP_RTR2RTS_QP"); 156 case MLXCX_OP_RTS2RTS_QP: 157 return ("MLXCX_OP_RTS2RTS_QP"); 158 case MLXCX_OP_SQERR2RTS_QP: 159 return ("MLXCX_OP_SQERR2RTS_QP"); 160 case MLXCX_OP__2ERR_QP: 161 return ("MLXCX_OP__2ERR_QP"); 162 case MLXCX_OP__2RST_QP: 163 return ("MLXCX_OP__2RST_QP"); 164 case MLXCX_OP_QUERY_QP: 165 return ("MLXCX_OP_QUERY_QP"); 166 case MLXCX_OP_SQD_RTS_QP: 167 return ("MLXCX_OP_SQD_RTS_QP"); 168 case MLXCX_OP_INIT2INIT_QP: 169 return ("MLXCX_OP_INIT2INIT_QP"); 170 case MLXCX_OP_CREATE_PSV: 171 return ("MLXCX_OP_CREATE_PSV"); 172 case MLXCX_OP_DESTROY_PSV: 173 return ("MLXCX_OP_DESTROY_PSV"); 174 case MLXCX_OP_CREATE_SRQ: 175 return ("MLXCX_OP_CREATE_SRQ"); 176 case MLXCX_OP_DESTROY_SRQ: 177 return ("MLXCX_OP_DESTROY_SRQ"); 178 case MLXCX_OP_QUERY_SRQ: 179 return ("MLXCX_OP_QUERY_SRQ"); 180 case MLXCX_OP_ARM_RQ: 181 return ("MLXCX_OP_ARM_RQ"); 182 case MLXCX_OP_CREATE_XRC_SRQ: 183 return ("MLXCX_OP_CREATE_XRC_SRQ"); 184 case MLXCX_OP_DESTROY_XRC_SRQ: 185 return ("MLXCX_OP_DESTROY_XRC_SRQ"); 186 case MLXCX_OP_QUERY_XRC_SRQ: 187 return ("MLXCX_OP_QUERY_XRC_SRQ"); 188 case MLXCX_OP_ARM_XRC_SRQ: 189 return ("MLXCX_OP_ARM_XRC_SRQ"); 190 case MLXCX_OP_CREATE_DCT: 191 return ("MLXCX_OP_CREATE_DCT"); 192 case MLXCX_OP_DESTROY_DCT: 193 return ("MLXCX_OP_DESTROY_DCT"); 194 case MLXCX_OP_DRAIN_DCT: 195 return ("MLXCX_OP_DRAIN_DCT"); 196 case MLXCX_OP_QUERY_DCT: 197 return ("MLXCX_OP_QUERY_DCT"); 198 case MLXCX_OP_ARM_DCT_FOR_KEY_VIOLATION: 199 return ("MLXCX_OP_ARM_DCT_FOR_KEY_VIOLATION"); 200 case MLXCX_OP_CREATE_XRQ: 201 return ("MLXCX_OP_CREATE_XRQ"); 202 case MLXCX_OP_DESTROY_XRQ: 203 return ("MLXCX_OP_DESTROY_XRQ"); 204 case MLXCX_OP_QUERY_XRQ: 205 return ("MLXCX_OP_QUERY_XRQ"); 206 case MLXCX_OP_CREATE_NVMF_BACKEND_CONTROLLER: 207 return ("MLXCX_OP_CREATE_NVMF_BACKEND_CONTROLLER"); 208 case MLXCX_OP_DESTROY_NVMF_BACKEND_CONTROLLER: 209 return ("MLXCX_OP_DESTROY_NVMF_BACKEND_CONTROLLER"); 210 case MLXCX_OP_QUERY_NVMF_BACKEND_CONTROLLER: 211 return ("MLXCX_OP_QUERY_NVMF_BACKEND_CONTROLLER"); 212 case MLXCX_OP_ATTACH_NVMF_NAMESPACE: 213 return ("MLXCX_OP_ATTACH_NVMF_NAMESPACE"); 214 case MLXCX_OP_DETACH_NVMF_NAMESPACE: 215 return ("MLXCX_OP_DETACH_NVMF_NAMESPACE"); 216 case MLXCX_OP_QUERY_XRQ_DC_PARAMS_ENTRY: 217 return ("MLXCX_OP_QUERY_XRQ_DC_PARAMS_ENTRY"); 218 case MLXCX_OP_SET_XRQ_DC_PARAMS_ENTRY: 219 return ("MLXCX_OP_SET_XRQ_DC_PARAMS_ENTRY"); 220 case MLXCX_OP_QUERY_XRQ_ERROR_PARAMS: 221 return ("MLXCX_OP_QUERY_XRQ_ERROR_PARAMS"); 222 case MLXCX_OP_QUERY_VPORT_STATE: 223 return ("MLXCX_OP_QUERY_VPORT_STATE"); 224 case MLXCX_OP_MODIFY_VPORT_STATE: 225 return ("MLXCX_OP_MODIFY_VPORT_STATE"); 226 case MLXCX_OP_QUERY_ESW_VPORT_CONTEXT: 227 return ("MLXCX_OP_QUERY_ESW_VPORT_CONTEXT"); 228 case MLXCX_OP_MODIFY_ESW_VPORT_CONTEXT: 229 return ("MLXCX_OP_MODIFY_ESW_VPORT_CONTEXT"); 230 case MLXCX_OP_QUERY_NIC_VPORT_CONTEXT: 231 return ("MLXCX_OP_QUERY_NIC_VPORT_CONTEXT"); 232 case MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT: 233 return ("MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT"); 234 case MLXCX_OP_QUERY_ROCE_ADDRESS: 235 return ("MLXCX_OP_QUERY_ROCE_ADDRESS"); 236 case MLXCX_OP_SET_ROCE_ADDRESS: 237 return ("MLXCX_OP_SET_ROCE_ADDRESS"); 238 case MLXCX_OP_QUERY_HCA_VPORT_CONTEXT: 239 return ("MLXCX_OP_QUERY_HCA_VPORT_CONTEXT"); 240 case MLXCX_OP_MODIFY_HCA_VPORT_CONTEXT: 241 return ("MLXCX_OP_MODIFY_HCA_VPORT_CONTEXT"); 242 case MLXCX_OP_QUERY_HCA_VPORT_GID: 243 return ("MLXCX_OP_QUERY_HCA_VPORT_GID"); 244 case MLXCX_OP_QUERY_HCA_VPORT_PKEY: 245 return ("MLXCX_OP_QUERY_HCA_VPORT_PKEY"); 246 case MLXCX_OP_QUERY_VPORT_COUNTER: 247 return ("MLXCX_OP_QUERY_VPORT_COUNTER"); 248 case MLXCX_OP_ALLOC_Q_COUNTER: 249 return ("MLXCX_OP_ALLOC_Q_COUNTER"); 250 case MLXCX_OP_DEALLOC_Q_COUNTER: 251 return ("MLXCX_OP_DEALLOC_Q_COUNTER"); 252 case MLXCX_OP_QUERY_Q_COUNTER: 253 return ("MLXCX_OP_QUERY_Q_COUNTER"); 254 case MLXCX_OP_SET_PP_RATE_LIMIT: 255 return ("MLXCX_OP_SET_PP_RATE_LIMIT"); 256 case MLXCX_OP_QUERY_PP_RATE_LIMIT: 257 return ("MLXCX_OP_QUERY_PP_RATE_LIMIT"); 258 case MLXCX_OP_ALLOC_PD: 259 return ("MLXCX_OP_ALLOC_PD"); 260 case MLXCX_OP_DEALLOC_PD: 261 return ("MLXCX_OP_DEALLOC_PD"); 262 case MLXCX_OP_ALLOC_UAR: 263 return ("MLXCX_OP_ALLOC_UAR"); 264 case MLXCX_OP_DEALLOC_UAR: 265 return ("MLXCX_OP_DEALLOC_UAR"); 266 case MLXCX_OP_CONFIG_INT_MODERATION: 267 return ("MLXCX_OP_CONFIG_INT_MODERATION"); 268 case MLXCX_OP_ACCESS_REG: 269 return ("MLXCX_OP_ACCESS_REG"); 270 case MLXCX_OP_ATTACH_TO_MCG: 271 return ("MLXCX_OP_ATTACH_TO_MCG"); 272 case MLXCX_OP_DETACH_FROM_MCG: 273 return ("MLXCX_OP_DETACH_FROM_MCG"); 274 case MLXCX_OP_MAD_IFC: 275 return ("MLXCX_OP_MAD_IFC"); 276 case MLXCX_OP_QUERY_MAD_DEMUX: 277 return ("MLXCX_OP_QUERY_MAD_DEMUX"); 278 case MLXCX_OP_SET_MAD_DEMUX: 279 return ("MLXCX_OP_SET_MAD_DEMUX"); 280 case MLXCX_OP_NOP: 281 return ("MLXCX_OP_NOP"); 282 case MLXCX_OP_ALLOC_XRCD: 283 return ("MLXCX_OP_ALLOC_XRCD"); 284 case MLXCX_OP_DEALLOC_XRCD: 285 return ("MLXCX_OP_DEALLOC_XRCD"); 286 case MLXCX_OP_ALLOC_TRANSPORT_DOMAIN: 287 return ("MLXCX_OP_ALLOC_TRANSPORT_DOMAIN"); 288 case MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN: 289 return ("MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN"); 290 case MLXCX_OP_QUERY_CONG_STATUS: 291 return ("MLXCX_OP_QUERY_CONG_STATUS"); 292 case MLXCX_OP_MODIFY_CONG_STATUS: 293 return ("MLXCX_OP_MODIFY_CONG_STATUS"); 294 case MLXCX_OP_QUERY_CONG_PARAMS: 295 return ("MLXCX_OP_QUERY_CONG_PARAMS"); 296 case MLXCX_OP_MODIFY_CONG_PARAMS: 297 return ("MLXCX_OP_MODIFY_CONG_PARAMS"); 298 case MLXCX_OP_QUERY_CONG_STATISTICS: 299 return ("MLXCX_OP_QUERY_CONG_STATISTICS"); 300 case MLXCX_OP_ADD_VXLAN_UDP_DPORT: 301 return ("MLXCX_OP_ADD_VXLAN_UDP_DPORT"); 302 case MLXCX_OP_DELETE_VXLAN_UDP_DPORT: 303 return ("MLXCX_OP_DELETE_VXLAN_UDP_DPORT"); 304 case MLXCX_OP_SET_L2_TABLE_ENTRY: 305 return ("MLXCX_OP_SET_L2_TABLE_ENTRY"); 306 case MLXCX_OP_QUERY_L2_TABLE_ENTRY: 307 return ("MLXCX_OP_QUERY_L2_TABLE_ENTRY"); 308 case MLXCX_OP_DELETE_L2_TABLE_ENTRY: 309 return ("MLXCX_OP_DELETE_L2_TABLE_ENTRY"); 310 case MLXCX_OP_SET_WOL_ROL: 311 return ("MLXCX_OP_SET_WOL_ROL"); 312 case MLXCX_OP_QUERY_WOL_ROL: 313 return ("MLXCX_OP_QUERY_WOL_ROL"); 314 case MLXCX_OP_CREATE_TIR: 315 return ("MLXCX_OP_CREATE_TIR"); 316 case MLXCX_OP_MODIFY_TIR: 317 return ("MLXCX_OP_MODIFY_TIR"); 318 case MLXCX_OP_DESTROY_TIR: 319 return ("MLXCX_OP_DESTROY_TIR"); 320 case MLXCX_OP_QUERY_TIR: 321 return ("MLXCX_OP_QUERY_TIR"); 322 case MLXCX_OP_CREATE_SQ: 323 return ("MLXCX_OP_CREATE_SQ"); 324 case MLXCX_OP_MODIFY_SQ: 325 return ("MLXCX_OP_MODIFY_SQ"); 326 case MLXCX_OP_DESTROY_SQ: 327 return ("MLXCX_OP_DESTROY_SQ"); 328 case MLXCX_OP_QUERY_SQ: 329 return ("MLXCX_OP_QUERY_SQ"); 330 case MLXCX_OP_CREATE_RQ: 331 return ("MLXCX_OP_CREATE_RQ"); 332 case MLXCX_OP_MODIFY_RQ: 333 return ("MLXCX_OP_MODIFY_RQ"); 334 case MLXCX_OP_DESTROY_RQ: 335 return ("MLXCX_OP_DESTROY_RQ"); 336 case MLXCX_OP_QUERY_RQ: 337 return ("MLXCX_OP_QUERY_RQ"); 338 case MLXCX_OP_CREATE_RMP: 339 return ("MLXCX_OP_CREATE_RMP"); 340 case MLXCX_OP_MODIFY_RMP: 341 return ("MLXCX_OP_MODIFY_RMP"); 342 case MLXCX_OP_DESTROY_RMP: 343 return ("MLXCX_OP_DESTROY_RMP"); 344 case MLXCX_OP_QUERY_RMP: 345 return ("MLXCX_OP_QUERY_RMP"); 346 case MLXCX_OP_CREATE_TIS: 347 return ("MLXCX_OP_CREATE_TIS"); 348 case MLXCX_OP_MODIFY_TIS: 349 return ("MLXCX_OP_MODIFY_TIS"); 350 case MLXCX_OP_DESTROY_TIS: 351 return ("MLXCX_OP_DESTROY_TIS"); 352 case MLXCX_OP_QUERY_TIS: 353 return ("MLXCX_OP_QUERY_TIS"); 354 case MLXCX_OP_CREATE_RQT: 355 return ("MLXCX_OP_CREATE_RQT"); 356 case MLXCX_OP_MODIFY_RQT: 357 return ("MLXCX_OP_MODIFY_RQT"); 358 case MLXCX_OP_DESTROY_RQT: 359 return ("MLXCX_OP_DESTROY_RQT"); 360 case MLXCX_OP_QUERY_RQT: 361 return ("MLXCX_OP_QUERY_RQT"); 362 case MLXCX_OP_SET_FLOW_TABLE_ROOT: 363 return ("MLXCX_OP_SET_FLOW_TABLE_ROOT"); 364 case MLXCX_OP_CREATE_FLOW_TABLE: 365 return ("MLXCX_OP_CREATE_FLOW_TABLE"); 366 case MLXCX_OP_DESTROY_FLOW_TABLE: 367 return ("MLXCX_OP_DESTROY_FLOW_TABLE"); 368 case MLXCX_OP_QUERY_FLOW_TABLE: 369 return ("MLXCX_OP_QUERY_FLOW_TABLE"); 370 case MLXCX_OP_CREATE_FLOW_GROUP: 371 return ("MLXCX_OP_CREATE_FLOW_GROUP"); 372 case MLXCX_OP_DESTROY_FLOW_GROUP: 373 return ("MLXCX_OP_DESTROY_FLOW_GROUP"); 374 case MLXCX_OP_QUERY_FLOW_GROUP: 375 return ("MLXCX_OP_QUERY_FLOW_GROUP"); 376 case MLXCX_OP_SET_FLOW_TABLE_ENTRY: 377 return ("MLXCX_OP_SET_FLOW_TABLE_ENTRY"); 378 case MLXCX_OP_QUERY_FLOW_TABLE_ENTRY: 379 return ("MLXCX_OP_QUERY_FLOW_TABLE_ENTRY"); 380 case MLXCX_OP_DELETE_FLOW_TABLE_ENTRY: 381 return ("MLXCX_OP_DELETE_FLOW_TABLE_ENTRY"); 382 case MLXCX_OP_ALLOC_FLOW_COUNTER: 383 return ("MLXCX_OP_ALLOC_FLOW_COUNTER"); 384 case MLXCX_OP_DEALLOC_FLOW_COUNTER: 385 return ("MLXCX_OP_DEALLOC_FLOW_COUNTER"); 386 case MLXCX_OP_QUERY_FLOW_COUNTER: 387 return ("MLXCX_OP_QUERY_FLOW_COUNTER"); 388 case MLXCX_OP_MODIFY_FLOW_TABLE: 389 return ("MLXCX_OP_MODIFY_FLOW_TABLE"); 390 case MLXCX_OP_ALLOC_ENCAP_HEADER: 391 return ("MLXCX_OP_ALLOC_ENCAP_HEADER"); 392 case MLXCX_OP_DEALLOC_ENCAP_HEADER: 393 return ("MLXCX_OP_DEALLOC_ENCAP_HEADER"); 394 case MLXCX_OP_QUERY_ENCAP_HEADER: 395 return ("MLXCX_OP_QUERY_ENCAP_HEADER"); 396 default: 397 return ("Unknown Opcode"); 398 } 399 } 400 401 const char * 402 mlxcx_port_status_string(mlxcx_port_status_t st) 403 { 404 switch (st) { 405 case MLXCX_PORT_STATUS_UP: 406 return ("UP"); 407 case MLXCX_PORT_STATUS_DOWN: 408 return ("DOWN"); 409 case MLXCX_PORT_STATUS_UP_ONCE: 410 return ("UP_ONCE"); 411 case MLXCX_PORT_STATUS_DISABLED: 412 return ("DISABLED"); 413 default: 414 return ("UNKNOWN"); 415 } 416 } 417 418 void 419 mlxcx_eth_proto_to_string(mlxcx_eth_proto_t p, char *buf, size_t size) 420 { 421 if (p & MLXCX_PROTO_SGMII) 422 (void) strlcat(buf, "SGMII|", size); 423 if (p & MLXCX_PROTO_1000BASE_KX) 424 (void) strlcat(buf, "1000BASE_KX|", size); 425 if (p & MLXCX_PROTO_10GBASE_CX4) 426 (void) strlcat(buf, "10GBASE_CX4|", size); 427 if (p & MLXCX_PROTO_10GBASE_KX4) 428 (void) strlcat(buf, "10GBASE_KX4|", size); 429 if (p & MLXCX_PROTO_10GBASE_KR) 430 (void) strlcat(buf, "10GBASE_KR|", size); 431 if (p & MLXCX_PROTO_40GBASE_CR4) 432 (void) strlcat(buf, "40GBASE_CR4|", size); 433 if (p & MLXCX_PROTO_40GBASE_KR4) 434 (void) strlcat(buf, "40GBASE_KR4|", size); 435 if (p & MLXCX_PROTO_SGMII_100BASE) 436 (void) strlcat(buf, "SGMII_100BASE|", size); 437 if (p & MLXCX_PROTO_10GBASE_CR) 438 (void) strlcat(buf, "10GBASE_CR|", size); 439 if (p & MLXCX_PROTO_10GBASE_SR) 440 (void) strlcat(buf, "10GBASE_SR|", size); 441 if (p & MLXCX_PROTO_10GBASE_ER_LR) 442 (void) strlcat(buf, "10GBASE_ER_LR|", size); 443 if (p & MLXCX_PROTO_40GBASE_SR4) 444 (void) strlcat(buf, "40GBASE_SR4|", size); 445 if (p & MLXCX_PROTO_40GBASE_LR4_ER4) 446 (void) strlcat(buf, "40GBASE_LR4_ER4|", size); 447 if (p & MLXCX_PROTO_50GBASE_SR2) 448 (void) strlcat(buf, "50GBASE_SR2|", size); 449 if (p & MLXCX_PROTO_100GBASE_CR4) 450 (void) strlcat(buf, "100GBASE_CR4|", size); 451 if (p & MLXCX_PROTO_100GBASE_SR4) 452 (void) strlcat(buf, "100GBASE_SR4|", size); 453 if (p & MLXCX_PROTO_100GBASE_KR4) 454 (void) strlcat(buf, "100GBASE_KR4|", size); 455 if (p & MLXCX_PROTO_25GBASE_CR) 456 (void) strlcat(buf, "25GBASE_CR|", size); 457 if (p & MLXCX_PROTO_25GBASE_KR) 458 (void) strlcat(buf, "25GBASE_KR|", size); 459 if (p & MLXCX_PROTO_25GBASE_SR) 460 (void) strlcat(buf, "25GBASE_SR|", size); 461 if (p & MLXCX_PROTO_50GBASE_CR2) 462 (void) strlcat(buf, "50GBASE_CR2|", size); 463 /* Chop off the trailing '|' */ 464 if (strlen(buf) > 0) 465 buf[strlen(buf) - 1] = '\0'; 466 } 467 468 void 469 mlxcx_cmd_queue_fini(mlxcx_t *mlxp) 470 { 471 mlxcx_cmd_queue_t *cmd = &mlxp->mlx_cmd; 472 473 if (cmd->mcmd_tokens != NULL) { 474 id_space_destroy(cmd->mcmd_tokens); 475 cmd->mcmd_tokens = NULL; 476 } 477 478 if (cmd->mcmd_taskq != NULL) { 479 ddi_taskq_destroy(cmd->mcmd_taskq); 480 cmd->mcmd_taskq = NULL; 481 } 482 483 cv_destroy(&cmd->mcmd_cv); 484 mutex_destroy(&cmd->mcmd_lock); 485 486 mlxcx_dma_free(&cmd->mcmd_dma); 487 } 488 489 boolean_t 490 mlxcx_cmd_queue_init(mlxcx_t *mlxp) 491 { 492 uint32_t tmp, cmd_low, cmd_high, i; 493 mlxcx_cmd_queue_t *cmd = &mlxp->mlx_cmd; 494 char buf[32]; 495 char tq_name[TASKQ_NAMELEN]; 496 const ddi_dma_cookie_t *ck; 497 498 ddi_device_acc_attr_t acc; 499 ddi_dma_attr_t attr; 500 501 tmp = mlxcx_get32(mlxp, MLXCX_ISS_FIRMWARE); 502 mlxp->mlx_fw_maj = MLXCX_ISS_FW_MAJOR(tmp); 503 mlxp->mlx_fw_min = MLXCX_ISS_FW_MINOR(tmp); 504 505 tmp = mlxcx_get32(mlxp, MLXCX_ISS_FW_CMD); 506 mlxp->mlx_fw_rev = MLXCX_ISS_FW_REV(tmp); 507 mlxp->mlx_cmd_rev = MLXCX_ISS_CMD_REV(tmp); 508 509 if (mlxp->mlx_cmd_rev != MLXCX_CMD_REVISION) { 510 mlxcx_warn(mlxp, "found unsupported command revision: %u, " 511 "expected %u", mlxp->mlx_cmd_rev, MLXCX_CMD_REVISION); 512 return (B_FALSE); 513 } 514 515 cmd_low = mlxcx_get32(mlxp, MLXCX_ISS_CMD_LOW); 516 cmd->mcmd_size_l2 = MLXCX_ISS_CMDQ_SIZE(cmd_low); 517 cmd->mcmd_stride_l2 = MLXCX_ISS_CMDQ_STRIDE(cmd_low); 518 cmd->mcmd_size = 1U << cmd->mcmd_size_l2; 519 520 if (cmd->mcmd_size > MLXCX_CMD_MAX) { 521 mlxcx_warn(mlxp, "command queue size %u is too " 522 "large. Maximum is %u", cmd->mcmd_size, MLXCX_CMD_MAX); 523 return (B_FALSE); 524 } 525 526 cmd->mcmd_mask = (uint32_t)((1ULL << cmd->mcmd_size) - 1); 527 528 mutex_init(&cmd->mcmd_lock, NULL, MUTEX_DRIVER, NULL); 529 cv_init(&cmd->mcmd_cv, NULL, CV_DRIVER, NULL); 530 531 (void) snprintf(buf, sizeof (buf), "mlxcx_tokens_%d", mlxp->mlx_inst); 532 if ((cmd->mcmd_tokens = id_space_create(buf, 1, UINT8_MAX)) == NULL) { 533 mlxcx_warn(mlxp, "failed to allocate token id space"); 534 mlxcx_cmd_queue_fini(mlxp); 535 return (B_FALSE); 536 } 537 538 (void) snprintf(tq_name, sizeof (tq_name), "cmdq_%d", mlxp->mlx_inst); 539 if ((cmd->mcmd_taskq = ddi_taskq_create(mlxp->mlx_dip, tq_name, 1, 540 TASKQ_DEFAULTPRI, 0)) == NULL) { 541 mlxcx_warn(mlxp, "failed to create command queue task queue"); 542 mlxcx_cmd_queue_fini(mlxp); 543 return (B_FALSE); 544 } 545 546 mlxcx_dma_acc_attr(mlxp, &acc); 547 mlxcx_dma_page_attr(mlxp, &attr); 548 549 if (!mlxcx_dma_alloc(mlxp, &cmd->mcmd_dma, &attr, &acc, B_TRUE, 550 MLXCX_CMD_DMA_PAGE_SIZE, B_TRUE)) { 551 mlxcx_warn(mlxp, "failed to allocate command dma buffer"); 552 mlxcx_cmd_queue_fini(mlxp); 553 return (B_FALSE); 554 } 555 556 ck = mlxcx_dma_cookie_one(&cmd->mcmd_dma); 557 cmd_high = (uint32_t)(ck->dmac_laddress >> 32); 558 cmd_low = (uint32_t)(ck->dmac_laddress & UINT32_MAX); 559 560 mlxcx_put32(mlxp, MLXCX_ISS_CMD_HIGH, cmd_high); 561 mlxcx_put32(mlxp, MLXCX_ISS_CMD_LOW, cmd_low); 562 563 /* 564 * Before this is ready, the initializing bit must become zero. 565 */ 566 for (i = 0; i < mlxcx_cmd_init_trys; i++) { 567 uint32_t init = mlxcx_get32(mlxp, MLXCX_ISS_INIT); 568 569 if (MLXCX_ISS_INITIALIZING(init) == 0) 570 break; 571 delay(drv_usectohz(mlxcx_cmd_init_delay)); 572 } 573 if (i == mlxcx_cmd_init_trys) { 574 mlxcx_warn(mlxp, "timed out initializing command queue"); 575 mlxcx_cmd_queue_fini(mlxp); 576 return (B_FALSE); 577 } 578 579 /* 580 * start in polling mode. 581 */ 582 mlxcx_cmd_eq_disable(mlxp); 583 584 return (B_TRUE); 585 } 586 587 void 588 mlxcx_cmd_eq_enable(mlxcx_t *mlxp) 589 { 590 mlxp->mlx_cmd.mcmd_polled = B_FALSE; 591 } 592 593 void 594 mlxcx_cmd_eq_disable(mlxcx_t *mlxp) 595 { 596 mlxp->mlx_cmd.mcmd_polled = B_TRUE; 597 } 598 599 static void 600 mlxcx_cmd_in_header_init(mlxcx_cmd_t *cmd, mlxcx_cmd_in_t *in, 601 mlxcx_cmd_op_t op, uint16_t mod) 602 { 603 ASSERT3U(op, <=, UINT16_MAX); 604 in->mci_opcode = to_be16(op); 605 in->mci_op_mod = to_be16(mod); 606 cmd->mlcmd_op = op; 607 } 608 609 static boolean_t 610 mlxcx_cmd_mbox_alloc(mlxcx_t *mlxp, list_t *listp, uint8_t nblocks) 611 { 612 uint8_t i; 613 ddi_device_acc_attr_t acc; 614 ddi_dma_attr_t attr; 615 616 mlxcx_dma_acc_attr(mlxp, &acc); 617 mlxcx_dma_page_attr(mlxp, &attr); 618 619 for (i = 0; i < nblocks; i++) { 620 mlxcx_cmd_mbox_t *mbox; 621 622 mbox = kmem_zalloc(sizeof (*mbox), KM_SLEEP); 623 if (!mlxcx_dma_alloc(mlxp, &mbox->mlbox_dma, &attr, &acc, 624 B_TRUE, sizeof (mlxcx_cmd_mailbox_t), B_TRUE)) { 625 mlxcx_warn(mlxp, "failed to allocate mailbox dma " 626 "buffer"); 627 kmem_free(mbox, sizeof (*mbox)); 628 /* 629 * mlxcx_cmd_fini will clean up any mboxes that we 630 * already placed onto listp. 631 */ 632 return (B_FALSE); 633 } 634 mbox->mlbox_data = (void *)mbox->mlbox_dma.mxdb_va; 635 list_insert_tail(listp, mbox); 636 } 637 638 return (B_TRUE); 639 } 640 641 static void 642 mlxcx_cmd_mbox_free(mlxcx_cmd_mbox_t *mbox) 643 { 644 mlxcx_dma_free(&mbox->mlbox_dma); 645 kmem_free(mbox, sizeof (mlxcx_cmd_mbox_t)); 646 } 647 648 static void 649 mlxcx_cmd_fini(mlxcx_t *mlxp, mlxcx_cmd_t *cmd) 650 { 651 mlxcx_cmd_mbox_t *mbox; 652 653 while ((mbox = list_remove_head(&cmd->mlcmd_mbox_out)) != NULL) { 654 mlxcx_cmd_mbox_free(mbox); 655 } 656 list_destroy(&cmd->mlcmd_mbox_out); 657 while ((mbox = list_remove_head(&cmd->mlcmd_mbox_in)) != NULL) { 658 mlxcx_cmd_mbox_free(mbox); 659 } 660 list_destroy(&cmd->mlcmd_mbox_in); 661 id_free(mlxp->mlx_cmd.mcmd_tokens, cmd->mlcmd_token); 662 cv_destroy(&cmd->mlcmd_cv); 663 mutex_destroy(&cmd->mlcmd_lock); 664 } 665 666 static void 667 mlxcx_cmd_init(mlxcx_t *mlxp, mlxcx_cmd_t *cmd) 668 { 669 bzero(cmd, sizeof (*cmd)); 670 mutex_init(&cmd->mlcmd_lock, NULL, MUTEX_DRIVER, 671 DDI_INTR_PRI(mlxp->mlx_async_intr_pri)); 672 cv_init(&cmd->mlcmd_cv, NULL, CV_DRIVER, NULL); 673 cmd->mlcmd_token = id_alloc(mlxp->mlx_cmd.mcmd_tokens); 674 cmd->mlcmd_poll = mlxp->mlx_cmd.mcmd_polled; 675 list_create(&cmd->mlcmd_mbox_in, sizeof (mlxcx_cmd_mbox_t), 676 offsetof(mlxcx_cmd_mbox_t, mlbox_node)); 677 list_create(&cmd->mlcmd_mbox_out, sizeof (mlxcx_cmd_mbox_t), 678 offsetof(mlxcx_cmd_mbox_t, mlbox_node)); 679 } 680 681 static void 682 mlxcx_cmd_prep_input(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd) 683 { 684 uint32_t rem = cmd->mlcmd_inlen; 685 uint8_t i; 686 const void *in = cmd->mlcmd_in; 687 uint32_t copy; 688 mlxcx_cmd_mbox_t *mbox; 689 const ddi_dma_cookie_t *ck; 690 691 copy = MIN(MLXCX_CMD_INLINE_INPUT_LEN, rem); 692 bcopy(in, ent->mce_input, copy); 693 694 rem -= copy; 695 in += copy; 696 697 if (rem == 0) { 698 ent->mce_in_mbox = to_be64(0); 699 VERIFY3U(cmd->mlcmd_nboxes_in, ==, 0); 700 return; 701 } 702 703 mbox = list_head(&cmd->mlcmd_mbox_in); 704 ck = mlxcx_dma_cookie_one(&mbox->mlbox_dma); 705 ent->mce_in_mbox = to_be64(ck->dmac_laddress); 706 for (i = 0; mbox != NULL; 707 mbox = list_next(&cmd->mlcmd_mbox_in, mbox), i++) { 708 mlxcx_cmd_mbox_t *next; 709 mlxcx_cmd_mailbox_t *mp = mbox->mlbox_data; 710 711 copy = MIN(MLXCX_CMD_MAILBOX_LEN, rem); 712 bcopy(in, mp->mlxb_data, copy); 713 rem -= copy; 714 in += copy; 715 716 mp->mlxb_token = cmd->mlcmd_token; 717 mp->mlxb_blockno = to_be32(i); 718 719 next = list_next(&cmd->mlcmd_mbox_in, mbox); 720 if (next == NULL) { 721 mp->mlxb_nextp = to_be64(0); 722 } else { 723 ck = mlxcx_dma_cookie_one(&next->mlbox_dma); 724 mp->mlxb_nextp = to_be64(ck->dmac_laddress); 725 } 726 MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORDEV); 727 } 728 VERIFY3U(i, ==, cmd->mlcmd_nboxes_in); 729 VERIFY0(rem); 730 } 731 732 static void 733 mlxcx_cmd_prep_output(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd) 734 { 735 uint8_t i; 736 mlxcx_cmd_mbox_t *mbox; 737 const ddi_dma_cookie_t *ck; 738 739 if (cmd->mlcmd_nboxes_out == 0) { 740 ent->mce_out_mbox = to_be64(0); 741 return; 742 } 743 744 mbox = list_head(&cmd->mlcmd_mbox_out); 745 ck = mlxcx_dma_cookie_one(&mbox->mlbox_dma); 746 ent->mce_out_mbox = to_be64(ck->dmac_laddress); 747 for (i = 0, mbox = list_head(&cmd->mlcmd_mbox_out); mbox != NULL; 748 mbox = list_next(&cmd->mlcmd_mbox_out, mbox), i++) { 749 mlxcx_cmd_mbox_t *next; 750 mlxcx_cmd_mailbox_t *mp = mbox->mlbox_data; 751 752 mp->mlxb_token = cmd->mlcmd_token; 753 mp->mlxb_blockno = to_be32(i); 754 755 next = list_next(&cmd->mlcmd_mbox_out, mbox); 756 if (next == NULL) { 757 mp->mlxb_nextp = to_be64(0); 758 } else { 759 ck = mlxcx_dma_cookie_one(&next->mlbox_dma); 760 mp->mlxb_nextp = to_be64(ck->dmac_laddress); 761 } 762 MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORDEV); 763 } 764 VERIFY3U(i, ==, cmd->mlcmd_nboxes_out); 765 } 766 767 static void 768 mlxcx_cmd_copy_output(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd) 769 { 770 void *out = cmd->mlcmd_out; 771 uint32_t rem = cmd->mlcmd_outlen; 772 uint32_t copy; 773 mlxcx_cmd_mbox_t *mbox; 774 775 copy = MIN(rem, MLXCX_CMD_INLINE_OUTPUT_LEN); 776 bcopy(ent->mce_output, out, copy); 777 out += copy; 778 rem -= copy; 779 780 if (rem == 0) { 781 VERIFY0(cmd->mlcmd_nboxes_out); 782 return; 783 } 784 785 for (mbox = list_head(&cmd->mlcmd_mbox_out); mbox != NULL; 786 mbox = list_next(&cmd->mlcmd_mbox_out, mbox)) { 787 MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORKERNEL); 788 copy = MIN(MLXCX_CMD_MAILBOX_LEN, rem); 789 bcopy(mbox->mlbox_data->mlxb_data, out, copy); 790 out += copy; 791 rem -= copy; 792 } 793 VERIFY0(rem); 794 } 795 796 static uint_t 797 mlxcx_cmd_reserve_slot(mlxcx_cmd_queue_t *cmdq) 798 { 799 uint_t slot; 800 801 mutex_enter(&cmdq->mcmd_lock); 802 slot = ddi_ffs(cmdq->mcmd_mask); 803 while (slot == 0) { 804 cv_wait(&cmdq->mcmd_cv, &cmdq->mcmd_lock); 805 slot = ddi_ffs(cmdq->mcmd_mask); 806 } 807 808 cmdq->mcmd_mask &= ~(1U << --slot); 809 810 ASSERT3P(cmdq->mcmd_active[slot], ==, NULL); 811 812 mutex_exit(&cmdq->mcmd_lock); 813 814 return (slot); 815 } 816 817 static void 818 mlxcx_cmd_release_slot(mlxcx_cmd_queue_t *cmdq, uint_t slot) 819 { 820 mutex_enter(&cmdq->mcmd_lock); 821 cmdq->mcmd_mask |= 1U << slot; 822 cv_broadcast(&cmdq->mcmd_cv); 823 mutex_exit(&cmdq->mcmd_lock); 824 } 825 826 static void 827 mlxcx_cmd_done(mlxcx_cmd_t *cmd, uint_t slot) 828 { 829 mlxcx_t *mlxp = cmd->mlcmd_mlxp; 830 mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd; 831 mlxcx_cmd_ent_t *ent; 832 833 /* 834 * Command is done. Save relevant data. Once we broadcast on the CV and 835 * drop the lock, we must not touch it again. 836 */ 837 MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORKERNEL); 838 839 ent = (mlxcx_cmd_ent_t *)(cmdq->mcmd_dma.mxdb_va + 840 (slot << cmdq->mcmd_stride_l2)); 841 842 mutex_enter(&cmd->mlcmd_lock); 843 cmd->mlcmd_status = MLXCX_CMD_STATUS(ent->mce_status); 844 if (cmd->mlcmd_status == 0) 845 mlxcx_cmd_copy_output(ent, cmd); 846 847 cmd->mlcmd_state = MLXCX_CMD_S_DONE; 848 cv_broadcast(&cmd->mlcmd_cv); 849 mutex_exit(&cmd->mlcmd_lock); 850 851 cmdq->mcmd_active[slot] = NULL; 852 mlxcx_cmd_release_slot(cmdq, slot); 853 } 854 855 static void 856 mlxcx_cmd_taskq(void *arg) 857 { 858 mlxcx_cmd_t *cmd = arg; 859 mlxcx_t *mlxp = cmd->mlcmd_mlxp; 860 mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd; 861 mlxcx_cmd_ent_t *ent; 862 uint_t poll, slot; 863 864 ASSERT3S(cmd->mlcmd_op, !=, 0); 865 866 slot = mlxcx_cmd_reserve_slot(cmdq); 867 ent = (mlxcx_cmd_ent_t *)(cmdq->mcmd_dma.mxdb_va + 868 (slot << cmdq->mcmd_stride_l2)); 869 870 cmdq->mcmd_active[slot] = cmd; 871 872 /* 873 * Command queue is currently ours as we set busy. 874 */ 875 bzero(ent, sizeof (*ent)); 876 ent->mce_type = MLXCX_CMD_TRANSPORT_PCI; 877 ent->mce_in_length = to_be32(cmd->mlcmd_inlen); 878 ent->mce_out_length = to_be32(cmd->mlcmd_outlen); 879 ent->mce_token = cmd->mlcmd_token; 880 ent->mce_sig = 0; 881 ent->mce_status = MLXCX_CMD_HW_OWNED; 882 mlxcx_cmd_prep_input(ent, cmd); 883 mlxcx_cmd_prep_output(ent, cmd); 884 MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORDEV); 885 886 mlxcx_put32(mlxp, MLXCX_ISS_CMD_DOORBELL, 1 << slot); 887 888 if (!cmd->mlcmd_poll) 889 return; 890 891 for (poll = 0; poll < mlxcx_cmd_tries; poll++) { 892 delay(drv_usectohz(mlxcx_cmd_delay)); 893 MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORKERNEL); 894 if ((ent->mce_status & MLXCX_CMD_HW_OWNED) == 0) 895 break; 896 } 897 898 /* 899 * Command is done (or timed out). Save relevant data. Once we broadcast 900 * on the CV and drop the lock, we must not touch the cmd again. 901 */ 902 903 if (poll == mlxcx_cmd_tries) { 904 mutex_enter(&cmd->mlcmd_lock); 905 cmd->mlcmd_status = MLXCX_CMD_R_TIMEOUT; 906 cmd->mlcmd_state = MLXCX_CMD_S_ERROR; 907 cv_broadcast(&cmd->mlcmd_cv); 908 mutex_exit(&cmd->mlcmd_lock); 909 910 mlxcx_fm_ereport(mlxp, DDI_FM_DEVICE_NO_RESPONSE); 911 912 cmdq->mcmd_active[slot] = NULL; 913 mlxcx_cmd_release_slot(cmdq, slot); 914 915 return; 916 } 917 918 mlxcx_cmd_done(cmd, slot); 919 } 920 921 void 922 mlxcx_cmd_completion(mlxcx_t *mlxp, mlxcx_eventq_ent_t *ent) 923 { 924 mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd; 925 mlxcx_evdata_cmd_completion_t *eqe_cmd = &ent->mleqe_cmd_completion; 926 mlxcx_cmd_t *cmd; 927 uint32_t comp_vec = from_be32(eqe_cmd->mled_cmd_completion_vec); 928 uint_t slot; 929 930 DTRACE_PROBE2(cmd_event, mlxcx_t *, mlxp, 931 mlxcx_evdata_cmd_completion_t *, eqe_cmd); 932 933 while ((slot = ddi_ffs(comp_vec)) != 0) { 934 comp_vec &= ~(1U << --slot); 935 936 cmd = cmdq->mcmd_active[slot]; 937 if (cmd->mlcmd_poll) 938 continue; 939 940 mlxcx_cmd_done(cmd, slot); 941 } 942 } 943 944 static boolean_t 945 mlxcx_cmd_send(mlxcx_t *mlxp, mlxcx_cmd_t *cmd, const void *in, uint32_t inlen, 946 void *out, uint32_t outlen) 947 { 948 if (inlen > MLXCX_CMD_INLINE_INPUT_LEN) { 949 uint32_t need = inlen - MLXCX_CMD_INLINE_INPUT_LEN; 950 uint8_t nblocks; 951 952 if (need / MLXCX_CMD_MAILBOX_LEN + 1 > UINT8_MAX) { 953 mlxcx_warn(mlxp, "requested too many input blocks for " 954 "%u byte input len", inlen); 955 return (B_FALSE); 956 } 957 958 nblocks = need / MLXCX_CMD_MAILBOX_LEN + 1; 959 if (!mlxcx_cmd_mbox_alloc(mlxp, &cmd->mlcmd_mbox_in, nblocks)) { 960 mlxcx_warn(mlxp, "failed to allocate %u blocks of " 961 "input mailbox", nblocks); 962 return (B_FALSE); 963 } 964 cmd->mlcmd_nboxes_in = nblocks; 965 } 966 967 if (outlen > MLXCX_CMD_INLINE_OUTPUT_LEN) { 968 uint32_t need = outlen - MLXCX_CMD_INLINE_OUTPUT_LEN; 969 uint8_t nblocks; 970 971 if (need / MLXCX_CMD_MAILBOX_LEN + 1 > UINT8_MAX) { 972 mlxcx_warn(mlxp, "requested too many output blocks for " 973 "%u byte output len", outlen); 974 return (B_FALSE); 975 } 976 977 nblocks = need / MLXCX_CMD_MAILBOX_LEN + 1; 978 if (!mlxcx_cmd_mbox_alloc(mlxp, &cmd->mlcmd_mbox_out, 979 nblocks)) { 980 mlxcx_warn(mlxp, "failed to allocate %u blocks of " 981 "output mailbox", nblocks); 982 return (B_FALSE); 983 } 984 cmd->mlcmd_nboxes_out = nblocks; 985 } 986 987 cmd->mlcmd_in = in; 988 cmd->mlcmd_inlen = inlen; 989 cmd->mlcmd_out = out; 990 cmd->mlcmd_outlen = outlen; 991 cmd->mlcmd_mlxp = mlxp; 992 993 /* 994 * Now that all allocations have been done, all that remains is for us 995 * to dispatch the request to process this to the taskq for it to be 996 * processed. 997 */ 998 if (ddi_taskq_dispatch(mlxp->mlx_cmd.mcmd_taskq, mlxcx_cmd_taskq, cmd, 999 DDI_SLEEP) != DDI_SUCCESS) { 1000 mlxcx_warn(mlxp, "failed to submit command to taskq"); 1001 return (B_FALSE); 1002 } 1003 1004 return (B_TRUE); 1005 } 1006 1007 static void 1008 mlxcx_cmd_wait(mlxcx_cmd_t *cmd) 1009 { 1010 mutex_enter(&cmd->mlcmd_lock); 1011 while (cmd->mlcmd_state == 0) { 1012 cv_wait(&cmd->mlcmd_cv, &cmd->mlcmd_lock); 1013 } 1014 mutex_exit(&cmd->mlcmd_lock); 1015 } 1016 1017 static boolean_t 1018 mlxcx_cmd_evaluate(mlxcx_t *mlxp, mlxcx_cmd_t *cmd) 1019 { 1020 mlxcx_cmd_out_t *out; 1021 1022 if ((cmd->mlcmd_state & MLXCX_CMD_S_ERROR) != 0) { 1023 mlxcx_warn(mlxp, "command %s (0x%x) failed due to an internal " 1024 "driver error", 1025 mlxcx_cmd_opcode_string(cmd->mlcmd_op), 1026 cmd->mlcmd_op); 1027 return (B_FALSE); 1028 } 1029 1030 if (cmd->mlcmd_status != 0) { 1031 mlxcx_warn(mlxp, "command %s (0x%x) failed with command queue " 1032 "error 0x%x", 1033 mlxcx_cmd_opcode_string(cmd->mlcmd_op), 1034 cmd->mlcmd_op, cmd->mlcmd_status); 1035 return (B_FALSE); 1036 } 1037 1038 out = cmd->mlcmd_out; 1039 if (out->mco_status != MLXCX_CMD_R_OK) { 1040 mlxcx_warn(mlxp, "command %s 0x%x failed with status code %s " 1041 "(0x%x)", mlxcx_cmd_opcode_string(cmd->mlcmd_op), 1042 cmd->mlcmd_op, mlxcx_cmd_response_string(out->mco_status), 1043 out->mco_status); 1044 return (B_FALSE); 1045 } 1046 1047 return (B_TRUE); 1048 } 1049 1050 boolean_t 1051 mlxcx_cmd_disable_hca(mlxcx_t *mlxp) 1052 { 1053 mlxcx_cmd_t cmd; 1054 mlxcx_cmd_disable_hca_in_t in; 1055 mlxcx_cmd_disable_hca_out_t out; 1056 boolean_t ret; 1057 1058 bzero(&in, sizeof (in)); 1059 bzero(&out, sizeof (out)); 1060 1061 mlxcx_cmd_init(mlxp, &cmd); 1062 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_disable_hca_head, 1063 MLXCX_OP_DISABLE_HCA, 0); 1064 in.mlxi_disable_hca_func = MLXCX_FUNCTION_SELF; 1065 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1066 mlxcx_cmd_fini(mlxp, &cmd); 1067 return (B_FALSE); 1068 } 1069 mlxcx_cmd_wait(&cmd); 1070 1071 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1072 mlxcx_cmd_fini(mlxp, &cmd); 1073 return (ret); 1074 } 1075 1076 boolean_t 1077 mlxcx_cmd_enable_hca(mlxcx_t *mlxp) 1078 { 1079 mlxcx_cmd_t cmd; 1080 mlxcx_cmd_enable_hca_in_t in; 1081 mlxcx_cmd_enable_hca_out_t out; 1082 boolean_t ret; 1083 1084 bzero(&in, sizeof (in)); 1085 bzero(&out, sizeof (out)); 1086 1087 mlxcx_cmd_init(mlxp, &cmd); 1088 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_enable_hca_head, 1089 MLXCX_OP_ENABLE_HCA, 0); 1090 in.mlxi_enable_hca_func = MLXCX_FUNCTION_SELF; 1091 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1092 mlxcx_cmd_fini(mlxp, &cmd); 1093 return (B_FALSE); 1094 } 1095 mlxcx_cmd_wait(&cmd); 1096 1097 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1098 mlxcx_cmd_fini(mlxp, &cmd); 1099 return (ret); 1100 } 1101 1102 boolean_t 1103 mlxcx_cmd_query_issi(mlxcx_t *mlxp, uint32_t *issip) 1104 { 1105 mlxcx_cmd_t cmd; 1106 mlxcx_cmd_query_issi_in_t in; 1107 mlxcx_cmd_query_issi_out_t out; 1108 boolean_t ret; 1109 1110 bzero(&in, sizeof (in)); 1111 bzero(&out, sizeof (out)); 1112 1113 mlxcx_cmd_init(mlxp, &cmd); 1114 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_issi_head, 1115 MLXCX_OP_QUERY_ISSI, 0); 1116 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1117 mlxcx_cmd_fini(mlxp, &cmd); 1118 return (B_FALSE); 1119 } 1120 mlxcx_cmd_wait(&cmd); 1121 1122 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1123 if (ret) { 1124 *issip = out.mlxo_supported_issi; 1125 } else if (cmd.mlcmd_status == 0 && 1126 out.mlxo_query_issi_head.mco_status == MLXCX_CMD_R_BAD_OP) { 1127 /* 1128 * The PRM says that if we get a bad operation, that means this 1129 * command isn't supported so it only supports version 1 of the 1130 * ISSI, which means bit zero should be set. 1131 */ 1132 ret = B_TRUE; 1133 *issip = 1; 1134 } 1135 mlxcx_cmd_fini(mlxp, &cmd); 1136 return (ret); 1137 } 1138 1139 boolean_t 1140 mlxcx_cmd_set_issi(mlxcx_t *mlxp, uint16_t issi) 1141 { 1142 mlxcx_cmd_t cmd; 1143 mlxcx_cmd_set_issi_in_t in; 1144 mlxcx_cmd_set_issi_out_t out; 1145 boolean_t ret; 1146 1147 bzero(&in, sizeof (in)); 1148 bzero(&out, sizeof (out)); 1149 1150 mlxcx_cmd_init(mlxp, &cmd); 1151 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_issi_head, 1152 MLXCX_OP_SET_ISSI, 0); 1153 in.mlxi_set_issi_current = to_be16(issi); 1154 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1155 mlxcx_cmd_fini(mlxp, &cmd); 1156 return (B_FALSE); 1157 } 1158 mlxcx_cmd_wait(&cmd); 1159 1160 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1161 mlxcx_cmd_fini(mlxp, &cmd); 1162 return (ret); 1163 } 1164 1165 boolean_t 1166 mlxcx_cmd_query_pages(mlxcx_t *mlxp, uint_t type, int32_t *npages) 1167 { 1168 mlxcx_cmd_t cmd; 1169 mlxcx_cmd_query_pages_in_t in; 1170 mlxcx_cmd_query_pages_out_t out; 1171 boolean_t ret; 1172 1173 switch (type) { 1174 case MLXCX_QUERY_PAGES_OPMOD_BOOT: 1175 case MLXCX_QUERY_PAGES_OPMOD_INIT: 1176 case MLXCX_QUERY_PAGES_OPMOD_REGULAR: 1177 break; 1178 default: 1179 mlxcx_warn(mlxp, "!passed invalid type to query pages: %u", 1180 type); 1181 return (B_FALSE); 1182 } 1183 1184 bzero(&in, sizeof (in)); 1185 bzero(&out, sizeof (out)); 1186 1187 mlxcx_cmd_init(mlxp, &cmd); 1188 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_pages_head, 1189 MLXCX_OP_QUERY_PAGES, type); 1190 in.mlxi_query_pages_func = MLXCX_FUNCTION_SELF; 1191 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1192 mlxcx_cmd_fini(mlxp, &cmd); 1193 return (B_FALSE); 1194 } 1195 mlxcx_cmd_wait(&cmd); 1196 1197 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1198 if (ret) { 1199 *npages = from_be32(out.mlxo_query_pages_npages); 1200 } 1201 mlxcx_cmd_fini(mlxp, &cmd); 1202 1203 return (ret); 1204 } 1205 1206 boolean_t 1207 mlxcx_cmd_give_pages(mlxcx_t *mlxp, uint_t type, int32_t npages, 1208 mlxcx_dev_page_t **pages) 1209 { 1210 mlxcx_cmd_t cmd; 1211 mlxcx_cmd_manage_pages_in_t *in; 1212 mlxcx_cmd_manage_pages_out_t out; 1213 size_t insize, outsize; 1214 boolean_t ret; 1215 uint32_t i; 1216 uint64_t pa; 1217 const ddi_dma_cookie_t *ck; 1218 1219 switch (type) { 1220 case MLXCX_MANAGE_PAGES_OPMOD_ALLOC_FAIL: 1221 if (npages != 0) { 1222 mlxcx_warn(mlxp, "passed non-zero number of pages (%d) " 1223 "but asked to fail page allocation", npages); 1224 return (B_FALSE); 1225 } 1226 break; 1227 case MLXCX_MANAGE_PAGES_OPMOD_GIVE_PAGES: 1228 ASSERT3S(npages, <=, MLXCX_MANAGE_PAGES_MAX_PAGES); 1229 if (npages <= 0) { 1230 mlxcx_warn(mlxp, "passed invalid number of pages (%d) " 1231 "to give pages", npages); 1232 return (B_FALSE); 1233 } 1234 break; 1235 default: 1236 mlxcx_warn(mlxp, "!passed invalid type to give pages: %u", 1237 type); 1238 return (B_FALSE); 1239 } 1240 1241 insize = offsetof(mlxcx_cmd_manage_pages_in_t, mlxi_manage_pages_pas) + 1242 npages * sizeof (uint64_t); 1243 outsize = offsetof(mlxcx_cmd_manage_pages_out_t, mlxo_manage_pages_pas); 1244 1245 in = kmem_zalloc(insize, KM_SLEEP); 1246 bzero(&out, sizeof (out)); 1247 1248 mlxcx_cmd_init(mlxp, &cmd); 1249 mlxcx_cmd_in_header_init(&cmd, &in->mlxi_manage_pages_head, 1250 MLXCX_OP_MANAGE_PAGES, type); 1251 in->mlxi_manage_pages_func = MLXCX_FUNCTION_SELF; 1252 in->mlxi_manage_pages_npages = to_be32(npages); 1253 for (i = 0; i < npages; i++) { 1254 ck = mlxcx_dma_cookie_one(&pages[i]->mxdp_dma); 1255 pa = ck->dmac_laddress; 1256 ASSERT3U(pa & 0xfff, ==, 0); 1257 ASSERT3U(ck->dmac_size, ==, MLXCX_HW_PAGE_SIZE); 1258 in->mlxi_manage_pages_pas[i] = to_be64(pa); 1259 } 1260 1261 if ((ret = mlxcx_cmd_send(mlxp, &cmd, in, insize, &out, outsize))) { 1262 mlxcx_cmd_wait(&cmd); 1263 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1264 } 1265 1266 mlxcx_cmd_fini(mlxp, &cmd); 1267 1268 kmem_free(in, insize); 1269 return (ret); 1270 } 1271 1272 boolean_t 1273 mlxcx_cmd_return_pages(mlxcx_t *mlxp, int32_t nreq, uint64_t *pas, 1274 int32_t *nret) 1275 { 1276 mlxcx_cmd_t cmd; 1277 mlxcx_cmd_manage_pages_in_t in; 1278 mlxcx_cmd_manage_pages_out_t *out; 1279 size_t insize, outsize; 1280 boolean_t ret; 1281 uint32_t i; 1282 1283 if (nreq <= 0) { 1284 mlxcx_warn(mlxp, "passed invalid number of pages (%d) " 1285 "to return pages", nreq); 1286 return (B_FALSE); 1287 } 1288 1289 insize = offsetof(mlxcx_cmd_manage_pages_in_t, mlxi_manage_pages_pas); 1290 outsize = offsetof(mlxcx_cmd_manage_pages_out_t, 1291 mlxo_manage_pages_pas) + nreq * sizeof (uint64_t); 1292 1293 bzero(&in, sizeof (in)); 1294 out = kmem_alloc(outsize, KM_SLEEP); 1295 1296 mlxcx_cmd_init(mlxp, &cmd); 1297 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_manage_pages_head, 1298 MLXCX_OP_MANAGE_PAGES, MLXCX_MANAGE_PAGES_OPMOD_RETURN_PAGES); 1299 in.mlxi_manage_pages_func = MLXCX_FUNCTION_SELF; 1300 in.mlxi_manage_pages_npages = to_be32(nreq); 1301 1302 if ((ret = mlxcx_cmd_send(mlxp, &cmd, &in, insize, out, outsize))) { 1303 mlxcx_cmd_wait(&cmd); 1304 1305 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1306 if (ret) { 1307 *nret = from_be32(out->mlxo_manage_pages_npages); 1308 for (i = 0; i < *nret; i++) { 1309 pas[i] = 1310 from_be64(out->mlxo_manage_pages_pas[i]); 1311 } 1312 } 1313 } 1314 1315 mlxcx_cmd_fini(mlxp, &cmd); 1316 1317 kmem_free(out, outsize); 1318 return (ret); 1319 } 1320 1321 boolean_t 1322 mlxcx_cmd_query_hca_cap(mlxcx_t *mlxp, mlxcx_hca_cap_type_t type, 1323 mlxcx_hca_cap_mode_t mode, mlxcx_hca_cap_t *capp) 1324 { 1325 mlxcx_cmd_t cmd; 1326 mlxcx_cmd_query_hca_cap_in_t in; 1327 mlxcx_cmd_query_hca_cap_out_t *out; 1328 boolean_t ret; 1329 uint16_t opmode; 1330 1331 bzero(&in, sizeof (in)); 1332 out = kmem_zalloc(sizeof (mlxcx_cmd_query_hca_cap_out_t), KM_SLEEP); 1333 mlxcx_cmd_init(mlxp, &cmd); 1334 1335 opmode = type << 1 | mode; 1336 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_hca_cap_head, 1337 MLXCX_OP_QUERY_HCA_CAP, opmode); 1338 1339 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), out, sizeof (*out))) { 1340 mlxcx_cmd_fini(mlxp, &cmd); 1341 kmem_free(out, sizeof (mlxcx_cmd_query_hca_cap_out_t)); 1342 return (B_FALSE); 1343 } 1344 mlxcx_cmd_wait(&cmd); 1345 1346 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1347 if (ret) { 1348 capp->mhc_mode = mode; 1349 capp->mhc_type = type; 1350 ASSERT3U(sizeof (out->mlxo_query_hca_cap_data), ==, 1351 sizeof (capp->mhc_bulk)); 1352 bcopy(out->mlxo_query_hca_cap_data, capp->mhc_bulk, 1353 sizeof (capp->mhc_bulk)); 1354 } 1355 mlxcx_cmd_fini(mlxp, &cmd); 1356 1357 kmem_free(out, sizeof (mlxcx_cmd_query_hca_cap_out_t)); 1358 return (B_TRUE); 1359 } 1360 1361 boolean_t 1362 mlxcx_cmd_init_hca(mlxcx_t *mlxp) 1363 { 1364 mlxcx_cmd_t cmd; 1365 mlxcx_cmd_init_hca_in_t in; 1366 mlxcx_cmd_init_hca_out_t out; 1367 boolean_t ret; 1368 1369 bzero(&in, sizeof (in)); 1370 bzero(&out, sizeof (out)); 1371 1372 mlxcx_cmd_init(mlxp, &cmd); 1373 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_init_hca_head, 1374 MLXCX_OP_INIT_HCA, 0); 1375 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1376 mlxcx_cmd_fini(mlxp, &cmd); 1377 return (B_FALSE); 1378 } 1379 mlxcx_cmd_wait(&cmd); 1380 1381 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1382 mlxcx_cmd_fini(mlxp, &cmd); 1383 return (ret); 1384 } 1385 1386 boolean_t 1387 mlxcx_cmd_set_driver_version(mlxcx_t *mlxp, const char *version) 1388 { 1389 mlxcx_cmd_t cmd; 1390 mlxcx_cmd_set_driver_version_in_t in; 1391 mlxcx_cmd_set_driver_version_out_t out; 1392 boolean_t ret; 1393 1394 bzero(&in, sizeof (in)); 1395 bzero(&out, sizeof (out)); 1396 1397 mlxcx_cmd_init(mlxp, &cmd); 1398 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_driver_version_head, 1399 MLXCX_OP_SET_DRIVER_VERSION, 0); 1400 VERIFY3U(strlcpy(in.mlxi_set_driver_version_version, version, 1401 sizeof (in.mlxi_set_driver_version_version)), <=, 1402 sizeof (in.mlxi_set_driver_version_version)); 1403 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1404 mlxcx_cmd_fini(mlxp, &cmd); 1405 return (B_FALSE); 1406 } 1407 mlxcx_cmd_wait(&cmd); 1408 1409 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1410 mlxcx_cmd_fini(mlxp, &cmd); 1411 return (ret); 1412 } 1413 1414 boolean_t 1415 mlxcx_cmd_alloc_uar(mlxcx_t *mlxp, mlxcx_uar_t *mlup) 1416 { 1417 mlxcx_cmd_t cmd; 1418 mlxcx_cmd_alloc_uar_in_t in; 1419 mlxcx_cmd_alloc_uar_out_t out; 1420 boolean_t ret; 1421 size_t i; 1422 1423 bzero(&in, sizeof (in)); 1424 bzero(&out, sizeof (out)); 1425 1426 mlxcx_cmd_init(mlxp, &cmd); 1427 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_uar_head, 1428 MLXCX_OP_ALLOC_UAR, 0); 1429 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1430 mlxcx_cmd_fini(mlxp, &cmd); 1431 return (B_FALSE); 1432 } 1433 mlxcx_cmd_wait(&cmd); 1434 1435 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1436 if (ret) { 1437 mlup->mlu_allocated = B_TRUE; 1438 mlup->mlu_num = from_be24(out.mlxo_alloc_uar_uar); 1439 VERIFY3U(mlup->mlu_num, >, 0); 1440 mlup->mlu_base = mlup->mlu_num * MLXCX_HW_PAGE_SIZE; 1441 1442 for (i = 0; i < MLXCX_BF_PER_UAR; ++i) { 1443 mlup->mlu_bf[i].mbf_even = mlup->mlu_base + 1444 MLXCX_BF_BASE + MLXCX_BF_SIZE * 2 * i; 1445 mlup->mlu_bf[i].mbf_odd = mlup->mlu_bf[i].mbf_even + 1446 MLXCX_BF_SIZE; 1447 } 1448 } 1449 mlxcx_cmd_fini(mlxp, &cmd); 1450 return (ret); 1451 } 1452 1453 boolean_t 1454 mlxcx_cmd_dealloc_uar(mlxcx_t *mlxp, mlxcx_uar_t *mlup) 1455 { 1456 mlxcx_cmd_t cmd; 1457 mlxcx_cmd_dealloc_uar_in_t in; 1458 mlxcx_cmd_dealloc_uar_out_t out; 1459 boolean_t ret; 1460 1461 bzero(&in, sizeof (in)); 1462 bzero(&out, sizeof (out)); 1463 1464 mlxcx_cmd_init(mlxp, &cmd); 1465 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_uar_head, 1466 MLXCX_OP_DEALLOC_UAR, 0); 1467 VERIFY(mlup->mlu_allocated); 1468 in.mlxi_dealloc_uar_uar = to_be24(mlup->mlu_num); 1469 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1470 mlxcx_cmd_fini(mlxp, &cmd); 1471 return (B_FALSE); 1472 } 1473 mlxcx_cmd_wait(&cmd); 1474 1475 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1476 if (ret) { 1477 mlup->mlu_allocated = B_FALSE; 1478 mlup->mlu_num = 0; 1479 } 1480 mlxcx_cmd_fini(mlxp, &cmd); 1481 return (ret); 1482 } 1483 1484 boolean_t 1485 mlxcx_cmd_alloc_pd(mlxcx_t *mlxp, mlxcx_pd_t *mlpd) 1486 { 1487 mlxcx_cmd_t cmd; 1488 mlxcx_cmd_alloc_pd_in_t in; 1489 mlxcx_cmd_alloc_pd_out_t out; 1490 boolean_t ret; 1491 1492 bzero(&in, sizeof (in)); 1493 bzero(&out, sizeof (out)); 1494 1495 mlxcx_cmd_init(mlxp, &cmd); 1496 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_pd_head, 1497 MLXCX_OP_ALLOC_PD, 0); 1498 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1499 mlxcx_cmd_fini(mlxp, &cmd); 1500 return (B_FALSE); 1501 } 1502 mlxcx_cmd_wait(&cmd); 1503 1504 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1505 if (ret) { 1506 mlpd->mlpd_allocated = B_TRUE; 1507 mlpd->mlpd_num = from_be24(out.mlxo_alloc_pd_pdn); 1508 } 1509 mlxcx_cmd_fini(mlxp, &cmd); 1510 return (ret); 1511 } 1512 1513 boolean_t 1514 mlxcx_cmd_dealloc_pd(mlxcx_t *mlxp, mlxcx_pd_t *mlpd) 1515 { 1516 mlxcx_cmd_t cmd; 1517 mlxcx_cmd_dealloc_pd_in_t in; 1518 mlxcx_cmd_dealloc_pd_out_t out; 1519 boolean_t ret; 1520 1521 bzero(&in, sizeof (in)); 1522 bzero(&out, sizeof (out)); 1523 1524 mlxcx_cmd_init(mlxp, &cmd); 1525 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_pd_head, 1526 MLXCX_OP_DEALLOC_PD, 0); 1527 VERIFY(mlpd->mlpd_allocated); 1528 in.mlxi_dealloc_pd_pdn = to_be24(mlpd->mlpd_num); 1529 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1530 mlxcx_cmd_fini(mlxp, &cmd); 1531 return (B_FALSE); 1532 } 1533 mlxcx_cmd_wait(&cmd); 1534 1535 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1536 if (ret) { 1537 mlpd->mlpd_allocated = B_FALSE; 1538 mlpd->mlpd_num = 0; 1539 } 1540 mlxcx_cmd_fini(mlxp, &cmd); 1541 return (ret); 1542 } 1543 1544 boolean_t 1545 mlxcx_cmd_alloc_tdom(mlxcx_t *mlxp, mlxcx_tdom_t *mltd) 1546 { 1547 mlxcx_cmd_t cmd; 1548 mlxcx_cmd_alloc_tdom_in_t in; 1549 mlxcx_cmd_alloc_tdom_out_t out; 1550 boolean_t ret; 1551 1552 bzero(&in, sizeof (in)); 1553 bzero(&out, sizeof (out)); 1554 1555 mlxcx_cmd_init(mlxp, &cmd); 1556 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_tdom_head, 1557 MLXCX_OP_ALLOC_TRANSPORT_DOMAIN, 0); 1558 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1559 mlxcx_cmd_fini(mlxp, &cmd); 1560 return (B_FALSE); 1561 } 1562 mlxcx_cmd_wait(&cmd); 1563 1564 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1565 if (ret) { 1566 mltd->mltd_allocated = B_TRUE; 1567 mltd->mltd_num = from_be24(out.mlxo_alloc_tdom_tdomn); 1568 } 1569 mlxcx_cmd_fini(mlxp, &cmd); 1570 return (ret); 1571 } 1572 1573 boolean_t 1574 mlxcx_cmd_dealloc_tdom(mlxcx_t *mlxp, mlxcx_tdom_t *mltd) 1575 { 1576 mlxcx_cmd_t cmd; 1577 mlxcx_cmd_dealloc_tdom_in_t in; 1578 mlxcx_cmd_dealloc_tdom_out_t out; 1579 boolean_t ret; 1580 1581 bzero(&in, sizeof (in)); 1582 bzero(&out, sizeof (out)); 1583 1584 mlxcx_cmd_init(mlxp, &cmd); 1585 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_tdom_head, 1586 MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN, 0); 1587 VERIFY(mltd->mltd_allocated); 1588 in.mlxi_dealloc_tdom_tdomn = to_be24(mltd->mltd_num); 1589 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1590 mlxcx_cmd_fini(mlxp, &cmd); 1591 return (B_FALSE); 1592 } 1593 mlxcx_cmd_wait(&cmd); 1594 1595 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1596 if (ret) { 1597 mltd->mltd_allocated = B_FALSE; 1598 mltd->mltd_num = 0; 1599 } 1600 mlxcx_cmd_fini(mlxp, &cmd); 1601 return (ret); 1602 } 1603 1604 boolean_t 1605 mlxcx_cmd_teardown_hca(mlxcx_t *mlxp) 1606 { 1607 mlxcx_cmd_t cmd; 1608 mlxcx_cmd_teardown_hca_in_t in; 1609 mlxcx_cmd_teardown_hca_out_t out; 1610 boolean_t ret; 1611 1612 bzero(&in, sizeof (in)); 1613 bzero(&out, sizeof (out)); 1614 1615 mlxcx_cmd_init(mlxp, &cmd); 1616 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_teardown_hca_head, 1617 MLXCX_OP_TEARDOWN_HCA, 0); 1618 in.mlxi_teardown_hca_profile = to_be16(MLXCX_TEARDOWN_HCA_GRACEFUL); 1619 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1620 mlxcx_cmd_fini(mlxp, &cmd); 1621 return (B_FALSE); 1622 } 1623 mlxcx_cmd_wait(&cmd); 1624 1625 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1626 mlxcx_cmd_fini(mlxp, &cmd); 1627 return (ret); 1628 } 1629 1630 boolean_t 1631 mlxcx_cmd_query_nic_vport_ctx(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1632 { 1633 mlxcx_cmd_t cmd; 1634 mlxcx_cmd_query_nic_vport_ctx_in_t in; 1635 mlxcx_cmd_query_nic_vport_ctx_out_t out; 1636 boolean_t ret; 1637 const mlxcx_nic_vport_ctx_t *ctx; 1638 1639 bzero(&in, sizeof (in)); 1640 bzero(&out, sizeof (out)); 1641 1642 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1643 mlxcx_cmd_init(mlxp, &cmd); 1644 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_nic_vport_ctx_head, 1645 MLXCX_OP_QUERY_NIC_VPORT_CONTEXT, MLXCX_VPORT_TYPE_VNIC); 1646 1647 in.mlxi_query_nic_vport_ctx_vport_number = to_be16(mlp->mlp_num); 1648 1649 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1650 mlxcx_cmd_fini(mlxp, &cmd); 1651 return (B_FALSE); 1652 } 1653 mlxcx_cmd_wait(&cmd); 1654 1655 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1656 if (ret) { 1657 ctx = &out.mlxo_query_nic_vport_ctx_context; 1658 mlp->mlp_guid = from_be64(ctx->mlnvc_port_guid); 1659 mlp->mlp_mtu = from_be16(ctx->mlnvc_mtu); 1660 bcopy(ctx->mlnvc_permanent_address, mlp->mlp_mac_address, 1661 sizeof (mlp->mlp_mac_address)); 1662 mlp->mlp_wqe_min_inline = get_bits64(ctx->mlnvc_flags, 1663 MLXCX_VPORT_CTX_MIN_WQE_INLINE); 1664 } 1665 mlxcx_cmd_fini(mlxp, &cmd); 1666 return (ret); 1667 } 1668 1669 static const char * 1670 mlxcx_reg_name(mlxcx_register_id_t rid) 1671 { 1672 switch (rid) { 1673 case MLXCX_REG_PMTU: 1674 return ("PMTU"); 1675 case MLXCX_REG_PAOS: 1676 return ("PAOS"); 1677 case MLXCX_REG_PTYS: 1678 return ("PTYS"); 1679 case MLXCX_REG_MSGI: 1680 return ("MSGI"); 1681 case MLXCX_REG_PMAOS: 1682 return ("PMAOS"); 1683 case MLXCX_REG_MLCR: 1684 return ("MLCR"); 1685 case MLXCX_REG_MCIA: 1686 return ("MCIA"); 1687 case MLXCX_REG_PPCNT: 1688 return ("PPCNT"); 1689 case MLXCX_REG_PPLM: 1690 return ("PPLM"); 1691 default: 1692 return ("???"); 1693 } 1694 } 1695 1696 boolean_t 1697 mlxcx_cmd_access_register(mlxcx_t *mlxp, mlxcx_cmd_reg_opmod_t opmod, 1698 mlxcx_register_id_t rid, mlxcx_register_data_t *data) 1699 { 1700 mlxcx_cmd_t cmd; 1701 mlxcx_cmd_access_register_in_t in; 1702 mlxcx_cmd_access_register_out_t out; 1703 boolean_t ret; 1704 size_t dsize, insize, outsize; 1705 1706 bzero(&in, sizeof (in)); 1707 bzero(&out, sizeof (out)); 1708 1709 mlxcx_cmd_init(mlxp, &cmd); 1710 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_access_register_head, 1711 MLXCX_OP_ACCESS_REG, opmod); 1712 1713 in.mlxi_access_register_register_id = to_be16(rid); 1714 1715 switch (rid) { 1716 case MLXCX_REG_PMTU: 1717 dsize = sizeof (mlxcx_reg_pmtu_t); 1718 break; 1719 case MLXCX_REG_PAOS: 1720 dsize = sizeof (mlxcx_reg_paos_t); 1721 break; 1722 case MLXCX_REG_PTYS: 1723 dsize = sizeof (mlxcx_reg_ptys_t); 1724 break; 1725 case MLXCX_REG_MLCR: 1726 dsize = sizeof (mlxcx_reg_mlcr_t); 1727 break; 1728 case MLXCX_REG_PMAOS: 1729 dsize = sizeof (mlxcx_reg_pmaos_t); 1730 break; 1731 case MLXCX_REG_MCIA: 1732 dsize = sizeof (mlxcx_reg_mcia_t); 1733 break; 1734 case MLXCX_REG_PPCNT: 1735 dsize = sizeof (mlxcx_reg_ppcnt_t); 1736 break; 1737 case MLXCX_REG_PPLM: 1738 dsize = sizeof (mlxcx_reg_pplm_t); 1739 break; 1740 default: 1741 dsize = 0; 1742 VERIFY(0); 1743 return (B_FALSE); 1744 } 1745 insize = dsize + offsetof(mlxcx_cmd_access_register_in_t, 1746 mlxi_access_register_data); 1747 outsize = dsize + offsetof(mlxcx_cmd_access_register_out_t, 1748 mlxo_access_register_data); 1749 1750 bcopy(data, &in.mlxi_access_register_data, dsize); 1751 1752 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, outsize)) { 1753 mlxcx_cmd_fini(mlxp, &cmd); 1754 return (B_FALSE); 1755 } 1756 mlxcx_cmd_wait(&cmd); 1757 1758 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1759 if (ret) { 1760 bcopy(&out.mlxo_access_register_data, data, dsize); 1761 } else { 1762 mlxcx_warn(mlxp, "failed OP_ACCESS_REG was for register " 1763 "%04x (%s)", rid, mlxcx_reg_name(rid)); 1764 } 1765 mlxcx_cmd_fini(mlxp, &cmd); 1766 return (ret); 1767 } 1768 1769 boolean_t 1770 mlxcx_cmd_query_port_mtu(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1771 { 1772 mlxcx_register_data_t data; 1773 boolean_t ret; 1774 1775 /* 1776 * Since we modify the port here we require that the caller is holding 1777 * the port mutex. 1778 */ 1779 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1780 bzero(&data, sizeof (data)); 1781 data.mlrd_pmtu.mlrd_pmtu_local_port = mlp->mlp_num + 1; 1782 1783 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1784 MLXCX_REG_PMTU, &data); 1785 1786 if (ret) { 1787 mlp->mlp_mtu = from_be16(data.mlrd_pmtu.mlrd_pmtu_admin_mtu); 1788 mlp->mlp_max_mtu = from_be16(data.mlrd_pmtu.mlrd_pmtu_max_mtu); 1789 } 1790 1791 return (ret); 1792 } 1793 1794 boolean_t 1795 mlxcx_cmd_query_module_status(mlxcx_t *mlxp, uint_t id, 1796 mlxcx_module_status_t *pstatus, mlxcx_module_error_type_t *perr) 1797 { 1798 mlxcx_register_data_t data; 1799 boolean_t ret; 1800 1801 bzero(&data, sizeof (data)); 1802 ASSERT3U(id, <, 0xff); 1803 data.mlrd_pmaos.mlrd_pmaos_module = (uint8_t)id; 1804 1805 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1806 MLXCX_REG_PMAOS, &data); 1807 1808 if (ret) { 1809 if (pstatus != NULL) 1810 *pstatus = data.mlrd_pmaos.mlrd_pmaos_oper_status; 1811 if (perr != NULL) 1812 *perr = data.mlrd_pmaos.mlrd_pmaos_error_type; 1813 } 1814 1815 return (ret); 1816 } 1817 1818 boolean_t 1819 mlxcx_cmd_set_port_mtu(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1820 { 1821 mlxcx_register_data_t data; 1822 boolean_t ret; 1823 1824 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1825 bzero(&data, sizeof (data)); 1826 data.mlrd_pmtu.mlrd_pmtu_local_port = mlp->mlp_num + 1; 1827 data.mlrd_pmtu.mlrd_pmtu_admin_mtu = to_be16(mlp->mlp_mtu); 1828 1829 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1830 MLXCX_REG_PMTU, &data); 1831 1832 return (ret); 1833 } 1834 1835 boolean_t 1836 mlxcx_cmd_set_port_led(mlxcx_t *mlxp, mlxcx_port_t *mlp, uint16_t sec) 1837 { 1838 mlxcx_register_data_t data; 1839 boolean_t ret; 1840 1841 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1842 bzero(&data, sizeof (data)); 1843 data.mlrd_mlcr.mlrd_mlcr_local_port = mlp->mlp_num + 1; 1844 set_bits8(&data.mlrd_mlcr.mlrd_mlcr_flags, MLXCX_MLCR_LED_TYPE, 1845 MLXCX_LED_TYPE_PORT); 1846 data.mlrd_mlcr.mlrd_mlcr_beacon_duration = to_be16(sec); 1847 1848 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1849 MLXCX_REG_MLCR, &data); 1850 1851 return (ret); 1852 } 1853 1854 boolean_t 1855 mlxcx_cmd_query_port_status(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1856 { 1857 mlxcx_register_data_t data; 1858 boolean_t ret; 1859 1860 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1861 bzero(&data, sizeof (data)); 1862 data.mlrd_paos.mlrd_paos_local_port = mlp->mlp_num + 1; 1863 1864 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1865 MLXCX_REG_PAOS, &data); 1866 1867 if (ret) { 1868 mlp->mlp_admin_status = data.mlrd_paos.mlrd_paos_admin_status; 1869 mlp->mlp_oper_status = data.mlrd_paos.mlrd_paos_oper_status; 1870 } 1871 1872 return (ret); 1873 } 1874 1875 boolean_t 1876 mlxcx_cmd_modify_port_status(mlxcx_t *mlxp, mlxcx_port_t *mlp, 1877 mlxcx_port_status_t status) 1878 { 1879 mlxcx_register_data_t data; 1880 boolean_t ret; 1881 1882 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1883 bzero(&data, sizeof (data)); 1884 data.mlrd_paos.mlrd_paos_local_port = mlp->mlp_num + 1; 1885 data.mlrd_paos.mlrd_paos_admin_status = status; 1886 set_bit32(&data.mlrd_paos.mlrd_paos_flags, MLXCX_PAOS_ADMIN_ST_EN); 1887 1888 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1889 MLXCX_REG_PAOS, &data); 1890 1891 return (ret); 1892 } 1893 1894 boolean_t 1895 mlxcx_cmd_query_port_speed(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1896 { 1897 mlxcx_register_data_t data; 1898 boolean_t ret; 1899 1900 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1901 bzero(&data, sizeof (data)); 1902 data.mlrd_ptys.mlrd_ptys_local_port = mlp->mlp_num + 1; 1903 set_bit8(&data.mlrd_ptys.mlrd_ptys_proto_mask, 1904 MLXCX_PTYS_PROTO_MASK_ETH); 1905 1906 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1907 MLXCX_REG_PTYS, &data); 1908 1909 if (ret) { 1910 if (get_bit8(data.mlrd_ptys.mlrd_ptys_autoneg_flags, 1911 MLXCX_AUTONEG_DISABLE)) { 1912 mlp->mlp_autoneg = B_FALSE; 1913 } else { 1914 mlp->mlp_autoneg = B_TRUE; 1915 } 1916 mlp->mlp_max_proto = 1917 from_bits32(data.mlrd_ptys.mlrd_ptys_proto_cap); 1918 mlp->mlp_admin_proto = 1919 from_bits32(data.mlrd_ptys.mlrd_ptys_proto_admin); 1920 mlp->mlp_oper_proto = 1921 from_bits32(data.mlrd_ptys.mlrd_ptys_proto_oper); 1922 } 1923 1924 return (ret); 1925 } 1926 1927 boolean_t 1928 mlxcx_cmd_query_port_fec(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1929 { 1930 mlxcx_register_data_t data; 1931 boolean_t ret; 1932 1933 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1934 bzero(&data, sizeof (data)); 1935 data.mlrd_pplm.mlrd_pplm_local_port = mlp->mlp_num + 1; 1936 1937 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1938 MLXCX_REG_PPLM, &data); 1939 1940 if (ret) { 1941 mlp->mlp_fec_active = 1942 from_be24(data.mlrd_pplm.mlrd_pplm_fec_mode_active); 1943 } 1944 1945 return (ret); 1946 } 1947 1948 boolean_t 1949 mlxcx_cmd_modify_port_fec(mlxcx_t *mlxp, mlxcx_port_t *mlp, 1950 mlxcx_pplm_fec_caps_t fec) 1951 { 1952 mlxcx_register_data_t data_in, data_out; 1953 mlxcx_pplm_fec_caps_t caps; 1954 mlxcx_reg_pplm_t *pplm_in, *pplm_out; 1955 boolean_t ret; 1956 1957 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1958 bzero(&data_in, sizeof (data_in)); 1959 pplm_in = &data_in.mlrd_pplm; 1960 pplm_in->mlrd_pplm_local_port = mlp->mlp_num + 1; 1961 1962 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1963 MLXCX_REG_PPLM, &data_in); 1964 1965 if (!ret) 1966 return (B_FALSE); 1967 1968 bzero(&data_out, sizeof (data_out)); 1969 pplm_out = &data_out.mlrd_pplm; 1970 pplm_out->mlrd_pplm_local_port = mlp->mlp_num + 1; 1971 1972 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1973 MLXCX_PPLM_CAP_56G); 1974 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1975 MLXCX_PPLM_CAP_56G, fec & caps); 1976 1977 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1978 MLXCX_PPLM_CAP_100G); 1979 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1980 MLXCX_PPLM_CAP_100G, fec & caps); 1981 1982 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1983 MLXCX_PPLM_CAP_50G); 1984 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1985 MLXCX_PPLM_CAP_50G, fec & caps); 1986 1987 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1988 MLXCX_PPLM_CAP_25G); 1989 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1990 MLXCX_PPLM_CAP_25G, fec & caps); 1991 1992 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1993 MLXCX_PPLM_CAP_10_40G); 1994 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1995 MLXCX_PPLM_CAP_10_40G, fec & caps); 1996 1997 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1998 MLXCX_REG_PPLM, &data_out); 1999 2000 return (ret); 2001 } 2002 2003 boolean_t 2004 mlxcx_cmd_modify_nic_vport_ctx(mlxcx_t *mlxp, mlxcx_port_t *mlp, 2005 mlxcx_modify_nic_vport_ctx_fields_t fields) 2006 { 2007 mlxcx_cmd_t cmd; 2008 mlxcx_cmd_modify_nic_vport_ctx_in_t in; 2009 mlxcx_cmd_modify_nic_vport_ctx_out_t out; 2010 boolean_t ret; 2011 mlxcx_nic_vport_ctx_t *ctx; 2012 2013 ASSERT(mutex_owned(&mlp->mlp_mtx)); 2014 bzero(&in, sizeof (in)); 2015 bzero(&out, sizeof (out)); 2016 2017 mlxcx_cmd_init(mlxp, &cmd); 2018 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_nic_vport_ctx_head, 2019 MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT, MLXCX_VPORT_TYPE_VNIC); 2020 2021 in.mlxi_modify_nic_vport_ctx_vport_number = to_be16(mlp->mlp_num); 2022 in.mlxi_modify_nic_vport_ctx_field_select = to_be32(fields); 2023 2024 ctx = &in.mlxi_modify_nic_vport_ctx_context; 2025 if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_PROMISC) { 2026 set_bit16(&ctx->mlnvc_promisc_list_type, 2027 MLXCX_VPORT_PROMISC_ALL); 2028 } 2029 if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_MTU) { 2030 ctx->mlnvc_mtu = to_be16(mlp->mlp_mtu); 2031 } 2032 2033 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2034 mlxcx_cmd_fini(mlxp, &cmd); 2035 return (B_FALSE); 2036 } 2037 mlxcx_cmd_wait(&cmd); 2038 2039 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2040 if (ret) { 2041 if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_PROMISC) { 2042 mlp->mlp_flags |= MLXCX_PORT_VPORT_PROMISC; 2043 } 2044 } 2045 mlxcx_cmd_fini(mlxp, &cmd); 2046 return (ret); 2047 } 2048 2049 boolean_t 2050 mlxcx_cmd_create_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq) 2051 { 2052 mlxcx_cmd_t cmd; 2053 mlxcx_cmd_create_eq_in_t in; 2054 mlxcx_cmd_create_eq_out_t out; 2055 boolean_t ret; 2056 mlxcx_eventq_ctx_t *ctx; 2057 size_t rem, insize; 2058 const ddi_dma_cookie_t *c; 2059 uint64_t pa, npages; 2060 2061 bzero(&in, sizeof (in)); 2062 bzero(&out, sizeof (out)); 2063 2064 ASSERT(mutex_owned(&mleq->mleq_mtx)); 2065 VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC); 2066 VERIFY0(mleq->mleq_state & MLXCX_EQ_CREATED); 2067 2068 mlxcx_cmd_init(mlxp, &cmd); 2069 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_eq_head, 2070 MLXCX_OP_CREATE_EQ, 0); 2071 2072 ctx = &in.mlxi_create_eq_context; 2073 ctx->mleqc_uar_page = to_be24(mleq->mleq_uar->mlu_num); 2074 ctx->mleqc_log_eq_size = mleq->mleq_entshift; 2075 ctx->mleqc_intr = mleq->mleq_intr_index; 2076 2077 in.mlxi_create_eq_event_bitmask = to_be64(mleq->mleq_events); 2078 2079 npages = 0; 2080 c = NULL; 2081 while ((c = mlxcx_dma_cookie_iter(&mleq->mleq_dma, c)) != NULL) { 2082 pa = c->dmac_laddress; 2083 rem = c->dmac_size; 2084 while (rem > 0) { 2085 ASSERT3U(pa & 0xfff, ==, 0); 2086 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 2087 in.mlxi_create_eq_pas[npages++] = to_be64(pa); 2088 rem -= MLXCX_HW_PAGE_SIZE; 2089 pa += MLXCX_HW_PAGE_SIZE; 2090 } 2091 } 2092 ASSERT3U(npages, <=, MLXCX_CREATE_QUEUE_MAX_PAGES); 2093 2094 insize = offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_pas) + 2095 sizeof (uint64_t) * npages; 2096 2097 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 2098 mlxcx_cmd_fini(mlxp, &cmd); 2099 return (B_FALSE); 2100 } 2101 mlxcx_cmd_wait(&cmd); 2102 2103 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2104 if (ret) { 2105 mleq->mleq_state |= MLXCX_EQ_CREATED; 2106 mleq->mleq_num = out.mlxo_create_eq_eqn; 2107 } 2108 mlxcx_cmd_fini(mlxp, &cmd); 2109 return (ret); 2110 } 2111 2112 boolean_t 2113 mlxcx_cmd_query_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq, 2114 mlxcx_eventq_ctx_t *ctxp) 2115 { 2116 mlxcx_cmd_t cmd; 2117 mlxcx_cmd_query_eq_in_t in; 2118 mlxcx_cmd_query_eq_out_t out; 2119 boolean_t ret; 2120 2121 bzero(&in, sizeof (in)); 2122 bzero(&out, sizeof (out)); 2123 2124 VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC); 2125 VERIFY(mleq->mleq_state & MLXCX_EQ_CREATED); 2126 2127 mlxcx_cmd_init(mlxp, &cmd); 2128 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_eq_head, 2129 MLXCX_OP_QUERY_EQ, 0); 2130 2131 in.mlxi_query_eq_eqn = mleq->mleq_num; 2132 2133 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2134 mlxcx_cmd_fini(mlxp, &cmd); 2135 return (B_FALSE); 2136 } 2137 mlxcx_cmd_wait(&cmd); 2138 2139 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2140 if (ret) { 2141 bcopy(&out.mlxo_query_eq_context, ctxp, 2142 sizeof (mlxcx_eventq_ctx_t)); 2143 } 2144 mlxcx_cmd_fini(mlxp, &cmd); 2145 return (ret); 2146 } 2147 2148 boolean_t 2149 mlxcx_cmd_destroy_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq) 2150 { 2151 mlxcx_cmd_t cmd; 2152 mlxcx_cmd_destroy_eq_in_t in; 2153 mlxcx_cmd_destroy_eq_out_t out; 2154 boolean_t ret; 2155 2156 bzero(&in, sizeof (in)); 2157 bzero(&out, sizeof (out)); 2158 2159 ASSERT(mutex_owned(&mleq->mleq_mtx)); 2160 VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC); 2161 VERIFY(mleq->mleq_state & MLXCX_EQ_CREATED); 2162 2163 mlxcx_cmd_init(mlxp, &cmd); 2164 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_eq_head, 2165 MLXCX_OP_DESTROY_EQ, 0); 2166 2167 in.mlxi_destroy_eq_eqn = mleq->mleq_num; 2168 2169 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2170 mlxcx_cmd_fini(mlxp, &cmd); 2171 return (B_FALSE); 2172 } 2173 mlxcx_cmd_wait(&cmd); 2174 2175 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2176 if (ret) { 2177 mleq->mleq_state |= MLXCX_EQ_DESTROYED; 2178 } 2179 mlxcx_cmd_fini(mlxp, &cmd); 2180 return (ret); 2181 } 2182 2183 boolean_t 2184 mlxcx_cmd_query_special_ctxs(mlxcx_t *mlxp) 2185 { 2186 mlxcx_cmd_t cmd; 2187 mlxcx_cmd_query_special_ctxs_in_t in; 2188 mlxcx_cmd_query_special_ctxs_out_t out; 2189 boolean_t ret; 2190 2191 bzero(&in, sizeof (in)); 2192 bzero(&out, sizeof (out)); 2193 2194 mlxcx_cmd_init(mlxp, &cmd); 2195 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_special_ctxs_head, 2196 MLXCX_OP_QUERY_SPECIAL_CONTEXTS, 0); 2197 2198 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2199 mlxcx_cmd_fini(mlxp, &cmd); 2200 return (B_FALSE); 2201 } 2202 mlxcx_cmd_wait(&cmd); 2203 2204 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2205 if (ret) { 2206 mlxp->mlx_rsvd_lkey = from_be32( 2207 out.mlxo_query_special_ctxs_resd_lkey); 2208 } 2209 mlxcx_cmd_fini(mlxp, &cmd); 2210 return (ret); 2211 } 2212 2213 boolean_t 2214 mlxcx_cmd_create_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq) 2215 { 2216 mlxcx_cmd_t cmd; 2217 mlxcx_cmd_create_cq_in_t in; 2218 mlxcx_cmd_create_cq_out_t out; 2219 boolean_t ret; 2220 mlxcx_completionq_ctx_t *ctx; 2221 size_t rem, insize; 2222 const ddi_dma_cookie_t *c; 2223 uint64_t pa, npages; 2224 2225 bzero(&in, sizeof (in)); 2226 bzero(&out, sizeof (out)); 2227 2228 ASSERT(mutex_owned(&mlcq->mlcq_mtx)); 2229 VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC); 2230 VERIFY0(mlcq->mlcq_state & MLXCX_CQ_CREATED); 2231 2232 mlxcx_cmd_init(mlxp, &cmd); 2233 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_cq_head, 2234 MLXCX_OP_CREATE_CQ, 0); 2235 2236 ctx = &in.mlxi_create_cq_context; 2237 ctx->mlcqc_uar_page = to_be24(mlcq->mlcq_uar->mlu_num); 2238 ctx->mlcqc_log_cq_size = mlcq->mlcq_entshift; 2239 ctx->mlcqc_eqn = mlcq->mlcq_eq->mleq_num; 2240 ctx->mlcqc_cq_period = to_be16(mlcq->mlcq_cqemod_period_usec); 2241 ctx->mlcqc_cq_max_count = to_be16(mlcq->mlcq_cqemod_count); 2242 2243 c = mlxcx_dma_cookie_one(&mlcq->mlcq_doorbell_dma); 2244 ctx->mlcqc_dbr_addr = to_be64(c->dmac_laddress); 2245 ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_completionq_doorbell_t)); 2246 2247 npages = 0; 2248 c = NULL; 2249 while ((c = mlxcx_dma_cookie_iter(&mlcq->mlcq_dma, c)) != NULL) { 2250 pa = c->dmac_laddress; 2251 rem = c->dmac_size; 2252 while (rem > 0) { 2253 ASSERT3U(pa & 0xfff, ==, 0); 2254 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 2255 in.mlxi_create_cq_pas[npages++] = to_be64(pa); 2256 rem -= MLXCX_HW_PAGE_SIZE; 2257 pa += MLXCX_HW_PAGE_SIZE; 2258 } 2259 } 2260 ASSERT3U(npages, <=, MLXCX_CREATE_QUEUE_MAX_PAGES); 2261 2262 insize = offsetof(mlxcx_cmd_create_cq_in_t, mlxi_create_cq_pas) + 2263 sizeof (uint64_t) * npages; 2264 2265 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 2266 mlxcx_cmd_fini(mlxp, &cmd); 2267 return (B_FALSE); 2268 } 2269 mlxcx_cmd_wait(&cmd); 2270 2271 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2272 if (ret) { 2273 atomic_or_uint(&mlcq->mlcq_state, MLXCX_CQ_CREATED); 2274 mlcq->mlcq_num = from_be24(out.mlxo_create_cq_cqn); 2275 } 2276 mlxcx_cmd_fini(mlxp, &cmd); 2277 return (ret); 2278 } 2279 2280 boolean_t 2281 mlxcx_cmd_query_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq, 2282 mlxcx_rq_ctx_t *ctxp) 2283 { 2284 mlxcx_cmd_t cmd; 2285 mlxcx_cmd_query_rq_in_t in; 2286 mlxcx_cmd_query_rq_out_t out; 2287 boolean_t ret; 2288 2289 bzero(&in, sizeof (in)); 2290 bzero(&out, sizeof (out)); 2291 2292 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2293 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2294 ASSERT3S(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ); 2295 2296 mlxcx_cmd_init(mlxp, &cmd); 2297 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_rq_head, 2298 MLXCX_OP_QUERY_RQ, 0); 2299 2300 in.mlxi_query_rq_rqn = to_be24(mlwq->mlwq_num); 2301 2302 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2303 mlxcx_cmd_fini(mlxp, &cmd); 2304 return (B_FALSE); 2305 } 2306 mlxcx_cmd_wait(&cmd); 2307 2308 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2309 if (ret) { 2310 bcopy(&out.mlxo_query_rq_context, ctxp, 2311 sizeof (mlxcx_rq_ctx_t)); 2312 } 2313 mlxcx_cmd_fini(mlxp, &cmd); 2314 return (ret); 2315 } 2316 2317 boolean_t 2318 mlxcx_cmd_query_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq, 2319 mlxcx_sq_ctx_t *ctxp) 2320 { 2321 mlxcx_cmd_t cmd; 2322 mlxcx_cmd_query_sq_in_t in; 2323 mlxcx_cmd_query_sq_out_t out; 2324 boolean_t ret; 2325 2326 bzero(&in, sizeof (in)); 2327 bzero(&out, sizeof (out)); 2328 2329 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2330 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2331 ASSERT3S(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_SENDQ); 2332 2333 mlxcx_cmd_init(mlxp, &cmd); 2334 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_sq_head, 2335 MLXCX_OP_QUERY_SQ, 0); 2336 2337 in.mlxi_query_sq_sqn = to_be24(mlwq->mlwq_num); 2338 2339 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2340 mlxcx_cmd_fini(mlxp, &cmd); 2341 return (B_FALSE); 2342 } 2343 mlxcx_cmd_wait(&cmd); 2344 2345 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2346 if (ret) { 2347 bcopy(&out.mlxo_query_sq_context, ctxp, 2348 sizeof (mlxcx_sq_ctx_t)); 2349 } 2350 mlxcx_cmd_fini(mlxp, &cmd); 2351 return (ret); 2352 } 2353 2354 boolean_t 2355 mlxcx_cmd_query_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq, 2356 mlxcx_completionq_ctx_t *ctxp) 2357 { 2358 mlxcx_cmd_t cmd; 2359 mlxcx_cmd_query_cq_in_t in; 2360 mlxcx_cmd_query_cq_out_t out; 2361 boolean_t ret; 2362 2363 bzero(&in, sizeof (in)); 2364 bzero(&out, sizeof (out)); 2365 2366 VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC); 2367 VERIFY(mlcq->mlcq_state & MLXCX_CQ_CREATED); 2368 2369 mlxcx_cmd_init(mlxp, &cmd); 2370 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_cq_head, 2371 MLXCX_OP_QUERY_CQ, 0); 2372 2373 in.mlxi_query_cq_cqn = to_be24(mlcq->mlcq_num); 2374 2375 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2376 mlxcx_cmd_fini(mlxp, &cmd); 2377 return (B_FALSE); 2378 } 2379 mlxcx_cmd_wait(&cmd); 2380 2381 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2382 if (ret) { 2383 bcopy(&out.mlxo_query_cq_context, ctxp, 2384 sizeof (mlxcx_completionq_ctx_t)); 2385 } 2386 mlxcx_cmd_fini(mlxp, &cmd); 2387 return (ret); 2388 } 2389 2390 boolean_t 2391 mlxcx_cmd_destroy_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq) 2392 { 2393 mlxcx_cmd_t cmd; 2394 mlxcx_cmd_destroy_cq_in_t in; 2395 mlxcx_cmd_destroy_cq_out_t out; 2396 boolean_t ret; 2397 2398 bzero(&in, sizeof (in)); 2399 bzero(&out, sizeof (out)); 2400 2401 ASSERT(mutex_owned(&mlcq->mlcq_mtx)); 2402 VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC); 2403 VERIFY(mlcq->mlcq_state & MLXCX_CQ_CREATED); 2404 2405 mlxcx_cmd_init(mlxp, &cmd); 2406 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_cq_head, 2407 MLXCX_OP_DESTROY_CQ, 0); 2408 2409 in.mlxi_destroy_cq_cqn = to_be24(mlcq->mlcq_num); 2410 2411 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2412 mlxcx_cmd_fini(mlxp, &cmd); 2413 return (B_FALSE); 2414 } 2415 mlxcx_cmd_wait(&cmd); 2416 2417 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2418 if (ret) { 2419 atomic_or_uint(&mlcq->mlcq_state, MLXCX_CQ_DESTROYED); 2420 } 2421 mlxcx_cmd_fini(mlxp, &cmd); 2422 return (ret); 2423 } 2424 2425 boolean_t 2426 mlxcx_cmd_create_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2427 { 2428 mlxcx_cmd_t cmd; 2429 mlxcx_cmd_create_rq_in_t in; 2430 mlxcx_cmd_create_rq_out_t out; 2431 boolean_t ret; 2432 mlxcx_rq_ctx_t *ctx; 2433 size_t rem, insize; 2434 const ddi_dma_cookie_t *c; 2435 uint64_t pa, npages; 2436 2437 bzero(&in, sizeof (in)); 2438 bzero(&out, sizeof (out)); 2439 2440 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2441 VERIFY3U(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ); 2442 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2443 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2444 2445 mlxcx_cmd_init(mlxp, &cmd); 2446 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_rq_head, 2447 MLXCX_OP_CREATE_RQ, 0); 2448 2449 ctx = &in.mlxi_create_rq_context; 2450 2451 set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_RLKEY); 2452 set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_FLUSH_IN_ERROR); 2453 set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_VLAN_STRIP_DISABLE); 2454 ctx->mlrqc_cqn = to_be24(mlwq->mlwq_cq->mlcq_num); 2455 2456 set_bits32(&ctx->mlrqc_wq.mlwqc_flags, MLXCX_WORKQ_CTX_TYPE, 2457 MLXCX_WORKQ_TYPE_CYCLIC); 2458 ctx->mlrqc_wq.mlwqc_pd = to_be24(mlwq->mlwq_pd->mlpd_num); 2459 ctx->mlrqc_wq.mlwqc_log_wq_sz = mlwq->mlwq_entshift; 2460 ctx->mlrqc_wq.mlwqc_log_wq_stride = MLXCX_RECVQ_STRIDE_SHIFT; 2461 2462 c = mlxcx_dma_cookie_one(&mlwq->mlwq_doorbell_dma); 2463 ctx->mlrqc_wq.mlwqc_dbr_addr = to_be64(c->dmac_laddress); 2464 ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_workq_doorbell_t)); 2465 2466 npages = 0; 2467 c = NULL; 2468 while ((c = mlxcx_dma_cookie_iter(&mlwq->mlwq_dma, c)) != NULL) { 2469 pa = c->dmac_laddress; 2470 rem = c->dmac_size; 2471 while (rem > 0) { 2472 ASSERT3U(pa & 0xfff, ==, 0); 2473 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 2474 ctx->mlrqc_wq.mlwqc_pas[npages++] = to_be64(pa); 2475 rem -= MLXCX_HW_PAGE_SIZE; 2476 pa += MLXCX_HW_PAGE_SIZE; 2477 } 2478 } 2479 ASSERT3U(npages, <=, MLXCX_WORKQ_CTX_MAX_ADDRESSES); 2480 2481 insize = offsetof(mlxcx_cmd_create_rq_in_t, mlxi_create_rq_context) + 2482 offsetof(mlxcx_rq_ctx_t, mlrqc_wq) + 2483 offsetof(mlxcx_workq_ctx_t, mlwqc_pas) + 2484 sizeof (uint64_t) * npages; 2485 2486 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 2487 mlxcx_cmd_fini(mlxp, &cmd); 2488 return (B_FALSE); 2489 } 2490 mlxcx_cmd_wait(&cmd); 2491 2492 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2493 if (ret) { 2494 mlwq->mlwq_state |= MLXCX_WQ_CREATED; 2495 mlwq->mlwq_num = from_be24(out.mlxo_create_rq_rqn); 2496 } 2497 mlxcx_cmd_fini(mlxp, &cmd); 2498 return (ret); 2499 } 2500 2501 boolean_t 2502 mlxcx_cmd_start_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2503 { 2504 mlxcx_cmd_t cmd; 2505 mlxcx_cmd_modify_rq_in_t in; 2506 mlxcx_cmd_modify_rq_out_t out; 2507 boolean_t ret; 2508 ddi_fm_error_t err; 2509 2510 bzero(&in, sizeof (in)); 2511 bzero(&out, sizeof (out)); 2512 2513 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2514 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2515 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2516 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 2517 2518 /* 2519 * Before starting the queue, we have to be sure that it is 2520 * empty and the doorbell and counters are set to 0. 2521 */ 2522 ASSERT(mutex_owned(&mlwq->mlwq_cq->mlcq_mtx)); 2523 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers)); 2524 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers_b)); 2525 2526 mlwq->mlwq_doorbell->mlwqd_recv_counter = to_be16(0); 2527 MLXCX_DMA_SYNC(mlwq->mlwq_doorbell_dma, DDI_DMA_SYNC_FORDEV); 2528 ddi_fm_dma_err_get(mlwq->mlwq_doorbell_dma.mxdb_dma_handle, &err, 2529 DDI_FME_VERSION); 2530 if (err.fme_status != DDI_FM_OK) 2531 return (B_FALSE); 2532 mlwq->mlwq_pc = 0; 2533 2534 mlxcx_cmd_init(mlxp, &cmd); 2535 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_rq_head, 2536 MLXCX_OP_MODIFY_RQ, 0); 2537 2538 in.mlxi_modify_rq_rqn = to_be24(mlwq->mlwq_num); 2539 2540 /* From state */ 2541 set_bits8(&in.mlxi_modify_rq_state, MLXCX_CMD_MODIFY_RQ_STATE, 2542 MLXCX_RQ_STATE_RST); 2543 /* To state */ 2544 set_bits32(&in.mlxi_modify_rq_context.mlrqc_flags, MLXCX_RQ_STATE, 2545 MLXCX_RQ_STATE_RDY); 2546 2547 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2548 mlxcx_cmd_fini(mlxp, &cmd); 2549 return (B_FALSE); 2550 } 2551 mlxcx_cmd_wait(&cmd); 2552 2553 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2554 if (ret) { 2555 mlwq->mlwq_state |= MLXCX_WQ_STARTED; 2556 } 2557 mlxcx_cmd_fini(mlxp, &cmd); 2558 return (ret); 2559 } 2560 2561 boolean_t 2562 mlxcx_cmd_stop_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2563 { 2564 mlxcx_cmd_t cmd; 2565 mlxcx_cmd_modify_rq_in_t in; 2566 mlxcx_cmd_modify_rq_out_t out; 2567 boolean_t ret; 2568 2569 bzero(&in, sizeof (in)); 2570 bzero(&out, sizeof (out)); 2571 2572 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2573 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2574 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2575 VERIFY(mlwq->mlwq_state & MLXCX_WQ_STARTED); 2576 2577 mlxcx_cmd_init(mlxp, &cmd); 2578 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_rq_head, 2579 MLXCX_OP_MODIFY_RQ, 0); 2580 2581 in.mlxi_modify_rq_rqn = to_be24(mlwq->mlwq_num); 2582 2583 /* From state */ 2584 set_bits8(&in.mlxi_modify_rq_state, MLXCX_CMD_MODIFY_RQ_STATE, 2585 MLXCX_RQ_STATE_RDY); 2586 /* To state */ 2587 set_bits32(&in.mlxi_modify_rq_context.mlrqc_flags, MLXCX_RQ_STATE, 2588 MLXCX_RQ_STATE_RST); 2589 2590 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2591 mlxcx_cmd_fini(mlxp, &cmd); 2592 return (B_FALSE); 2593 } 2594 mlxcx_cmd_wait(&cmd); 2595 2596 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2597 if (ret) { 2598 mlwq->mlwq_state &= ~MLXCX_WQ_STARTED; 2599 } 2600 mlxcx_cmd_fini(mlxp, &cmd); 2601 return (ret); 2602 } 2603 2604 boolean_t 2605 mlxcx_cmd_destroy_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2606 { 2607 mlxcx_cmd_t cmd; 2608 mlxcx_cmd_destroy_rq_in_t in; 2609 mlxcx_cmd_destroy_rq_out_t out; 2610 boolean_t ret; 2611 2612 bzero(&in, sizeof (in)); 2613 bzero(&out, sizeof (out)); 2614 2615 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2616 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2617 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2618 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 2619 2620 mlxcx_cmd_init(mlxp, &cmd); 2621 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_rq_head, 2622 MLXCX_OP_DESTROY_RQ, 0); 2623 2624 in.mlxi_destroy_rq_rqn = to_be24(mlwq->mlwq_num); 2625 2626 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2627 mlxcx_cmd_fini(mlxp, &cmd); 2628 return (B_FALSE); 2629 } 2630 mlxcx_cmd_wait(&cmd); 2631 2632 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2633 if (ret) { 2634 mlwq->mlwq_state |= MLXCX_WQ_DESTROYED; 2635 } 2636 mlxcx_cmd_fini(mlxp, &cmd); 2637 return (ret); 2638 } 2639 2640 boolean_t 2641 mlxcx_cmd_create_tir(mlxcx_t *mlxp, mlxcx_tir_t *mltir) 2642 { 2643 mlxcx_cmd_t cmd; 2644 mlxcx_cmd_create_tir_in_t in; 2645 mlxcx_cmd_create_tir_out_t out; 2646 mlxcx_tir_ctx_t *ctx; 2647 boolean_t ret; 2648 2649 bzero(&in, sizeof (in)); 2650 bzero(&out, sizeof (out)); 2651 2652 VERIFY0(mltir->mltir_state & MLXCX_TIR_CREATED); 2653 2654 mlxcx_cmd_init(mlxp, &cmd); 2655 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_tir_head, 2656 MLXCX_OP_CREATE_TIR, 0); 2657 2658 ctx = &in.mlxi_create_tir_context; 2659 ctx->mltirc_transport_domain = to_be24(mltir->mltir_tdom->mltd_num); 2660 set_bits8(&ctx->mltirc_disp_type, MLXCX_TIR_CTX_DISP_TYPE, 2661 mltir->mltir_type); 2662 switch (mltir->mltir_type) { 2663 case MLXCX_TIR_INDIRECT: 2664 VERIFY(mltir->mltir_rqtable != NULL); 2665 VERIFY(mltir->mltir_rqtable->mlrqt_state & MLXCX_RQT_CREATED); 2666 ctx->mltirc_indirect_table = 2667 to_be24(mltir->mltir_rqtable->mlrqt_num); 2668 set_bits8(&ctx->mltirc_hash_lb, MLXCX_TIR_RX_HASH_FN, 2669 mltir->mltir_hash_fn); 2670 bcopy(mltir->mltir_toeplitz_key, 2671 ctx->mltirc_rx_hash_toeplitz_key, 2672 sizeof (ctx->mltirc_rx_hash_toeplitz_key)); 2673 set_bits32(&ctx->mltirc_rx_hash_fields_outer, 2674 MLXCX_RX_HASH_L3_TYPE, mltir->mltir_l3_type); 2675 set_bits32(&ctx->mltirc_rx_hash_fields_outer, 2676 MLXCX_RX_HASH_L4_TYPE, mltir->mltir_l4_type); 2677 set_bits32(&ctx->mltirc_rx_hash_fields_outer, 2678 MLXCX_RX_HASH_FIELDS, mltir->mltir_hash_fields); 2679 break; 2680 case MLXCX_TIR_DIRECT: 2681 VERIFY(mltir->mltir_rq != NULL); 2682 VERIFY(mltir->mltir_rq->mlwq_state & MLXCX_WQ_CREATED); 2683 ctx->mltirc_inline_rqn = to_be24(mltir->mltir_rq->mlwq_num); 2684 break; 2685 default: 2686 VERIFY(0); 2687 } 2688 2689 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2690 mlxcx_cmd_fini(mlxp, &cmd); 2691 return (B_FALSE); 2692 } 2693 mlxcx_cmd_wait(&cmd); 2694 2695 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2696 if (ret) { 2697 mltir->mltir_state |= MLXCX_TIR_CREATED; 2698 mltir->mltir_num = from_be24(out.mlxo_create_tir_tirn); 2699 } 2700 mlxcx_cmd_fini(mlxp, &cmd); 2701 return (ret); 2702 } 2703 2704 boolean_t 2705 mlxcx_cmd_destroy_tir(mlxcx_t *mlxp, mlxcx_tir_t *mltir) 2706 { 2707 mlxcx_cmd_t cmd; 2708 mlxcx_cmd_destroy_tir_in_t in; 2709 mlxcx_cmd_destroy_tir_out_t out; 2710 boolean_t ret; 2711 2712 bzero(&in, sizeof (in)); 2713 bzero(&out, sizeof (out)); 2714 2715 VERIFY(mltir->mltir_state & MLXCX_TIR_CREATED); 2716 VERIFY0(mltir->mltir_state & MLXCX_TIR_DESTROYED); 2717 2718 mlxcx_cmd_init(mlxp, &cmd); 2719 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_tir_head, 2720 MLXCX_OP_DESTROY_TIR, 0); 2721 2722 in.mlxi_destroy_tir_tirn = to_be24(mltir->mltir_num); 2723 2724 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2725 mlxcx_cmd_fini(mlxp, &cmd); 2726 return (B_FALSE); 2727 } 2728 mlxcx_cmd_wait(&cmd); 2729 2730 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2731 if (ret) { 2732 mltir->mltir_state |= MLXCX_TIR_DESTROYED; 2733 } 2734 mlxcx_cmd_fini(mlxp, &cmd); 2735 return (ret); 2736 } 2737 2738 boolean_t 2739 mlxcx_cmd_create_tis(mlxcx_t *mlxp, mlxcx_tis_t *mltis) 2740 { 2741 mlxcx_cmd_t cmd; 2742 mlxcx_cmd_create_tis_in_t in; 2743 mlxcx_cmd_create_tis_out_t out; 2744 mlxcx_tis_ctx_t *ctx; 2745 boolean_t ret; 2746 2747 bzero(&in, sizeof (in)); 2748 bzero(&out, sizeof (out)); 2749 2750 VERIFY0(mltis->mltis_state & MLXCX_TIS_CREATED); 2751 2752 mlxcx_cmd_init(mlxp, &cmd); 2753 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_tis_head, 2754 MLXCX_OP_CREATE_TIS, 0); 2755 2756 ctx = &in.mlxi_create_tis_context; 2757 ctx->mltisc_transport_domain = to_be24(mltis->mltis_tdom->mltd_num); 2758 2759 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2760 mlxcx_cmd_fini(mlxp, &cmd); 2761 return (B_FALSE); 2762 } 2763 mlxcx_cmd_wait(&cmd); 2764 2765 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2766 if (ret) { 2767 mltis->mltis_state |= MLXCX_TIS_CREATED; 2768 mltis->mltis_num = from_be24(out.mlxo_create_tis_tisn); 2769 } 2770 mlxcx_cmd_fini(mlxp, &cmd); 2771 return (ret); 2772 } 2773 2774 boolean_t 2775 mlxcx_cmd_destroy_tis(mlxcx_t *mlxp, mlxcx_tis_t *mltis) 2776 { 2777 mlxcx_cmd_t cmd; 2778 mlxcx_cmd_destroy_tis_in_t in; 2779 mlxcx_cmd_destroy_tis_out_t out; 2780 boolean_t ret; 2781 2782 bzero(&in, sizeof (in)); 2783 bzero(&out, sizeof (out)); 2784 2785 VERIFY(mltis->mltis_state & MLXCX_TIR_CREATED); 2786 VERIFY0(mltis->mltis_state & MLXCX_TIR_DESTROYED); 2787 2788 mlxcx_cmd_init(mlxp, &cmd); 2789 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_tis_head, 2790 MLXCX_OP_DESTROY_TIS, 0); 2791 2792 in.mlxi_destroy_tis_tisn = to_be24(mltis->mltis_num); 2793 2794 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2795 mlxcx_cmd_fini(mlxp, &cmd); 2796 return (B_FALSE); 2797 } 2798 mlxcx_cmd_wait(&cmd); 2799 2800 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2801 if (ret) { 2802 mltis->mltis_state |= MLXCX_TIS_DESTROYED; 2803 } 2804 mlxcx_cmd_fini(mlxp, &cmd); 2805 return (ret); 2806 } 2807 2808 boolean_t 2809 mlxcx_cmd_create_flow_table(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft) 2810 { 2811 mlxcx_cmd_t cmd; 2812 mlxcx_cmd_create_flow_table_in_t in; 2813 mlxcx_cmd_create_flow_table_out_t out; 2814 mlxcx_flow_table_ctx_t *ctx; 2815 boolean_t ret; 2816 2817 bzero(&in, sizeof (in)); 2818 bzero(&out, sizeof (out)); 2819 2820 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2821 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2822 2823 mlxcx_cmd_init(mlxp, &cmd); 2824 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_flow_table_head, 2825 MLXCX_OP_CREATE_FLOW_TABLE, 0); 2826 2827 in.mlxi_create_flow_table_vport_number = 2828 to_be16(mlft->mlft_port->mlp_num); 2829 in.mlxi_create_flow_table_table_type = mlft->mlft_type; 2830 ctx = &in.mlxi_create_flow_table_context; 2831 ctx->mlftc_log_size = mlft->mlft_entshift; 2832 ctx->mlftc_level = mlft->mlft_level; 2833 2834 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2835 mlxcx_cmd_fini(mlxp, &cmd); 2836 return (B_FALSE); 2837 } 2838 mlxcx_cmd_wait(&cmd); 2839 2840 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2841 if (ret) { 2842 mlft->mlft_num = from_be24(out.mlxo_create_flow_table_table_id); 2843 mlft->mlft_state |= MLXCX_FLOW_TABLE_CREATED; 2844 } 2845 mlxcx_cmd_fini(mlxp, &cmd); 2846 return (ret); 2847 } 2848 2849 boolean_t 2850 mlxcx_cmd_destroy_flow_table(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft) 2851 { 2852 mlxcx_cmd_t cmd; 2853 mlxcx_cmd_destroy_flow_table_in_t in; 2854 mlxcx_cmd_destroy_flow_table_out_t out; 2855 boolean_t ret; 2856 2857 bzero(&in, sizeof (in)); 2858 bzero(&out, sizeof (out)); 2859 2860 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2861 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2862 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 2863 2864 mlxcx_cmd_init(mlxp, &cmd); 2865 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_flow_table_head, 2866 MLXCX_OP_DESTROY_FLOW_TABLE, 0); 2867 2868 in.mlxi_destroy_flow_table_vport_number = 2869 to_be16(mlft->mlft_port->mlp_num); 2870 in.mlxi_destroy_flow_table_table_type = mlft->mlft_type; 2871 in.mlxi_destroy_flow_table_table_id = to_be24(mlft->mlft_num); 2872 2873 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2874 mlxcx_cmd_fini(mlxp, &cmd); 2875 return (B_FALSE); 2876 } 2877 mlxcx_cmd_wait(&cmd); 2878 2879 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2880 if (ret) { 2881 mlft->mlft_state |= MLXCX_FLOW_TABLE_DESTROYED; 2882 } 2883 mlxcx_cmd_fini(mlxp, &cmd); 2884 return (ret); 2885 } 2886 2887 boolean_t 2888 mlxcx_cmd_set_flow_table_root(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft) 2889 { 2890 mlxcx_cmd_t cmd; 2891 mlxcx_cmd_set_flow_table_root_in_t in; 2892 mlxcx_cmd_set_flow_table_root_out_t out; 2893 boolean_t ret; 2894 2895 bzero(&in, sizeof (in)); 2896 bzero(&out, sizeof (out)); 2897 2898 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2899 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2900 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 2901 2902 mlxcx_cmd_init(mlxp, &cmd); 2903 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_flow_table_root_head, 2904 MLXCX_OP_SET_FLOW_TABLE_ROOT, 0); 2905 2906 in.mlxi_set_flow_table_root_vport_number = 2907 to_be16(mlft->mlft_port->mlp_num); 2908 in.mlxi_set_flow_table_root_table_type = mlft->mlft_type; 2909 in.mlxi_set_flow_table_root_table_id = to_be24(mlft->mlft_num); 2910 2911 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2912 mlxcx_cmd_fini(mlxp, &cmd); 2913 return (B_FALSE); 2914 } 2915 mlxcx_cmd_wait(&cmd); 2916 2917 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2918 if (ret) { 2919 mlft->mlft_state |= MLXCX_FLOW_TABLE_ROOT; 2920 } 2921 mlxcx_cmd_fini(mlxp, &cmd); 2922 return (ret); 2923 } 2924 2925 boolean_t 2926 mlxcx_cmd_create_flow_group(mlxcx_t *mlxp, mlxcx_flow_group_t *mlfg) 2927 { 2928 mlxcx_cmd_t cmd; 2929 mlxcx_cmd_create_flow_group_in_t in; 2930 mlxcx_cmd_create_flow_group_out_t out; 2931 boolean_t ret; 2932 const mlxcx_flow_table_t *mlft; 2933 mlxcx_flow_header_match_t *hdrs; 2934 mlxcx_flow_params_match_t *params; 2935 2936 bzero(&in, sizeof (in)); 2937 bzero(&out, sizeof (out)); 2938 2939 mlft = mlfg->mlfg_table; 2940 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2941 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2942 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 2943 VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED); 2944 2945 mlxcx_cmd_init(mlxp, &cmd); 2946 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_flow_group_head, 2947 MLXCX_OP_CREATE_FLOW_GROUP, 0); 2948 2949 in.mlxi_create_flow_group_vport_number = 2950 to_be16(mlft->mlft_port->mlp_num); 2951 in.mlxi_create_flow_group_table_type = mlft->mlft_type; 2952 in.mlxi_create_flow_group_table_id = to_be24(mlft->mlft_num); 2953 in.mlxi_create_flow_group_start_flow_index = 2954 to_be32(mlfg->mlfg_start_idx); 2955 in.mlxi_create_flow_group_end_flow_index = 2956 to_be32(mlfg->mlfg_start_idx + (mlfg->mlfg_size - 1)); 2957 2958 hdrs = &in.mlxi_create_flow_group_match_criteria.mlfm_outer_headers; 2959 params = &in.mlxi_create_flow_group_match_criteria.mlfm_misc_parameters; 2960 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SMAC) { 2961 in.mlxi_create_flow_group_match_criteria_en |= 2962 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2963 (void) memset(&hdrs->mlfh_smac, 0xff, sizeof (hdrs->mlfh_smac)); 2964 } 2965 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DMAC) { 2966 in.mlxi_create_flow_group_match_criteria_en |= 2967 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2968 (void) memset(&hdrs->mlfh_dmac, 0xff, sizeof (hdrs->mlfh_dmac)); 2969 } 2970 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN) { 2971 in.mlxi_create_flow_group_match_criteria_en |= 2972 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2973 set_bit24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_CVLAN_TAG); 2974 set_bit24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_SVLAN_TAG); 2975 } 2976 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VID) { 2977 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN); 2978 set_bits16(&hdrs->mlfh_first_vid_flags, 2979 MLXCX_FLOW_HDR_FIRST_VID, UINT16_MAX); 2980 } 2981 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER) { 2982 in.mlxi_create_flow_group_match_criteria_en |= 2983 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2984 set_bits24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_IP_VERSION, 2985 UINT32_MAX); 2986 } 2987 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) { 2988 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 2989 (void) memset(&hdrs->mlfh_src_ip, 0xff, 2990 sizeof (hdrs->mlfh_src_ip)); 2991 } 2992 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) { 2993 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 2994 (void) memset(&hdrs->mlfh_src_ip, 0xff, 2995 sizeof (hdrs->mlfh_dst_ip)); 2996 } 2997 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_PROTO) { 2998 in.mlxi_create_flow_group_match_criteria_en |= 2999 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 3000 hdrs->mlfh_ip_protocol = UINT8_MAX; 3001 } 3002 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) { 3003 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3004 (void) memset(&hdrs->mlfh_src_ip, 0xff, 3005 sizeof (hdrs->mlfh_src_ip)); 3006 } 3007 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) { 3008 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3009 (void) memset(&hdrs->mlfh_src_ip, 0xff, 3010 sizeof (hdrs->mlfh_dst_ip)); 3011 } 3012 3013 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SQN) { 3014 in.mlxi_create_flow_group_match_criteria_en |= 3015 MLXCX_FLOW_GROUP_MATCH_MISC_PARAMS; 3016 params->mlfp_source_sqn = to_be24(UINT32_MAX); 3017 } 3018 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VXLAN) { 3019 in.mlxi_create_flow_group_match_criteria_en |= 3020 MLXCX_FLOW_GROUP_MATCH_MISC_PARAMS; 3021 params->mlfp_vxlan_vni = to_be24(UINT32_MAX); 3022 } 3023 3024 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3025 mlxcx_cmd_fini(mlxp, &cmd); 3026 return (B_FALSE); 3027 } 3028 mlxcx_cmd_wait(&cmd); 3029 3030 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3031 if (ret) { 3032 mlfg->mlfg_state |= MLXCX_FLOW_GROUP_CREATED; 3033 mlfg->mlfg_num = from_be24(out.mlxo_create_flow_group_group_id); 3034 } 3035 mlxcx_cmd_fini(mlxp, &cmd); 3036 return (ret); 3037 } 3038 3039 boolean_t 3040 mlxcx_cmd_destroy_flow_group(mlxcx_t *mlxp, mlxcx_flow_group_t *mlfg) 3041 { 3042 mlxcx_cmd_t cmd; 3043 mlxcx_cmd_destroy_flow_group_in_t in; 3044 mlxcx_cmd_destroy_flow_group_out_t out; 3045 boolean_t ret; 3046 const mlxcx_flow_table_t *mlft; 3047 3048 bzero(&in, sizeof (in)); 3049 bzero(&out, sizeof (out)); 3050 3051 mlft = mlfg->mlfg_table; 3052 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3053 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3054 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3055 VERIFY(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED); 3056 VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_DESTROYED); 3057 3058 mlxcx_cmd_init(mlxp, &cmd); 3059 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_flow_group_head, 3060 MLXCX_OP_DESTROY_FLOW_GROUP, 0); 3061 3062 in.mlxi_destroy_flow_group_vport_number = 3063 to_be16(mlft->mlft_port->mlp_num); 3064 in.mlxi_destroy_flow_group_table_type = mlft->mlft_type; 3065 in.mlxi_destroy_flow_group_table_id = to_be24(mlft->mlft_num); 3066 in.mlxi_destroy_flow_group_group_id = to_be32(mlfg->mlfg_num); 3067 3068 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3069 mlxcx_cmd_fini(mlxp, &cmd); 3070 return (B_FALSE); 3071 } 3072 mlxcx_cmd_wait(&cmd); 3073 3074 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3075 if (ret) { 3076 mlfg->mlfg_state |= MLXCX_FLOW_GROUP_DESTROYED; 3077 } 3078 mlxcx_cmd_fini(mlxp, &cmd); 3079 return (ret); 3080 } 3081 3082 boolean_t 3083 mlxcx_cmd_set_flow_table_entry(mlxcx_t *mlxp, mlxcx_flow_entry_t *mlfe) 3084 { 3085 mlxcx_cmd_t cmd; 3086 mlxcx_cmd_set_flow_table_entry_in_t in; 3087 mlxcx_cmd_set_flow_table_entry_out_t out; 3088 boolean_t ret; 3089 size_t insize; 3090 mlxcx_flow_entry_ctx_t *ctx; 3091 const mlxcx_flow_table_t *mlft; 3092 mlxcx_flow_group_t *mlfg; 3093 mlxcx_flow_dest_t *d; 3094 uint_t i; 3095 mlxcx_flow_header_match_t *hdrs; 3096 mlxcx_flow_params_match_t *params; 3097 mlxcx_cmd_set_flow_table_entry_opmod_t opmod; 3098 3099 bzero(&in, sizeof (in)); 3100 bzero(&out, sizeof (out)); 3101 3102 mlft = mlfe->mlfe_table; 3103 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3104 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3105 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3106 3107 mlfg = mlfe->mlfe_group; 3108 VERIFY(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED); 3109 VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_DESTROYED); 3110 3111 opmod = MLXCX_CMD_FLOW_ENTRY_SET_NEW; 3112 if (mlfe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED) { 3113 ASSERT(mlfe->mlfe_state & MLXCX_FLOW_ENTRY_DIRTY); 3114 opmod = MLXCX_CMD_FLOW_ENTRY_MODIFY; 3115 } 3116 3117 mlxcx_cmd_init(mlxp, &cmd); 3118 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_flow_table_entry_head, 3119 MLXCX_OP_SET_FLOW_TABLE_ENTRY, opmod); 3120 3121 in.mlxi_set_flow_table_entry_vport_number = 3122 to_be16(mlft->mlft_port->mlp_num); 3123 in.mlxi_set_flow_table_entry_table_type = mlft->mlft_type; 3124 in.mlxi_set_flow_table_entry_table_id = to_be24(mlft->mlft_num); 3125 in.mlxi_set_flow_table_entry_flow_index = to_be32(mlfe->mlfe_index); 3126 3127 if (mlfe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED) { 3128 set_bit8(&in.mlxi_set_flow_table_entry_modify_bitmask, 3129 MLXCX_CMD_FLOW_ENTRY_SET_ACTION); 3130 set_bit8(&in.mlxi_set_flow_table_entry_modify_bitmask, 3131 MLXCX_CMD_FLOW_ENTRY_SET_DESTINATION); 3132 } 3133 3134 ctx = &in.mlxi_set_flow_table_entry_context; 3135 ctx->mlfec_group_id = to_be32(mlfg->mlfg_num); 3136 3137 insize = offsetof(mlxcx_cmd_set_flow_table_entry_in_t, 3138 mlxi_set_flow_table_entry_context) + 3139 offsetof(mlxcx_flow_entry_ctx_t, mlfec_destination); 3140 3141 ctx->mlfec_action = to_be16(mlfe->mlfe_action); 3142 3143 switch (mlfe->mlfe_action) { 3144 case MLXCX_FLOW_ACTION_ALLOW: 3145 case MLXCX_FLOW_ACTION_DROP: 3146 break; 3147 case MLXCX_FLOW_ACTION_FORWARD: 3148 ASSERT3U(mlfe->mlfe_ndest, <=, MLXCX_FLOW_MAX_DESTINATIONS); 3149 ASSERT3U(mlfe->mlfe_ndest, <=, 3150 mlxp->mlx_caps->mlc_max_rx_fe_dest); 3151 ctx->mlfec_destination_list_size = to_be24(mlfe->mlfe_ndest); 3152 for (i = 0; i < mlfe->mlfe_ndest; ++i) { 3153 insize += sizeof (mlxcx_flow_dest_t); 3154 d = &ctx->mlfec_destination[i]; 3155 if (mlfe->mlfe_dest[i].mlfed_tir != NULL) { 3156 d->mlfd_destination_type = MLXCX_FLOW_DEST_TIR; 3157 d->mlfd_destination_id = to_be24( 3158 mlfe->mlfe_dest[i].mlfed_tir->mltir_num); 3159 } else if (mlfe->mlfe_dest[i].mlfed_flow != NULL) { 3160 d->mlfd_destination_type = 3161 MLXCX_FLOW_DEST_FLOW_TABLE; 3162 d->mlfd_destination_id = to_be24( 3163 mlfe->mlfe_dest[i].mlfed_flow->mlft_num); 3164 } else { 3165 /* Invalid flow entry destination */ 3166 VERIFY(0); 3167 } 3168 } 3169 break; 3170 case MLXCX_FLOW_ACTION_COUNT: 3171 /* We don't support count actions yet. */ 3172 VERIFY(0); 3173 break; 3174 case MLXCX_FLOW_ACTION_ENCAP: 3175 case MLXCX_FLOW_ACTION_DECAP: 3176 /* We don't support encap/decap actions yet. */ 3177 VERIFY(0); 3178 break; 3179 } 3180 3181 hdrs = &ctx->mlfec_match_value.mlfm_outer_headers; 3182 params = &ctx->mlfec_match_value.mlfm_misc_parameters; 3183 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SMAC) { 3184 bcopy(mlfe->mlfe_smac, hdrs->mlfh_smac, 3185 sizeof (hdrs->mlfh_smac)); 3186 } 3187 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DMAC) { 3188 bcopy(mlfe->mlfe_dmac, hdrs->mlfh_dmac, 3189 sizeof (hdrs->mlfh_dmac)); 3190 } 3191 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN) { 3192 switch (mlfe->mlfe_vlan_type) { 3193 case MLXCX_VLAN_TYPE_CVLAN: 3194 set_bit24(&hdrs->mlfh_tcp_ip_flags, 3195 MLXCX_FLOW_HDR_CVLAN_TAG); 3196 break; 3197 case MLXCX_VLAN_TYPE_SVLAN: 3198 set_bit24(&hdrs->mlfh_tcp_ip_flags, 3199 MLXCX_FLOW_HDR_SVLAN_TAG); 3200 break; 3201 default: 3202 break; 3203 } 3204 } 3205 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VID) { 3206 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN); 3207 set_bits16(&hdrs->mlfh_first_vid_flags, 3208 MLXCX_FLOW_HDR_FIRST_VID, mlfe->mlfe_vid); 3209 } 3210 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER) { 3211 set_bits24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_IP_VERSION, 3212 mlfe->mlfe_ip_version); 3213 } 3214 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) { 3215 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3216 bcopy(mlfe->mlfe_srcip, hdrs->mlfh_src_ip, 3217 sizeof (hdrs->mlfh_src_ip)); 3218 } 3219 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) { 3220 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3221 bcopy(mlfe->mlfe_dstip, hdrs->mlfh_src_ip, 3222 sizeof (hdrs->mlfh_dst_ip)); 3223 } 3224 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_PROTO) { 3225 hdrs->mlfh_ip_protocol = mlfe->mlfe_ip_proto; 3226 } 3227 3228 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SQN) { 3229 params->mlfp_source_sqn = to_be24(mlfe->mlfe_sqn); 3230 } 3231 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VXLAN) { 3232 params->mlfp_vxlan_vni = to_be24(mlfe->mlfe_vxlan_vni); 3233 } 3234 3235 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 3236 mlxcx_cmd_fini(mlxp, &cmd); 3237 return (B_FALSE); 3238 } 3239 mlxcx_cmd_wait(&cmd); 3240 3241 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3242 if (ret) { 3243 mlfe->mlfe_state |= MLXCX_FLOW_ENTRY_CREATED; 3244 mlfe->mlfe_state &= ~MLXCX_FLOW_ENTRY_DIRTY; 3245 mlfg->mlfg_state |= MLXCX_FLOW_GROUP_BUSY; 3246 } 3247 mlxcx_cmd_fini(mlxp, &cmd); 3248 return (ret); 3249 } 3250 3251 boolean_t 3252 mlxcx_cmd_delete_flow_table_entry(mlxcx_t *mlxp, mlxcx_flow_entry_t *mlfe) 3253 { 3254 mlxcx_cmd_t cmd; 3255 mlxcx_cmd_delete_flow_table_entry_in_t in; 3256 mlxcx_cmd_delete_flow_table_entry_out_t out; 3257 boolean_t ret; 3258 const mlxcx_flow_table_t *mlft; 3259 3260 bzero(&in, sizeof (in)); 3261 bzero(&out, sizeof (out)); 3262 3263 mlft = mlfe->mlfe_table; 3264 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3265 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3266 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3267 3268 mlxcx_cmd_init(mlxp, &cmd); 3269 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_delete_flow_table_entry_head, 3270 MLXCX_OP_DELETE_FLOW_TABLE_ENTRY, 0); 3271 3272 in.mlxi_delete_flow_table_entry_vport_number = 3273 to_be16(mlft->mlft_port->mlp_num); 3274 in.mlxi_delete_flow_table_entry_table_type = mlft->mlft_type; 3275 in.mlxi_delete_flow_table_entry_table_id = to_be24(mlft->mlft_num); 3276 in.mlxi_delete_flow_table_entry_flow_index = to_be32(mlfe->mlfe_index); 3277 3278 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3279 mlxcx_cmd_fini(mlxp, &cmd); 3280 return (B_FALSE); 3281 } 3282 mlxcx_cmd_wait(&cmd); 3283 3284 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3285 if (ret) { 3286 /* 3287 * Note that flow entries have a different lifecycle to most 3288 * other things we create -- we have to be able to re-use them 3289 * after they have been deleted, since they exist at a fixed 3290 * position in their flow table. 3291 * 3292 * So we clear the CREATED bit here for them to let us call 3293 * create_flow_table_entry() on the same entry again later. 3294 */ 3295 mlfe->mlfe_state &= ~MLXCX_FLOW_ENTRY_CREATED; 3296 mlfe->mlfe_state |= MLXCX_FLOW_ENTRY_DELETED; 3297 } 3298 mlxcx_cmd_fini(mlxp, &cmd); 3299 return (ret); 3300 } 3301 3302 boolean_t 3303 mlxcx_cmd_create_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3304 { 3305 mlxcx_cmd_t cmd; 3306 mlxcx_cmd_create_sq_in_t in; 3307 mlxcx_cmd_create_sq_out_t out; 3308 boolean_t ret; 3309 mlxcx_sq_ctx_t *ctx; 3310 size_t rem, insize; 3311 const ddi_dma_cookie_t *c; 3312 uint64_t pa, npages; 3313 3314 bzero(&in, sizeof (in)); 3315 bzero(&out, sizeof (out)); 3316 3317 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3318 VERIFY3U(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_SENDQ); 3319 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3320 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3321 3322 mlxcx_cmd_init(mlxp, &cmd); 3323 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_sq_head, 3324 MLXCX_OP_CREATE_SQ, 0); 3325 3326 ctx = &in.mlxi_create_sq_context; 3327 3328 set_bit32(&ctx->mlsqc_flags, MLXCX_SQ_FLAGS_RLKEY); 3329 set_bit32(&ctx->mlsqc_flags, MLXCX_SQ_FLAGS_FLUSH_IN_ERROR); 3330 set_bits32(&ctx->mlsqc_flags, MLXCX_SQ_MIN_WQE_INLINE, 3331 mlwq->mlwq_inline_mode); 3332 ctx->mlsqc_cqn = to_be24(mlwq->mlwq_cq->mlcq_num); 3333 3334 VERIFY(mlwq->mlwq_tis != NULL); 3335 ctx->mlsqc_tis_lst_sz = to_be16(1); 3336 ctx->mlsqc_tis_num = to_be24(mlwq->mlwq_tis->mltis_num); 3337 3338 set_bits32(&ctx->mlsqc_wq.mlwqc_flags, MLXCX_WORKQ_CTX_TYPE, 3339 MLXCX_WORKQ_TYPE_CYCLIC); 3340 ctx->mlsqc_wq.mlwqc_pd = to_be24(mlwq->mlwq_pd->mlpd_num); 3341 ctx->mlsqc_wq.mlwqc_uar_page = to_be24(mlwq->mlwq_uar->mlu_num); 3342 ctx->mlsqc_wq.mlwqc_log_wq_sz = mlwq->mlwq_entshift; 3343 ctx->mlsqc_wq.mlwqc_log_wq_stride = MLXCX_SENDQ_STRIDE_SHIFT; 3344 3345 c = mlxcx_dma_cookie_one(&mlwq->mlwq_doorbell_dma); 3346 ctx->mlsqc_wq.mlwqc_dbr_addr = to_be64(c->dmac_laddress); 3347 ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_workq_doorbell_t)); 3348 3349 npages = 0; 3350 c = NULL; 3351 while ((c = mlxcx_dma_cookie_iter(&mlwq->mlwq_dma, c)) != NULL) { 3352 pa = c->dmac_laddress; 3353 rem = c->dmac_size; 3354 while (rem > 0) { 3355 ASSERT3U(pa & 0xfff, ==, 0); 3356 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 3357 ctx->mlsqc_wq.mlwqc_pas[npages++] = to_be64(pa); 3358 rem -= MLXCX_HW_PAGE_SIZE; 3359 pa += MLXCX_HW_PAGE_SIZE; 3360 } 3361 } 3362 ASSERT3U(npages, <=, MLXCX_WORKQ_CTX_MAX_ADDRESSES); 3363 3364 insize = offsetof(mlxcx_cmd_create_sq_in_t, mlxi_create_sq_context) + 3365 offsetof(mlxcx_sq_ctx_t, mlsqc_wq) + 3366 offsetof(mlxcx_workq_ctx_t, mlwqc_pas) + 3367 sizeof (uint64_t) * npages; 3368 3369 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 3370 mlxcx_cmd_fini(mlxp, &cmd); 3371 return (B_FALSE); 3372 } 3373 mlxcx_cmd_wait(&cmd); 3374 3375 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3376 if (ret) { 3377 mlwq->mlwq_state |= MLXCX_WQ_CREATED; 3378 mlwq->mlwq_num = from_be24(out.mlxo_create_sq_sqn); 3379 } 3380 mlxcx_cmd_fini(mlxp, &cmd); 3381 return (ret); 3382 } 3383 3384 boolean_t 3385 mlxcx_cmd_start_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3386 { 3387 mlxcx_cmd_t cmd; 3388 mlxcx_cmd_modify_sq_in_t in; 3389 mlxcx_cmd_modify_sq_out_t out; 3390 boolean_t ret; 3391 ddi_fm_error_t err; 3392 3393 bzero(&in, sizeof (in)); 3394 bzero(&out, sizeof (out)); 3395 3396 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3397 ASSERT(mlwq->mlwq_cq != NULL); 3398 3399 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3400 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3401 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 3402 3403 /* 3404 * Before starting the queue, we have to be sure that it is 3405 * empty and the doorbell and counters are set to 0. 3406 */ 3407 ASSERT(mutex_owned(&mlwq->mlwq_cq->mlcq_mtx)); 3408 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers)); 3409 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers_b)); 3410 3411 mlwq->mlwq_doorbell->mlwqd_recv_counter = to_be16(0); 3412 MLXCX_DMA_SYNC(mlwq->mlwq_doorbell_dma, DDI_DMA_SYNC_FORDEV); 3413 ddi_fm_dma_err_get(mlwq->mlwq_doorbell_dma.mxdb_dma_handle, &err, 3414 DDI_FME_VERSION); 3415 if (err.fme_status != DDI_FM_OK) 3416 return (B_FALSE); 3417 mlwq->mlwq_pc = 0; 3418 3419 mlxcx_cmd_init(mlxp, &cmd); 3420 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_sq_head, 3421 MLXCX_OP_MODIFY_SQ, 0); 3422 3423 in.mlxi_modify_sq_sqn = to_be24(mlwq->mlwq_num); 3424 3425 /* From state */ 3426 set_bits8(&in.mlxi_modify_sq_state, MLXCX_CMD_MODIFY_SQ_STATE, 3427 MLXCX_SQ_STATE_RST); 3428 /* To state */ 3429 set_bits32(&in.mlxi_modify_sq_context.mlsqc_flags, MLXCX_SQ_STATE, 3430 MLXCX_SQ_STATE_RDY); 3431 3432 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3433 mlxcx_cmd_fini(mlxp, &cmd); 3434 return (B_FALSE); 3435 } 3436 mlxcx_cmd_wait(&cmd); 3437 3438 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3439 if (ret) { 3440 mlwq->mlwq_state |= MLXCX_WQ_STARTED; 3441 } 3442 mlxcx_cmd_fini(mlxp, &cmd); 3443 return (ret); 3444 } 3445 3446 boolean_t 3447 mlxcx_cmd_stop_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3448 { 3449 mlxcx_cmd_t cmd; 3450 mlxcx_cmd_modify_sq_in_t in; 3451 mlxcx_cmd_modify_sq_out_t out; 3452 boolean_t ret; 3453 3454 bzero(&in, sizeof (in)); 3455 bzero(&out, sizeof (out)); 3456 3457 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3458 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3459 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3460 VERIFY(mlwq->mlwq_state & MLXCX_WQ_STARTED); 3461 3462 mlxcx_cmd_init(mlxp, &cmd); 3463 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_sq_head, 3464 MLXCX_OP_MODIFY_SQ, 0); 3465 3466 in.mlxi_modify_sq_sqn = to_be24(mlwq->mlwq_num); 3467 3468 /* From state */ 3469 set_bits8(&in.mlxi_modify_sq_state, MLXCX_CMD_MODIFY_SQ_STATE, 3470 MLXCX_SQ_STATE_RDY); 3471 /* To state */ 3472 set_bits32(&in.mlxi_modify_sq_context.mlsqc_flags, MLXCX_SQ_STATE, 3473 MLXCX_SQ_STATE_RST); 3474 3475 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3476 mlxcx_cmd_fini(mlxp, &cmd); 3477 return (B_FALSE); 3478 } 3479 mlxcx_cmd_wait(&cmd); 3480 3481 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3482 if (ret) { 3483 mlwq->mlwq_state &= ~MLXCX_WQ_STARTED; 3484 } 3485 mlxcx_cmd_fini(mlxp, &cmd); 3486 return (ret); 3487 } 3488 3489 boolean_t 3490 mlxcx_cmd_destroy_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3491 { 3492 mlxcx_cmd_t cmd; 3493 mlxcx_cmd_destroy_sq_in_t in; 3494 mlxcx_cmd_destroy_sq_out_t out; 3495 boolean_t ret; 3496 3497 bzero(&in, sizeof (in)); 3498 bzero(&out, sizeof (out)); 3499 3500 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3501 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3502 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3503 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 3504 3505 mlxcx_cmd_init(mlxp, &cmd); 3506 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_sq_head, 3507 MLXCX_OP_DESTROY_SQ, 0); 3508 3509 in.mlxi_destroy_sq_sqn = to_be24(mlwq->mlwq_num); 3510 3511 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3512 mlxcx_cmd_fini(mlxp, &cmd); 3513 return (B_FALSE); 3514 } 3515 mlxcx_cmd_wait(&cmd); 3516 3517 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3518 if (ret) { 3519 mlwq->mlwq_state |= MLXCX_WQ_DESTROYED; 3520 } 3521 mlxcx_cmd_fini(mlxp, &cmd); 3522 return (ret); 3523 } 3524 3525 boolean_t 3526 mlxcx_cmd_create_rqt(mlxcx_t *mlxp, mlxcx_rqtable_t *mlrqt) 3527 { 3528 mlxcx_cmd_t cmd; 3529 mlxcx_cmd_create_rqt_in_t in; 3530 mlxcx_cmd_create_rqt_out_t out; 3531 mlxcx_rqtable_ctx_t *ctx; 3532 boolean_t ret; 3533 uint_t i; 3534 3535 bzero(&in, sizeof (in)); 3536 bzero(&out, sizeof (out)); 3537 3538 VERIFY0(mlrqt->mlrqt_state & MLXCX_RQT_CREATED); 3539 3540 mlxcx_cmd_init(mlxp, &cmd); 3541 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_rqt_head, 3542 MLXCX_OP_CREATE_RQT, 0); 3543 3544 ctx = &in.mlxi_create_rqt_context; 3545 ASSERT3U(mlrqt->mlrqt_max, <=, MLXCX_RQT_MAX_RQ_REFS); 3546 ASSERT3U(mlrqt->mlrqt_max, <=, mlxp->mlx_caps->mlc_max_rqt_size); 3547 ctx->mlrqtc_max_size = to_be16(mlrqt->mlrqt_max); 3548 ctx->mlrqtc_actual_size = to_be16(mlrqt->mlrqt_used); 3549 for (i = 0; i < mlrqt->mlrqt_used; ++i) { 3550 ctx->mlrqtc_rqref[i].mlrqtr_rqn = to_be24( 3551 mlrqt->mlrqt_rq[i]->mlwq_num); 3552 } 3553 3554 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3555 mlxcx_cmd_fini(mlxp, &cmd); 3556 return (B_FALSE); 3557 } 3558 mlxcx_cmd_wait(&cmd); 3559 3560 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3561 if (ret) { 3562 mlrqt->mlrqt_num = from_be24(out.mlxo_create_rqt_rqtn); 3563 mlrqt->mlrqt_state |= MLXCX_RQT_CREATED; 3564 mlrqt->mlrqt_state &= ~MLXCX_RQT_DIRTY; 3565 } 3566 mlxcx_cmd_fini(mlxp, &cmd); 3567 return (ret); 3568 } 3569 3570 boolean_t 3571 mlxcx_cmd_destroy_rqt(mlxcx_t *mlxp, mlxcx_rqtable_t *mlrqt) 3572 { 3573 mlxcx_cmd_t cmd; 3574 mlxcx_cmd_destroy_rqt_in_t in; 3575 mlxcx_cmd_destroy_rqt_out_t out; 3576 boolean_t ret; 3577 3578 bzero(&in, sizeof (in)); 3579 bzero(&out, sizeof (out)); 3580 3581 VERIFY(mlrqt->mlrqt_state & MLXCX_RQT_CREATED); 3582 VERIFY0(mlrqt->mlrqt_state & MLXCX_RQT_DESTROYED); 3583 3584 mlxcx_cmd_init(mlxp, &cmd); 3585 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_rqt_head, 3586 MLXCX_OP_DESTROY_RQT, 0); 3587 3588 in.mlxi_destroy_rqt_rqtn = to_be24(mlrqt->mlrqt_num); 3589 3590 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3591 mlxcx_cmd_fini(mlxp, &cmd); 3592 return (B_FALSE); 3593 } 3594 mlxcx_cmd_wait(&cmd); 3595 3596 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3597 if (ret) { 3598 mlrqt->mlrqt_state |= MLXCX_RQT_DESTROYED; 3599 } 3600 mlxcx_cmd_fini(mlxp, &cmd); 3601 return (ret); 3602 } 3603 3604 boolean_t 3605 mlxcx_cmd_set_int_mod(mlxcx_t *mlxp, uint_t intr, uint_t min_delay) 3606 { 3607 mlxcx_cmd_t cmd; 3608 mlxcx_cmd_config_int_mod_in_t in; 3609 mlxcx_cmd_config_int_mod_out_t out; 3610 boolean_t ret; 3611 3612 bzero(&in, sizeof (in)); 3613 bzero(&out, sizeof (out)); 3614 3615 mlxcx_cmd_init(mlxp, &cmd); 3616 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_config_int_mod_head, 3617 MLXCX_OP_CONFIG_INT_MODERATION, MLXCX_CMD_CONFIG_INT_MOD_WRITE); 3618 3619 in.mlxi_config_int_mod_int_vector = to_be16(intr); 3620 in.mlxi_config_int_mod_min_delay = to_be16(min_delay); 3621 3622 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3623 mlxcx_cmd_fini(mlxp, &cmd); 3624 return (B_FALSE); 3625 } 3626 mlxcx_cmd_wait(&cmd); 3627 3628 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3629 mlxcx_cmd_fini(mlxp, &cmd); 3630 return (ret); 3631 } 3632 3633 /* 3634 * CTASSERTs here are for the structs in mlxcx_reg.h, to check they match 3635 * against offsets from the PRM. 3636 * 3637 * They're not in the header file, to avoid them being used by multiple .c 3638 * files. 3639 */ 3640 3641 CTASSERT(offsetof(mlxcx_eventq_ent_t, mleqe_unknown_data) == 0x20); 3642 CTASSERT(offsetof(mlxcx_eventq_ent_t, mleqe_signature) == 0x3c + 2); 3643 CTASSERT(sizeof (mlxcx_eventq_ent_t) == 64); 3644 3645 CTASSERT(offsetof(mlxcx_completionq_error_ent_t, mlcqee_byte_cnt) == 0x2C); 3646 CTASSERT(offsetof(mlxcx_completionq_error_ent_t, mlcqee_wqe_opcode) == 0x38); 3647 3648 CTASSERT(sizeof (mlxcx_completionq_error_ent_t) == 3649 sizeof (mlxcx_completionq_ent_t)); 3650 CTASSERT(sizeof (mlxcx_wqe_control_seg_t) == (1 << 4)); 3651 3652 CTASSERT(offsetof(mlxcx_wqe_eth_seg_t, mles_inline_headers) == 0x0e); 3653 CTASSERT(sizeof (mlxcx_wqe_eth_seg_t) == (1 << 5)); 3654 3655 CTASSERT(sizeof (mlxcx_wqe_data_seg_t) == (1 << 4)); 3656 3657 CTASSERT(sizeof (mlxcx_sendq_ent_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT)); 3658 3659 CTASSERT(sizeof (mlxcx_sendq_bf_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT)); 3660 3661 CTASSERT(sizeof (mlxcx_sendq_extra_ent_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT)); 3662 3663 CTASSERT(sizeof (mlxcx_recvq_ent_t) == (1 << MLXCX_RECVQ_STRIDE_SHIFT)); 3664 3665 CTASSERT(offsetof(mlxcx_workq_ctx_t, mlwqc_dbr_addr) == 0x10); 3666 CTASSERT(offsetof(mlxcx_workq_ctx_t, mlwqc_pas) == 0xc0); 3667 3668 CTASSERT(offsetof(mlxcx_rq_ctx_t, mlrqc_cqn) == 0x09); 3669 CTASSERT(offsetof(mlxcx_rq_ctx_t, mlrqc_wq) == 0x30); 3670 3671 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_cqn) == 0x09); 3672 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_tis_lst_sz) == 0x20); 3673 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_tis_num) == 0x2d); 3674 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_wq) == 0x30); 3675 3676 CTASSERT(sizeof (mlxcx_tis_ctx_t) == 0xa0); 3677 CTASSERT(offsetof(mlxcx_tis_ctx_t, mltisc_transport_domain) == 0x25); 3678 3679 CTASSERT(offsetof(mlxcx_rqtable_ctx_t, mlrqtc_max_size) == 0x16); 3680 CTASSERT(offsetof(mlxcx_rqtable_ctx_t, mlrqtc_rqref) == 0xF0); 3681 3682 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_event_bitmask) == 3683 0x58); 3684 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_pas) == 0x110); 3685 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_context) == 0x10); 3686 3687 CTASSERT(offsetof(mlxcx_cmd_create_tir_in_t, mlxi_create_tir_context) == 0x20); 3688 3689 CTASSERT(offsetof(mlxcx_cmd_create_tis_in_t, mlxi_create_tis_context) == 0x20); 3690 3691 CTASSERT(offsetof(mlxcx_cmd_query_special_ctxs_out_t, 3692 mlxo_query_special_ctxs_resd_lkey) == 0x0c); 3693 3694 CTASSERT(offsetof(mlxcx_cmd_query_cq_out_t, mlxo_query_cq_context) == 0x10); 3695 CTASSERT(offsetof(mlxcx_cmd_query_cq_out_t, mlxo_query_cq_pas) == 0x110); 3696 3697 CTASSERT(offsetof(mlxcx_cmd_query_rq_out_t, mlxo_query_rq_context) == 0x20); 3698 3699 CTASSERT(offsetof(mlxcx_cmd_create_sq_in_t, mlxi_create_sq_context) == 0x20); 3700 3701 CTASSERT(offsetof(mlxcx_cmd_modify_sq_in_t, mlxi_modify_sq_context) == 0x20); 3702 3703 CTASSERT(offsetof(mlxcx_cmd_query_sq_out_t, mlxo_query_sq_context) == 0x20); 3704 3705 CTASSERT(offsetof(mlxcx_cmd_create_rqt_in_t, mlxi_create_rqt_context) == 0x20); 3706 3707 CTASSERT(offsetof(mlxcx_reg_pmtu_t, mlrd_pmtu_oper_mtu) == 0x0C); 3708 3709 CTASSERT(sizeof (mlxcx_reg_ptys_t) == 64); 3710 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_cap) == 0x0c); 3711 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_admin) == 0x18); 3712 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_partner_advert) == 0x30); 3713 3714 CTASSERT(offsetof(mlxcx_reg_mcia_t, mlrd_mcia_data) == 0x10); 3715 3716 CTASSERT(offsetof(mlxcx_ppcnt_ieee_802_3_t, 3717 mlppc_ieee_802_3_in_range_len_err) == 0x50); 3718 CTASSERT(offsetof(mlxcx_ppcnt_ieee_802_3_t, 3719 mlppc_ieee_802_3_pause_tx) == 0x90); 3720 3721 CTASSERT(sizeof (mlxcx_reg_ppcnt_t) == 256); 3722 CTASSERT(offsetof(mlxcx_reg_ppcnt_t, mlrd_ppcnt_data) == 0x08); 3723 3724 CTASSERT(offsetof(mlxcx_cmd_access_register_in_t, 3725 mlxi_access_register_argument) == 0x0C); 3726 CTASSERT(offsetof(mlxcx_cmd_access_register_in_t, 3727 mlxi_access_register_data) == 0x10); 3728 3729 CTASSERT(offsetof(mlxcx_cmd_access_register_out_t, 3730 mlxo_access_register_data) == 0x10); 3731