xref: /illumos-gate/usr/src/uts/common/io/mlxcx/mlxcx_cmd.c (revision 85e4aa97d43a3e54749256ea56a405043c700b6b)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2020, The University of Queensland
14  * Copyright (c) 2018, Joyent, Inc.
15  * Copyright 2020 RackTop Systems, Inc.
16  * Copyright 2023 MNX Cloud, Inc.
17  */
18 
19 /*
20  * Controls the management of commands that are issues to and from the HCA
21  * command queue.
22  */
23 
24 #include <mlxcx.h>
25 
26 #include <sys/debug.h>
27 #include <sys/sysmacros.h>
28 
29 /*
30  * When we start up the command queue, it will undergo some internal
31  * initialization after we set the command queue address. These values allow us
32  * to control how much time we should wait for that to occur.
33  */
34 clock_t mlxcx_cmd_init_delay = 1000 * 10; /* 10 ms in us */
35 uint_t mlxcx_cmd_init_trys = 100; /* Wait at most 1s */
36 
37 clock_t mlxcx_cmd_delay = 1000 * 1; /* 1 ms in us */
38 uint_t mlxcx_cmd_tries = 5000; /* Wait at most 1s */
39 
40 /*
41  * This macro is used to identify that we care about our own function that we're
42  * communicating with. We always use this function.
43  */
44 #define	MLXCX_FUNCTION_SELF	(to_be16(0))
45 
46 static const char *
mlxcx_cmd_response_string(mlxcx_cmd_ret_t ret)47 mlxcx_cmd_response_string(mlxcx_cmd_ret_t ret)
48 {
49 	switch (ret) {
50 	case MLXCX_CMD_R_OK:
51 		return ("MLXCX_CMD_R_OK");
52 	case MLXCX_CMD_R_INTERNAL_ERR:
53 		return ("MLXCX_CMD_R_INTERNAL_ERR");
54 	case MLXCX_CMD_R_BAD_OP:
55 		return ("MLXCX_CMD_R_BAD_OP");
56 	case MLXCX_CMD_R_BAD_PARAM:
57 		return ("MLXCX_CMD_R_BAD_PARAM");
58 	case MLXCX_CMD_R_BAD_SYS_STATE:
59 		return ("MLXCX_CMD_R_BAD_SYS_STATE");
60 	case MLXCX_CMD_R_BAD_RESOURCE:
61 		return ("MLXCX_CMD_R_BAD_RESOURCE");
62 	case MLXCX_CMD_R_RESOURCE_BUSY:
63 		return ("MLXCX_CMD_R_RESOURCE_BUSY");
64 	case MLXCX_CMD_R_EXCEED_LIM:
65 		return ("MLXCX_CMD_R_EXCEED_LIM");
66 	case MLXCX_CMD_R_BAD_RES_STATE:
67 		return ("MLXCX_CMD_R_BAD_RES_STATE");
68 	case MLXCX_CMD_R_BAD_INDEX:
69 		return ("MLXCX_CMD_R_BAD_INDEX");
70 	case MLXCX_CMD_R_NO_RESOURCES:
71 		return ("MLXCX_CMD_R_NO_RESOURCES");
72 	case MLXCX_CMD_R_BAD_INPUT_LEN:
73 		return ("MLXCX_CMD_R_BAD_INPUT_LEN");
74 	case MLXCX_CMD_R_BAD_OUTPUT_LEN:
75 		return ("MLXCX_CMD_R_BAD_OUTPUT_LEN");
76 	case MLXCX_CMD_R_BAD_RESOURCE_STATE:
77 		return ("MLXCX_CMD_R_BAD_RESOURCE_STATE");
78 	case MLXCX_CMD_R_BAD_PKT:
79 		return ("MLXCX_CMD_R_BAD_PKT");
80 	case MLXCX_CMD_R_BAD_SIZE:
81 		return ("MLXCX_CMD_R_BAD_SIZE");
82 	default:
83 		return ("Unknown command");
84 	}
85 }
86 
87 static const char *
mlxcx_cmd_opcode_string(mlxcx_cmd_op_t op)88 mlxcx_cmd_opcode_string(mlxcx_cmd_op_t op)
89 {
90 	switch (op) {
91 	case MLXCX_OP_QUERY_HCA_CAP:
92 		return ("MLXCX_OP_QUERY_HCA_CAP");
93 	case MLXCX_OP_QUERY_ADAPTER:
94 		return ("MLXCX_OP_QUERY_ADAPTER");
95 	case MLXCX_OP_INIT_HCA:
96 		return ("MLXCX_OP_INIT_HCA");
97 	case MLXCX_OP_TEARDOWN_HCA:
98 		return ("MLXCX_OP_TEARDOWN_HCA");
99 	case MLXCX_OP_ENABLE_HCA:
100 		return ("MLXCX_OP_ENABLE_HCA");
101 	case MLXCX_OP_DISABLE_HCA:
102 		return ("MLXCX_OP_DISABLE_HCA");
103 	case MLXCX_OP_QUERY_PAGES:
104 		return ("MLXCX_OP_QUERY_PAGES");
105 	case MLXCX_OP_MANAGE_PAGES:
106 		return ("MLXCX_OP_MANAGE_PAGES");
107 	case MLXCX_OP_SET_HCA_CAP:
108 		return ("MLXCX_OP_SET_HCA_CAP");
109 	case MLXCX_OP_QUERY_ISSI:
110 		return ("MLXCX_OP_QUERY_ISSI");
111 	case MLXCX_OP_SET_ISSI:
112 		return ("MLXCX_OP_SET_ISSI");
113 	case MLXCX_OP_SET_DRIVER_VERSION:
114 		return ("MLXCX_OP_SET_DRIVER_VERSION");
115 	case MLXCX_OP_QUERY_OTHER_HCA_CAP:
116 		return ("MLXCX_OP_QUERY_OTHER_HCA_CAP");
117 	case MLXCX_OP_MODIFY_OTHER_HCA_CAP:
118 		return ("MLXCX_OP_MODIFY_OTHER_HCA_CAP");
119 	case MLXCX_OP_SET_TUNNELED_OPERATIONS:
120 		return ("MLXCX_OP_SET_TUNNELED_OPERATIONS");
121 	case MLXCX_OP_CREATE_MKEY:
122 		return ("MLXCX_OP_CREATE_MKEY");
123 	case MLXCX_OP_QUERY_MKEY:
124 		return ("MLXCX_OP_QUERY_MKEY");
125 	case MLXCX_OP_DESTROY_MKEY:
126 		return ("MLXCX_OP_DESTROY_MKEY");
127 	case MLXCX_OP_QUERY_SPECIAL_CONTEXTS:
128 		return ("MLXCX_OP_QUERY_SPECIAL_CONTEXTS");
129 	case MLXCX_OP_PAGE_FAULT_RESUME:
130 		return ("MLXCX_OP_PAGE_FAULT_RESUME");
131 	case MLXCX_OP_CREATE_EQ:
132 		return ("MLXCX_OP_CREATE_EQ");
133 	case MLXCX_OP_DESTROY_EQ:
134 		return ("MLXCX_OP_DESTROY_EQ");
135 	case MLXCX_OP_QUERY_EQ:
136 		return ("MLXCX_OP_QUERY_EQ");
137 	case MLXCX_OP_GEN_EQE:
138 		return ("MLXCX_OP_GEN_EQE");
139 	case MLXCX_OP_CREATE_CQ:
140 		return ("MLXCX_OP_CREATE_CQ");
141 	case MLXCX_OP_DESTROY_CQ:
142 		return ("MLXCX_OP_DESTROY_CQ");
143 	case MLXCX_OP_QUERY_CQ:
144 		return ("MLXCX_OP_QUERY_CQ");
145 	case MLXCX_OP_MODIFY_CQ:
146 		return ("MLXCX_OP_MODIFY_CQ");
147 	case MLXCX_OP_CREATE_QP:
148 		return ("MLXCX_OP_CREATE_QP");
149 	case MLXCX_OP_DESTROY_QP:
150 		return ("MLXCX_OP_DESTROY_QP");
151 	case MLXCX_OP_RST2INIT_QP:
152 		return ("MLXCX_OP_RST2INIT_QP");
153 	case MLXCX_OP_INIT2RTR_QP:
154 		return ("MLXCX_OP_INIT2RTR_QP");
155 	case MLXCX_OP_RTR2RTS_QP:
156 		return ("MLXCX_OP_RTR2RTS_QP");
157 	case MLXCX_OP_RTS2RTS_QP:
158 		return ("MLXCX_OP_RTS2RTS_QP");
159 	case MLXCX_OP_SQERR2RTS_QP:
160 		return ("MLXCX_OP_SQERR2RTS_QP");
161 	case MLXCX_OP__2ERR_QP:
162 		return ("MLXCX_OP__2ERR_QP");
163 	case MLXCX_OP__2RST_QP:
164 		return ("MLXCX_OP__2RST_QP");
165 	case MLXCX_OP_QUERY_QP:
166 		return ("MLXCX_OP_QUERY_QP");
167 	case MLXCX_OP_SQD_RTS_QP:
168 		return ("MLXCX_OP_SQD_RTS_QP");
169 	case MLXCX_OP_INIT2INIT_QP:
170 		return ("MLXCX_OP_INIT2INIT_QP");
171 	case MLXCX_OP_CREATE_PSV:
172 		return ("MLXCX_OP_CREATE_PSV");
173 	case MLXCX_OP_DESTROY_PSV:
174 		return ("MLXCX_OP_DESTROY_PSV");
175 	case MLXCX_OP_CREATE_SRQ:
176 		return ("MLXCX_OP_CREATE_SRQ");
177 	case MLXCX_OP_DESTROY_SRQ:
178 		return ("MLXCX_OP_DESTROY_SRQ");
179 	case MLXCX_OP_QUERY_SRQ:
180 		return ("MLXCX_OP_QUERY_SRQ");
181 	case MLXCX_OP_ARM_RQ:
182 		return ("MLXCX_OP_ARM_RQ");
183 	case MLXCX_OP_CREATE_XRC_SRQ:
184 		return ("MLXCX_OP_CREATE_XRC_SRQ");
185 	case MLXCX_OP_DESTROY_XRC_SRQ:
186 		return ("MLXCX_OP_DESTROY_XRC_SRQ");
187 	case MLXCX_OP_QUERY_XRC_SRQ:
188 		return ("MLXCX_OP_QUERY_XRC_SRQ");
189 	case MLXCX_OP_ARM_XRC_SRQ:
190 		return ("MLXCX_OP_ARM_XRC_SRQ");
191 	case MLXCX_OP_CREATE_DCT:
192 		return ("MLXCX_OP_CREATE_DCT");
193 	case MLXCX_OP_DESTROY_DCT:
194 		return ("MLXCX_OP_DESTROY_DCT");
195 	case MLXCX_OP_DRAIN_DCT:
196 		return ("MLXCX_OP_DRAIN_DCT");
197 	case MLXCX_OP_QUERY_DCT:
198 		return ("MLXCX_OP_QUERY_DCT");
199 	case MLXCX_OP_ARM_DCT_FOR_KEY_VIOLATION:
200 		return ("MLXCX_OP_ARM_DCT_FOR_KEY_VIOLATION");
201 	case MLXCX_OP_CREATE_XRQ:
202 		return ("MLXCX_OP_CREATE_XRQ");
203 	case MLXCX_OP_DESTROY_XRQ:
204 		return ("MLXCX_OP_DESTROY_XRQ");
205 	case MLXCX_OP_QUERY_XRQ:
206 		return ("MLXCX_OP_QUERY_XRQ");
207 	case MLXCX_OP_CREATE_NVMF_BACKEND_CONTROLLER:
208 		return ("MLXCX_OP_CREATE_NVMF_BACKEND_CONTROLLER");
209 	case MLXCX_OP_DESTROY_NVMF_BACKEND_CONTROLLER:
210 		return ("MLXCX_OP_DESTROY_NVMF_BACKEND_CONTROLLER");
211 	case MLXCX_OP_QUERY_NVMF_BACKEND_CONTROLLER:
212 		return ("MLXCX_OP_QUERY_NVMF_BACKEND_CONTROLLER");
213 	case MLXCX_OP_ATTACH_NVMF_NAMESPACE:
214 		return ("MLXCX_OP_ATTACH_NVMF_NAMESPACE");
215 	case MLXCX_OP_DETACH_NVMF_NAMESPACE:
216 		return ("MLXCX_OP_DETACH_NVMF_NAMESPACE");
217 	case MLXCX_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
218 		return ("MLXCX_OP_QUERY_XRQ_DC_PARAMS_ENTRY");
219 	case MLXCX_OP_SET_XRQ_DC_PARAMS_ENTRY:
220 		return ("MLXCX_OP_SET_XRQ_DC_PARAMS_ENTRY");
221 	case MLXCX_OP_QUERY_XRQ_ERROR_PARAMS:
222 		return ("MLXCX_OP_QUERY_XRQ_ERROR_PARAMS");
223 	case MLXCX_OP_QUERY_VPORT_STATE:
224 		return ("MLXCX_OP_QUERY_VPORT_STATE");
225 	case MLXCX_OP_MODIFY_VPORT_STATE:
226 		return ("MLXCX_OP_MODIFY_VPORT_STATE");
227 	case MLXCX_OP_QUERY_ESW_VPORT_CONTEXT:
228 		return ("MLXCX_OP_QUERY_ESW_VPORT_CONTEXT");
229 	case MLXCX_OP_MODIFY_ESW_VPORT_CONTEXT:
230 		return ("MLXCX_OP_MODIFY_ESW_VPORT_CONTEXT");
231 	case MLXCX_OP_QUERY_NIC_VPORT_CONTEXT:
232 		return ("MLXCX_OP_QUERY_NIC_VPORT_CONTEXT");
233 	case MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT:
234 		return ("MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT");
235 	case MLXCX_OP_QUERY_ROCE_ADDRESS:
236 		return ("MLXCX_OP_QUERY_ROCE_ADDRESS");
237 	case MLXCX_OP_SET_ROCE_ADDRESS:
238 		return ("MLXCX_OP_SET_ROCE_ADDRESS");
239 	case MLXCX_OP_QUERY_HCA_VPORT_CONTEXT:
240 		return ("MLXCX_OP_QUERY_HCA_VPORT_CONTEXT");
241 	case MLXCX_OP_MODIFY_HCA_VPORT_CONTEXT:
242 		return ("MLXCX_OP_MODIFY_HCA_VPORT_CONTEXT");
243 	case MLXCX_OP_QUERY_HCA_VPORT_GID:
244 		return ("MLXCX_OP_QUERY_HCA_VPORT_GID");
245 	case MLXCX_OP_QUERY_HCA_VPORT_PKEY:
246 		return ("MLXCX_OP_QUERY_HCA_VPORT_PKEY");
247 	case MLXCX_OP_QUERY_VPORT_COUNTER:
248 		return ("MLXCX_OP_QUERY_VPORT_COUNTER");
249 	case MLXCX_OP_ALLOC_Q_COUNTER:
250 		return ("MLXCX_OP_ALLOC_Q_COUNTER");
251 	case MLXCX_OP_DEALLOC_Q_COUNTER:
252 		return ("MLXCX_OP_DEALLOC_Q_COUNTER");
253 	case MLXCX_OP_QUERY_Q_COUNTER:
254 		return ("MLXCX_OP_QUERY_Q_COUNTER");
255 	case MLXCX_OP_SET_PP_RATE_LIMIT:
256 		return ("MLXCX_OP_SET_PP_RATE_LIMIT");
257 	case MLXCX_OP_QUERY_PP_RATE_LIMIT:
258 		return ("MLXCX_OP_QUERY_PP_RATE_LIMIT");
259 	case MLXCX_OP_ALLOC_PD:
260 		return ("MLXCX_OP_ALLOC_PD");
261 	case MLXCX_OP_DEALLOC_PD:
262 		return ("MLXCX_OP_DEALLOC_PD");
263 	case MLXCX_OP_ALLOC_UAR:
264 		return ("MLXCX_OP_ALLOC_UAR");
265 	case MLXCX_OP_DEALLOC_UAR:
266 		return ("MLXCX_OP_DEALLOC_UAR");
267 	case MLXCX_OP_CONFIG_INT_MODERATION:
268 		return ("MLXCX_OP_CONFIG_INT_MODERATION");
269 	case MLXCX_OP_ACCESS_REG:
270 		return ("MLXCX_OP_ACCESS_REG");
271 	case MLXCX_OP_ATTACH_TO_MCG:
272 		return ("MLXCX_OP_ATTACH_TO_MCG");
273 	case MLXCX_OP_DETACH_FROM_MCG:
274 		return ("MLXCX_OP_DETACH_FROM_MCG");
275 	case MLXCX_OP_MAD_IFC:
276 		return ("MLXCX_OP_MAD_IFC");
277 	case MLXCX_OP_QUERY_MAD_DEMUX:
278 		return ("MLXCX_OP_QUERY_MAD_DEMUX");
279 	case MLXCX_OP_SET_MAD_DEMUX:
280 		return ("MLXCX_OP_SET_MAD_DEMUX");
281 	case MLXCX_OP_NOP:
282 		return ("MLXCX_OP_NOP");
283 	case MLXCX_OP_ALLOC_XRCD:
284 		return ("MLXCX_OP_ALLOC_XRCD");
285 	case MLXCX_OP_DEALLOC_XRCD:
286 		return ("MLXCX_OP_DEALLOC_XRCD");
287 	case MLXCX_OP_ALLOC_TRANSPORT_DOMAIN:
288 		return ("MLXCX_OP_ALLOC_TRANSPORT_DOMAIN");
289 	case MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN:
290 		return ("MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN");
291 	case MLXCX_OP_QUERY_CONG_STATUS:
292 		return ("MLXCX_OP_QUERY_CONG_STATUS");
293 	case MLXCX_OP_MODIFY_CONG_STATUS:
294 		return ("MLXCX_OP_MODIFY_CONG_STATUS");
295 	case MLXCX_OP_QUERY_CONG_PARAMS:
296 		return ("MLXCX_OP_QUERY_CONG_PARAMS");
297 	case MLXCX_OP_MODIFY_CONG_PARAMS:
298 		return ("MLXCX_OP_MODIFY_CONG_PARAMS");
299 	case MLXCX_OP_QUERY_CONG_STATISTICS:
300 		return ("MLXCX_OP_QUERY_CONG_STATISTICS");
301 	case MLXCX_OP_ADD_VXLAN_UDP_DPORT:
302 		return ("MLXCX_OP_ADD_VXLAN_UDP_DPORT");
303 	case MLXCX_OP_DELETE_VXLAN_UDP_DPORT:
304 		return ("MLXCX_OP_DELETE_VXLAN_UDP_DPORT");
305 	case MLXCX_OP_SET_L2_TABLE_ENTRY:
306 		return ("MLXCX_OP_SET_L2_TABLE_ENTRY");
307 	case MLXCX_OP_QUERY_L2_TABLE_ENTRY:
308 		return ("MLXCX_OP_QUERY_L2_TABLE_ENTRY");
309 	case MLXCX_OP_DELETE_L2_TABLE_ENTRY:
310 		return ("MLXCX_OP_DELETE_L2_TABLE_ENTRY");
311 	case MLXCX_OP_SET_WOL_ROL:
312 		return ("MLXCX_OP_SET_WOL_ROL");
313 	case MLXCX_OP_QUERY_WOL_ROL:
314 		return ("MLXCX_OP_QUERY_WOL_ROL");
315 	case MLXCX_OP_CREATE_TIR:
316 		return ("MLXCX_OP_CREATE_TIR");
317 	case MLXCX_OP_MODIFY_TIR:
318 		return ("MLXCX_OP_MODIFY_TIR");
319 	case MLXCX_OP_DESTROY_TIR:
320 		return ("MLXCX_OP_DESTROY_TIR");
321 	case MLXCX_OP_QUERY_TIR:
322 		return ("MLXCX_OP_QUERY_TIR");
323 	case MLXCX_OP_CREATE_SQ:
324 		return ("MLXCX_OP_CREATE_SQ");
325 	case MLXCX_OP_MODIFY_SQ:
326 		return ("MLXCX_OP_MODIFY_SQ");
327 	case MLXCX_OP_DESTROY_SQ:
328 		return ("MLXCX_OP_DESTROY_SQ");
329 	case MLXCX_OP_QUERY_SQ:
330 		return ("MLXCX_OP_QUERY_SQ");
331 	case MLXCX_OP_CREATE_RQ:
332 		return ("MLXCX_OP_CREATE_RQ");
333 	case MLXCX_OP_MODIFY_RQ:
334 		return ("MLXCX_OP_MODIFY_RQ");
335 	case MLXCX_OP_DESTROY_RQ:
336 		return ("MLXCX_OP_DESTROY_RQ");
337 	case MLXCX_OP_QUERY_RQ:
338 		return ("MLXCX_OP_QUERY_RQ");
339 	case MLXCX_OP_CREATE_RMP:
340 		return ("MLXCX_OP_CREATE_RMP");
341 	case MLXCX_OP_MODIFY_RMP:
342 		return ("MLXCX_OP_MODIFY_RMP");
343 	case MLXCX_OP_DESTROY_RMP:
344 		return ("MLXCX_OP_DESTROY_RMP");
345 	case MLXCX_OP_QUERY_RMP:
346 		return ("MLXCX_OP_QUERY_RMP");
347 	case MLXCX_OP_CREATE_TIS:
348 		return ("MLXCX_OP_CREATE_TIS");
349 	case MLXCX_OP_MODIFY_TIS:
350 		return ("MLXCX_OP_MODIFY_TIS");
351 	case MLXCX_OP_DESTROY_TIS:
352 		return ("MLXCX_OP_DESTROY_TIS");
353 	case MLXCX_OP_QUERY_TIS:
354 		return ("MLXCX_OP_QUERY_TIS");
355 	case MLXCX_OP_CREATE_RQT:
356 		return ("MLXCX_OP_CREATE_RQT");
357 	case MLXCX_OP_MODIFY_RQT:
358 		return ("MLXCX_OP_MODIFY_RQT");
359 	case MLXCX_OP_DESTROY_RQT:
360 		return ("MLXCX_OP_DESTROY_RQT");
361 	case MLXCX_OP_QUERY_RQT:
362 		return ("MLXCX_OP_QUERY_RQT");
363 	case MLXCX_OP_SET_FLOW_TABLE_ROOT:
364 		return ("MLXCX_OP_SET_FLOW_TABLE_ROOT");
365 	case MLXCX_OP_CREATE_FLOW_TABLE:
366 		return ("MLXCX_OP_CREATE_FLOW_TABLE");
367 	case MLXCX_OP_DESTROY_FLOW_TABLE:
368 		return ("MLXCX_OP_DESTROY_FLOW_TABLE");
369 	case MLXCX_OP_QUERY_FLOW_TABLE:
370 		return ("MLXCX_OP_QUERY_FLOW_TABLE");
371 	case MLXCX_OP_CREATE_FLOW_GROUP:
372 		return ("MLXCX_OP_CREATE_FLOW_GROUP");
373 	case MLXCX_OP_DESTROY_FLOW_GROUP:
374 		return ("MLXCX_OP_DESTROY_FLOW_GROUP");
375 	case MLXCX_OP_QUERY_FLOW_GROUP:
376 		return ("MLXCX_OP_QUERY_FLOW_GROUP");
377 	case MLXCX_OP_SET_FLOW_TABLE_ENTRY:
378 		return ("MLXCX_OP_SET_FLOW_TABLE_ENTRY");
379 	case MLXCX_OP_QUERY_FLOW_TABLE_ENTRY:
380 		return ("MLXCX_OP_QUERY_FLOW_TABLE_ENTRY");
381 	case MLXCX_OP_DELETE_FLOW_TABLE_ENTRY:
382 		return ("MLXCX_OP_DELETE_FLOW_TABLE_ENTRY");
383 	case MLXCX_OP_ALLOC_FLOW_COUNTER:
384 		return ("MLXCX_OP_ALLOC_FLOW_COUNTER");
385 	case MLXCX_OP_DEALLOC_FLOW_COUNTER:
386 		return ("MLXCX_OP_DEALLOC_FLOW_COUNTER");
387 	case MLXCX_OP_QUERY_FLOW_COUNTER:
388 		return ("MLXCX_OP_QUERY_FLOW_COUNTER");
389 	case MLXCX_OP_MODIFY_FLOW_TABLE:
390 		return ("MLXCX_OP_MODIFY_FLOW_TABLE");
391 	case MLXCX_OP_ALLOC_ENCAP_HEADER:
392 		return ("MLXCX_OP_ALLOC_ENCAP_HEADER");
393 	case MLXCX_OP_DEALLOC_ENCAP_HEADER:
394 		return ("MLXCX_OP_DEALLOC_ENCAP_HEADER");
395 	case MLXCX_OP_QUERY_ENCAP_HEADER:
396 		return ("MLXCX_OP_QUERY_ENCAP_HEADER");
397 	default:
398 		return ("Unknown Opcode");
399 	}
400 }
401 
402 const char *
mlxcx_port_status_string(mlxcx_port_status_t st)403 mlxcx_port_status_string(mlxcx_port_status_t st)
404 {
405 	switch (st) {
406 	case MLXCX_PORT_STATUS_UP:
407 		return ("UP");
408 	case MLXCX_PORT_STATUS_DOWN:
409 		return ("DOWN");
410 	case MLXCX_PORT_STATUS_UP_ONCE:
411 		return ("UP_ONCE");
412 	case MLXCX_PORT_STATUS_DISABLED:
413 		return ("DISABLED");
414 	default:
415 		return ("UNKNOWN");
416 	}
417 }
418 
419 void
mlxcx_eth_proto_to_string(mlxcx_eth_proto_t p,mlxcx_ext_eth_proto_t ep,char * buf,size_t size)420 mlxcx_eth_proto_to_string(mlxcx_eth_proto_t p, mlxcx_ext_eth_proto_t ep,
421     char *buf, size_t size)
422 {
423 	if (p & MLXCX_PROTO_SGMII)
424 		(void) strlcat(buf, "SGMII|", size);
425 	if (p & MLXCX_PROTO_1000BASE_KX)
426 		(void) strlcat(buf, "1000BASE_KX|", size);
427 	if (p & MLXCX_PROTO_10GBASE_CX4)
428 		(void) strlcat(buf, "10GBASE_CX4|", size);
429 	if (p & MLXCX_PROTO_10GBASE_KX4)
430 		(void) strlcat(buf, "10GBASE_KX4|", size);
431 	if (p & MLXCX_PROTO_10GBASE_KR)
432 		(void) strlcat(buf, "10GBASE_KR|", size);
433 	if (p & MLXCX_PROTO_40GBASE_CR4)
434 		(void) strlcat(buf, "40GBASE_CR4|", size);
435 	if (p & MLXCX_PROTO_40GBASE_KR4)
436 		(void) strlcat(buf, "40GBASE_KR4|", size);
437 	if (p & MLXCX_PROTO_SGMII_100BASE)
438 		(void) strlcat(buf, "SGMII_100BASE|", size);
439 	if (p & MLXCX_PROTO_10GBASE_CR)
440 		(void) strlcat(buf, "10GBASE_CR|", size);
441 	if (p & MLXCX_PROTO_10GBASE_SR)
442 		(void) strlcat(buf, "10GBASE_SR|", size);
443 	if (p & MLXCX_PROTO_10GBASE_ER_LR)
444 		(void) strlcat(buf, "10GBASE_ER_LR|", size);
445 	if (p & MLXCX_PROTO_40GBASE_SR4)
446 		(void) strlcat(buf, "40GBASE_SR4|", size);
447 	if (p & MLXCX_PROTO_40GBASE_LR4_ER4)
448 		(void) strlcat(buf, "40GBASE_LR4_ER4|", size);
449 	if (p & MLXCX_PROTO_50GBASE_SR2)
450 		(void) strlcat(buf, "50GBASE_SR2|", size);
451 	if (p & MLXCX_PROTO_100GBASE_CR4)
452 		(void) strlcat(buf, "100GBASE_CR4|", size);
453 	if (p & MLXCX_PROTO_100GBASE_SR4)
454 		(void) strlcat(buf, "100GBASE_SR4|", size);
455 	if (p & MLXCX_PROTO_100GBASE_KR4)
456 		(void) strlcat(buf, "100GBASE_KR4|", size);
457 	if (p & MLXCX_PROTO_100GBASE_LR4_ER4)
458 		(void) strlcat(buf, "100GBASE_LR4_ER4|", size);
459 	if (p & MLXCX_PROTO_100BASE_TX)
460 		(void) strlcat(buf, "100BASE_TX|", size);
461 	if (p & MLXCX_PROTO_1000BASE_T)
462 		(void) strlcat(buf, "1000BASE_T|", size);
463 	if (p & MLXCX_PROTO_10GBASE_T)
464 		(void) strlcat(buf, "10GBASE_T|", size);
465 	if (p & MLXCX_PROTO_25GBASE_CR)
466 		(void) strlcat(buf, "25GBASE_CR|", size);
467 	if (p & MLXCX_PROTO_25GBASE_KR)
468 		(void) strlcat(buf, "25GBASE_KR|", size);
469 	if (p & MLXCX_PROTO_25GBASE_SR)
470 		(void) strlcat(buf, "25GBASE_SR|", size);
471 	if (p & MLXCX_PROTO_50GBASE_CR2)
472 		(void) strlcat(buf, "50GBASE_CR2|", size);
473 	if (p & MLXCX_PROTO_50GBASE_KR2)
474 		(void) strlcat(buf, "50GBASE_KR2|", size);
475 
476 	/* Now, for the extended bits... */
477 	if (ep & MLXCX_EXTPROTO_SGMII_100BASE)
478 		(void) strlcat(buf, "SGMII_100BASE|", size);
479 	if (ep & MLXCX_EXTPROTO_1000BASE_X_SGMII)
480 		(void) strlcat(buf, "1000BASE_X_SGMII|", size);
481 	if (ep & MLXCX_EXTPROTO_5GBASE_R)
482 		(void) strlcat(buf, "5GBASE_R|", size);
483 	if (ep & MLXCX_EXTPROTO_10GBASE_XFI_XAUI_1)
484 		(void) strlcat(buf, "10GBASE_XFI_XAUI_1|", size);
485 	if (ep & MLXCX_EXTPROTO_40GBASE_XLAUI_4_XLPPI_4)
486 		(void) strlcat(buf, "40GBASE_XLAUI_4_XLPPI_4|", size);
487 	if (ep & MLXCX_EXTPROTO_25GAUI_1_25GBASE_CR_KR)
488 		(void) strlcat(buf, "25GAUI_1_25GBASE_CR_KR|", size);
489 	if (ep & MLXCX_EXTPROTO_50GAUI_2_LAUI_2_50GBASE_CR2_KR2)
490 		(void) strlcat(buf, "50GAUI_2_LAUI_2_50GBASE_CR2_KR2|", size);
491 	if (ep & MLXCX_EXTPROTO_50GAUI_1_LAUI_1_50GBASE_CR_KR)
492 		(void) strlcat(buf, "50GAUI_1_LAUI_1_50GBASE_CR_KR|", size);
493 	if (ep & MLXCX_EXTPROTO_CAUI_4_100GBASE_CR4_KR4)
494 		(void) strlcat(buf, "CAUI_4_100GBASE_CR4_KR4|", size);
495 	if (ep & MLXCX_EXTPROTO_100GAUI_2_100GBASE_CR2_KR2)
496 		(void) strlcat(buf, "100GAUI_2_100GBASE_CR2_KR2|", size);
497 	if (ep & MLXCX_EXTPROTO_100GAUI_1_100GBASE_CR_KR)
498 		(void) strlcat(buf, "100GAUI_1_100GBASE_CR_KR|", size);
499 	/* Print these if we need 'em for debugging... */
500 	if (ep & MLXCX_EXTPROTO_200GAUI_4_200GBASE_CR4_KR4)
501 		(void) strlcat(buf, "200GAUI_4_200GBASE_CR4_KR4|", size);
502 	if (ep & MLXCX_EXTPROTO_200GAUI_2_200GBASE_CR2_KR2)
503 		(void) strlcat(buf, "200GAUI_2_200GBASE_CR2_KR2|", size);
504 	if (ep & MLXCX_EXTPROTO_400GAUI_8_400GBASE_CR8)
505 		(void) strlcat(buf, "400GAUI_8_400GBASE_CR8|", size);
506 	if (ep & MLXCX_EXTPROTO_400GAUI_4_400GBASE_CR4)
507 		(void) strlcat(buf, "400GAUI_4_400GBASE_CR4|", size);
508 
509 	/* Chop off the trailing '|' */
510 	if (strlen(buf) > 0)
511 		buf[strlen(buf) - 1] = '\0';
512 }
513 
514 void
mlxcx_cmd_queue_fini(mlxcx_t * mlxp)515 mlxcx_cmd_queue_fini(mlxcx_t *mlxp)
516 {
517 	mlxcx_cmd_queue_t *cmd = &mlxp->mlx_cmd;
518 
519 	if (cmd->mcmd_tokens != NULL) {
520 		id_space_destroy(cmd->mcmd_tokens);
521 		cmd->mcmd_tokens = NULL;
522 	}
523 
524 	if (cmd->mcmd_taskq != NULL) {
525 		ddi_taskq_destroy(cmd->mcmd_taskq);
526 		cmd->mcmd_taskq = NULL;
527 	}
528 
529 	cv_destroy(&cmd->mcmd_cv);
530 	mutex_destroy(&cmd->mcmd_lock);
531 
532 	mlxcx_dma_free(&cmd->mcmd_dma);
533 }
534 
535 boolean_t
mlxcx_cmd_queue_init(mlxcx_t * mlxp)536 mlxcx_cmd_queue_init(mlxcx_t *mlxp)
537 {
538 	uint32_t tmp, cmd_low, cmd_high, i;
539 	mlxcx_cmd_queue_t *cmd = &mlxp->mlx_cmd;
540 	char buf[32];
541 	char tq_name[TASKQ_NAMELEN];
542 	const ddi_dma_cookie_t *ck;
543 
544 	ddi_device_acc_attr_t acc;
545 	ddi_dma_attr_t attr;
546 
547 	tmp = mlxcx_get32(mlxp, MLXCX_ISS_FIRMWARE);
548 	mlxp->mlx_fw_maj = MLXCX_ISS_FW_MAJOR(tmp);
549 	mlxp->mlx_fw_min = MLXCX_ISS_FW_MINOR(tmp);
550 
551 	tmp = mlxcx_get32(mlxp, MLXCX_ISS_FW_CMD);
552 	mlxp->mlx_fw_rev = MLXCX_ISS_FW_REV(tmp);
553 	mlxp->mlx_cmd_rev = MLXCX_ISS_CMD_REV(tmp);
554 
555 	if (mlxp->mlx_cmd_rev != MLXCX_CMD_REVISION) {
556 		mlxcx_warn(mlxp, "found unsupported command revision: %u, "
557 		    "expected %u", mlxp->mlx_cmd_rev, MLXCX_CMD_REVISION);
558 		return (B_FALSE);
559 	}
560 
561 	cmd_low = mlxcx_get32(mlxp, MLXCX_ISS_CMD_LOW);
562 	cmd->mcmd_size_l2 = MLXCX_ISS_CMDQ_SIZE(cmd_low);
563 	cmd->mcmd_stride_l2 = MLXCX_ISS_CMDQ_STRIDE(cmd_low);
564 	cmd->mcmd_size = 1U << cmd->mcmd_size_l2;
565 
566 	if (cmd->mcmd_size > MLXCX_CMD_MAX) {
567 		mlxcx_warn(mlxp, "command queue size %u is too "
568 		    "large. Maximum is %u", cmd->mcmd_size, MLXCX_CMD_MAX);
569 		return (B_FALSE);
570 	}
571 
572 	cmd->mcmd_mask = (uint32_t)((1ULL << cmd->mcmd_size) - 1);
573 
574 	mutex_init(&cmd->mcmd_lock, NULL, MUTEX_DRIVER, NULL);
575 	cv_init(&cmd->mcmd_cv, NULL, CV_DRIVER, NULL);
576 
577 	(void) snprintf(buf, sizeof (buf), "mlxcx_tokens_%d", mlxp->mlx_inst);
578 	if ((cmd->mcmd_tokens = id_space_create(buf, 1, UINT8_MAX)) == NULL) {
579 		mlxcx_warn(mlxp, "failed to allocate token id space");
580 		mlxcx_cmd_queue_fini(mlxp);
581 		return (B_FALSE);
582 	}
583 
584 	(void) snprintf(tq_name, sizeof (tq_name), "cmdq_%d", mlxp->mlx_inst);
585 	if ((cmd->mcmd_taskq = ddi_taskq_create(mlxp->mlx_dip, tq_name, 1,
586 	    TASKQ_DEFAULTPRI, 0)) == NULL) {
587 		mlxcx_warn(mlxp, "failed to create command queue task queue");
588 		mlxcx_cmd_queue_fini(mlxp);
589 		return (B_FALSE);
590 	}
591 
592 	mlxcx_dma_acc_attr(mlxp, &acc);
593 	mlxcx_dma_page_attr(mlxp, &attr);
594 
595 	if (!mlxcx_dma_alloc(mlxp, &cmd->mcmd_dma, &attr, &acc, B_TRUE,
596 	    MLXCX_CMD_DMA_PAGE_SIZE, B_TRUE)) {
597 		mlxcx_warn(mlxp, "failed to allocate command dma buffer");
598 		mlxcx_cmd_queue_fini(mlxp);
599 		return (B_FALSE);
600 	}
601 
602 	ck = mlxcx_dma_cookie_one(&cmd->mcmd_dma);
603 	cmd_high = (uint32_t)(ck->dmac_laddress >> 32);
604 	cmd_low = (uint32_t)(ck->dmac_laddress & UINT32_MAX);
605 
606 	mlxcx_put32(mlxp, MLXCX_ISS_CMD_HIGH, cmd_high);
607 	mlxcx_put32(mlxp, MLXCX_ISS_CMD_LOW, cmd_low);
608 
609 	/*
610 	 * Before this is ready, the initializing bit must become zero.
611 	 */
612 	for (i = 0; i < mlxcx_cmd_init_trys; i++) {
613 		uint32_t init = mlxcx_get32(mlxp, MLXCX_ISS_INIT);
614 
615 		if (MLXCX_ISS_INITIALIZING(init) == 0)
616 			break;
617 		delay(drv_usectohz(mlxcx_cmd_init_delay));
618 	}
619 	if (i == mlxcx_cmd_init_trys) {
620 		mlxcx_warn(mlxp, "timed out initializing command queue");
621 		mlxcx_cmd_queue_fini(mlxp);
622 		return (B_FALSE);
623 	}
624 
625 	/*
626 	 * start in polling mode.
627 	 */
628 	mlxcx_cmd_eq_disable(mlxp);
629 
630 	return (B_TRUE);
631 }
632 
633 void
mlxcx_cmd_eq_enable(mlxcx_t * mlxp)634 mlxcx_cmd_eq_enable(mlxcx_t *mlxp)
635 {
636 	mlxp->mlx_cmd.mcmd_polled = B_FALSE;
637 }
638 
639 void
mlxcx_cmd_eq_disable(mlxcx_t * mlxp)640 mlxcx_cmd_eq_disable(mlxcx_t *mlxp)
641 {
642 	mlxp->mlx_cmd.mcmd_polled = B_TRUE;
643 }
644 
645 static void
mlxcx_cmd_in_header_init(mlxcx_cmd_t * cmd,mlxcx_cmd_in_t * in,mlxcx_cmd_op_t op,uint16_t mod)646 mlxcx_cmd_in_header_init(mlxcx_cmd_t *cmd, mlxcx_cmd_in_t *in,
647     mlxcx_cmd_op_t op, uint16_t mod)
648 {
649 	ASSERT3U(op, <=, UINT16_MAX);
650 	in->mci_opcode = to_be16(op);
651 	in->mci_op_mod = to_be16(mod);
652 	cmd->mlcmd_op = op;
653 }
654 
655 static boolean_t
mlxcx_cmd_mbox_alloc(mlxcx_t * mlxp,list_t * listp,uint8_t nblocks)656 mlxcx_cmd_mbox_alloc(mlxcx_t *mlxp, list_t *listp, uint8_t nblocks)
657 {
658 	uint8_t i;
659 	ddi_device_acc_attr_t acc;
660 	ddi_dma_attr_t attr;
661 
662 	mlxcx_dma_acc_attr(mlxp, &acc);
663 	mlxcx_dma_page_attr(mlxp, &attr);
664 
665 	for (i = 0; i < nblocks; i++) {
666 		mlxcx_cmd_mbox_t *mbox;
667 
668 		mbox = kmem_zalloc(sizeof (*mbox), KM_SLEEP);
669 		if (!mlxcx_dma_alloc(mlxp, &mbox->mlbox_dma, &attr, &acc,
670 		    B_TRUE, sizeof (mlxcx_cmd_mailbox_t), B_TRUE)) {
671 			mlxcx_warn(mlxp, "failed to allocate mailbox dma "
672 			    "buffer");
673 			kmem_free(mbox, sizeof (*mbox));
674 			/*
675 			 * mlxcx_cmd_fini will clean up any mboxes that we
676 			 * already placed onto listp.
677 			 */
678 			return (B_FALSE);
679 		}
680 		mbox->mlbox_data = (void *)mbox->mlbox_dma.mxdb_va;
681 		list_insert_tail(listp, mbox);
682 	}
683 
684 	return (B_TRUE);
685 }
686 
687 static void
mlxcx_cmd_mbox_free(mlxcx_cmd_mbox_t * mbox)688 mlxcx_cmd_mbox_free(mlxcx_cmd_mbox_t *mbox)
689 {
690 	mlxcx_dma_free(&mbox->mlbox_dma);
691 	kmem_free(mbox, sizeof (mlxcx_cmd_mbox_t));
692 }
693 
694 static void
mlxcx_cmd_fini(mlxcx_t * mlxp,mlxcx_cmd_t * cmd)695 mlxcx_cmd_fini(mlxcx_t *mlxp, mlxcx_cmd_t *cmd)
696 {
697 	mlxcx_cmd_mbox_t *mbox;
698 
699 	while ((mbox = list_remove_head(&cmd->mlcmd_mbox_out)) != NULL) {
700 		mlxcx_cmd_mbox_free(mbox);
701 	}
702 	list_destroy(&cmd->mlcmd_mbox_out);
703 	while ((mbox = list_remove_head(&cmd->mlcmd_mbox_in)) != NULL) {
704 		mlxcx_cmd_mbox_free(mbox);
705 	}
706 	list_destroy(&cmd->mlcmd_mbox_in);
707 	id_free(mlxp->mlx_cmd.mcmd_tokens, cmd->mlcmd_token);
708 	cv_destroy(&cmd->mlcmd_cv);
709 	mutex_destroy(&cmd->mlcmd_lock);
710 }
711 
712 static void
mlxcx_cmd_init(mlxcx_t * mlxp,mlxcx_cmd_t * cmd)713 mlxcx_cmd_init(mlxcx_t *mlxp, mlxcx_cmd_t *cmd)
714 {
715 	bzero(cmd, sizeof (*cmd));
716 	mutex_init(&cmd->mlcmd_lock, NULL, MUTEX_DRIVER,
717 	    DDI_INTR_PRI(mlxp->mlx_async_intr_pri));
718 	cv_init(&cmd->mlcmd_cv, NULL, CV_DRIVER, NULL);
719 	cmd->mlcmd_token = id_alloc(mlxp->mlx_cmd.mcmd_tokens);
720 	cmd->mlcmd_poll = mlxp->mlx_cmd.mcmd_polled;
721 	list_create(&cmd->mlcmd_mbox_in, sizeof (mlxcx_cmd_mbox_t),
722 	    offsetof(mlxcx_cmd_mbox_t, mlbox_node));
723 	list_create(&cmd->mlcmd_mbox_out, sizeof (mlxcx_cmd_mbox_t),
724 	    offsetof(mlxcx_cmd_mbox_t, mlbox_node));
725 }
726 
727 static void
mlxcx_cmd_prep_input(mlxcx_cmd_ent_t * ent,mlxcx_cmd_t * cmd)728 mlxcx_cmd_prep_input(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd)
729 {
730 	uint32_t rem = cmd->mlcmd_inlen;
731 	uint8_t i;
732 	const void *in = cmd->mlcmd_in;
733 	uint32_t copy;
734 	mlxcx_cmd_mbox_t *mbox;
735 	const ddi_dma_cookie_t *ck;
736 
737 	copy = MIN(MLXCX_CMD_INLINE_INPUT_LEN, rem);
738 	bcopy(in, ent->mce_input, copy);
739 
740 	rem -= copy;
741 	in += copy;
742 
743 	if (rem == 0) {
744 		ent->mce_in_mbox = to_be64(0);
745 		VERIFY3U(cmd->mlcmd_nboxes_in, ==, 0);
746 		return;
747 	}
748 
749 	mbox = list_head(&cmd->mlcmd_mbox_in);
750 	ck = mlxcx_dma_cookie_one(&mbox->mlbox_dma);
751 	ent->mce_in_mbox = to_be64(ck->dmac_laddress);
752 	for (i = 0; mbox != NULL;
753 	    mbox = list_next(&cmd->mlcmd_mbox_in, mbox), i++) {
754 		mlxcx_cmd_mbox_t *next;
755 		mlxcx_cmd_mailbox_t *mp = mbox->mlbox_data;
756 
757 		copy = MIN(MLXCX_CMD_MAILBOX_LEN, rem);
758 		bcopy(in, mp->mlxb_data, copy);
759 		rem -= copy;
760 		in += copy;
761 
762 		mp->mlxb_token = cmd->mlcmd_token;
763 		mp->mlxb_blockno = to_be32(i);
764 
765 		next = list_next(&cmd->mlcmd_mbox_in, mbox);
766 		if (next == NULL) {
767 			mp->mlxb_nextp = to_be64(0);
768 		} else {
769 			ck = mlxcx_dma_cookie_one(&next->mlbox_dma);
770 			mp->mlxb_nextp = to_be64(ck->dmac_laddress);
771 		}
772 		MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORDEV);
773 	}
774 	VERIFY3U(i, ==, cmd->mlcmd_nboxes_in);
775 	VERIFY0(rem);
776 }
777 
778 static void
mlxcx_cmd_prep_output(mlxcx_cmd_ent_t * ent,mlxcx_cmd_t * cmd)779 mlxcx_cmd_prep_output(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd)
780 {
781 	uint8_t i;
782 	mlxcx_cmd_mbox_t *mbox;
783 	const ddi_dma_cookie_t *ck;
784 
785 	if (cmd->mlcmd_nboxes_out == 0) {
786 		ent->mce_out_mbox = to_be64(0);
787 		return;
788 	}
789 
790 	mbox = list_head(&cmd->mlcmd_mbox_out);
791 	ck = mlxcx_dma_cookie_one(&mbox->mlbox_dma);
792 	ent->mce_out_mbox = to_be64(ck->dmac_laddress);
793 	for (i = 0, mbox = list_head(&cmd->mlcmd_mbox_out); mbox != NULL;
794 	    mbox = list_next(&cmd->mlcmd_mbox_out, mbox), i++) {
795 		mlxcx_cmd_mbox_t *next;
796 		mlxcx_cmd_mailbox_t *mp = mbox->mlbox_data;
797 
798 		mp->mlxb_token = cmd->mlcmd_token;
799 		mp->mlxb_blockno = to_be32(i);
800 
801 		next = list_next(&cmd->mlcmd_mbox_out, mbox);
802 		if (next == NULL) {
803 			mp->mlxb_nextp = to_be64(0);
804 		} else {
805 			ck = mlxcx_dma_cookie_one(&next->mlbox_dma);
806 			mp->mlxb_nextp = to_be64(ck->dmac_laddress);
807 		}
808 		MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORDEV);
809 	}
810 	VERIFY3U(i, ==, cmd->mlcmd_nboxes_out);
811 }
812 
813 static void
mlxcx_cmd_copy_output(mlxcx_cmd_ent_t * ent,mlxcx_cmd_t * cmd)814 mlxcx_cmd_copy_output(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd)
815 {
816 	void *out = cmd->mlcmd_out;
817 	uint32_t rem = cmd->mlcmd_outlen;
818 	uint32_t copy;
819 	mlxcx_cmd_mbox_t *mbox;
820 
821 	copy = MIN(rem, MLXCX_CMD_INLINE_OUTPUT_LEN);
822 	bcopy(ent->mce_output, out, copy);
823 	out += copy;
824 	rem -= copy;
825 
826 	if (rem == 0) {
827 		VERIFY0(cmd->mlcmd_nboxes_out);
828 		return;
829 	}
830 
831 	for (mbox = list_head(&cmd->mlcmd_mbox_out); mbox != NULL;
832 	    mbox = list_next(&cmd->mlcmd_mbox_out, mbox)) {
833 		MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORKERNEL);
834 		copy = MIN(MLXCX_CMD_MAILBOX_LEN, rem);
835 		bcopy(mbox->mlbox_data->mlxb_data, out, copy);
836 		out += copy;
837 		rem -= copy;
838 	}
839 	VERIFY0(rem);
840 }
841 
842 static uint_t
mlxcx_cmd_reserve_slot(mlxcx_cmd_queue_t * cmdq)843 mlxcx_cmd_reserve_slot(mlxcx_cmd_queue_t *cmdq)
844 {
845 	uint_t slot;
846 
847 	mutex_enter(&cmdq->mcmd_lock);
848 	slot = ddi_ffs(cmdq->mcmd_mask);
849 	while (slot == 0) {
850 		cv_wait(&cmdq->mcmd_cv, &cmdq->mcmd_lock);
851 		slot = ddi_ffs(cmdq->mcmd_mask);
852 	}
853 
854 	cmdq->mcmd_mask &= ~(1U << --slot);
855 
856 	ASSERT3P(cmdq->mcmd_active[slot], ==, NULL);
857 
858 	mutex_exit(&cmdq->mcmd_lock);
859 
860 	return (slot);
861 }
862 
863 static void
mlxcx_cmd_release_slot(mlxcx_cmd_queue_t * cmdq,uint_t slot)864 mlxcx_cmd_release_slot(mlxcx_cmd_queue_t *cmdq, uint_t slot)
865 {
866 	mutex_enter(&cmdq->mcmd_lock);
867 	cmdq->mcmd_mask |= 1U << slot;
868 	cv_broadcast(&cmdq->mcmd_cv);
869 	mutex_exit(&cmdq->mcmd_lock);
870 }
871 
872 static void
mlxcx_cmd_done(mlxcx_cmd_t * cmd,uint_t slot)873 mlxcx_cmd_done(mlxcx_cmd_t *cmd, uint_t slot)
874 {
875 	mlxcx_t *mlxp = cmd->mlcmd_mlxp;
876 	mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd;
877 	mlxcx_cmd_ent_t *ent;
878 
879 	/*
880 	 * Command is done. Save relevant data. Once we broadcast on the CV and
881 	 * drop the lock, we must not touch it again.
882 	 */
883 	MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORKERNEL);
884 
885 	ent = (mlxcx_cmd_ent_t *)(cmdq->mcmd_dma.mxdb_va +
886 	    (slot << cmdq->mcmd_stride_l2));
887 
888 	mutex_enter(&cmd->mlcmd_lock);
889 	cmd->mlcmd_status = MLXCX_CMD_STATUS(ent->mce_status);
890 	if (cmd->mlcmd_status == 0)
891 		mlxcx_cmd_copy_output(ent, cmd);
892 
893 	cmd->mlcmd_state = MLXCX_CMD_S_DONE;
894 	cv_broadcast(&cmd->mlcmd_cv);
895 	mutex_exit(&cmd->mlcmd_lock);
896 
897 	cmdq->mcmd_active[slot] = NULL;
898 	mlxcx_cmd_release_slot(cmdq, slot);
899 }
900 
901 static void
mlxcx_cmd_taskq(void * arg)902 mlxcx_cmd_taskq(void *arg)
903 {
904 	mlxcx_cmd_t *cmd = arg;
905 	mlxcx_t *mlxp = cmd->mlcmd_mlxp;
906 	mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd;
907 	mlxcx_cmd_ent_t *ent;
908 	uint_t poll, slot;
909 
910 	ASSERT3S(cmd->mlcmd_op, !=, 0);
911 
912 	slot = mlxcx_cmd_reserve_slot(cmdq);
913 	ent = (mlxcx_cmd_ent_t *)(cmdq->mcmd_dma.mxdb_va +
914 	    (slot << cmdq->mcmd_stride_l2));
915 
916 	cmdq->mcmd_active[slot] = cmd;
917 
918 	/*
919 	 * Command queue is currently ours as we set busy.
920 	 */
921 	bzero(ent, sizeof (*ent));
922 	ent->mce_type = MLXCX_CMD_TRANSPORT_PCI;
923 	ent->mce_in_length = to_be32(cmd->mlcmd_inlen);
924 	ent->mce_out_length = to_be32(cmd->mlcmd_outlen);
925 	ent->mce_token = cmd->mlcmd_token;
926 	ent->mce_sig = 0;
927 	ent->mce_status = MLXCX_CMD_HW_OWNED;
928 	mlxcx_cmd_prep_input(ent, cmd);
929 	mlxcx_cmd_prep_output(ent, cmd);
930 	MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORDEV);
931 
932 	mlxcx_put32(mlxp, MLXCX_ISS_CMD_DOORBELL, 1 << slot);
933 
934 	if (!cmd->mlcmd_poll)
935 		return;
936 
937 	for (poll = 0; poll < mlxcx_cmd_tries; poll++) {
938 		delay(drv_usectohz(mlxcx_cmd_delay));
939 		MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORKERNEL);
940 		if ((ent->mce_status & MLXCX_CMD_HW_OWNED) == 0)
941 			break;
942 	}
943 
944 	/*
945 	 * Command is done (or timed out). Save relevant data. Once we broadcast
946 	 * on the CV and drop the lock, we must not touch the cmd again.
947 	 */
948 
949 	if (poll == mlxcx_cmd_tries) {
950 		mutex_enter(&cmd->mlcmd_lock);
951 		cmd->mlcmd_status = MLXCX_CMD_R_TIMEOUT;
952 		cmd->mlcmd_state = MLXCX_CMD_S_ERROR;
953 		cv_broadcast(&cmd->mlcmd_cv);
954 		mutex_exit(&cmd->mlcmd_lock);
955 
956 		mlxcx_fm_ereport(mlxp, DDI_FM_DEVICE_NO_RESPONSE);
957 
958 		cmdq->mcmd_active[slot] = NULL;
959 		mlxcx_cmd_release_slot(cmdq, slot);
960 
961 		return;
962 	}
963 
964 	mlxcx_cmd_done(cmd, slot);
965 }
966 
967 void
mlxcx_cmd_completion(mlxcx_t * mlxp,mlxcx_eventq_ent_t * ent)968 mlxcx_cmd_completion(mlxcx_t *mlxp, mlxcx_eventq_ent_t *ent)
969 {
970 	mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd;
971 	mlxcx_evdata_cmd_completion_t *eqe_cmd = &ent->mleqe_cmd_completion;
972 	mlxcx_cmd_t *cmd;
973 	uint32_t comp_vec = from_be32(eqe_cmd->mled_cmd_completion_vec);
974 	uint_t slot;
975 
976 	DTRACE_PROBE2(cmd_event, mlxcx_t *, mlxp,
977 	    mlxcx_evdata_cmd_completion_t *, eqe_cmd);
978 
979 	while ((slot = ddi_ffs(comp_vec)) != 0) {
980 		comp_vec &= ~(1U << --slot);
981 
982 		cmd = cmdq->mcmd_active[slot];
983 		if (cmd->mlcmd_poll)
984 			continue;
985 
986 		mlxcx_cmd_done(cmd, slot);
987 	}
988 }
989 
990 static boolean_t
mlxcx_cmd_send(mlxcx_t * mlxp,mlxcx_cmd_t * cmd,const void * in,uint32_t inlen,void * out,uint32_t outlen)991 mlxcx_cmd_send(mlxcx_t *mlxp, mlxcx_cmd_t *cmd, const void *in, uint32_t inlen,
992     void *out, uint32_t outlen)
993 {
994 	if (inlen > MLXCX_CMD_INLINE_INPUT_LEN) {
995 		uint32_t need = inlen - MLXCX_CMD_INLINE_INPUT_LEN;
996 		uint8_t nblocks;
997 
998 		if (need / MLXCX_CMD_MAILBOX_LEN + 1 > UINT8_MAX) {
999 			mlxcx_warn(mlxp, "requested too many input blocks for "
1000 			    "%u byte input len", inlen);
1001 			return (B_FALSE);
1002 		}
1003 
1004 		nblocks = need / MLXCX_CMD_MAILBOX_LEN + 1;
1005 		if (!mlxcx_cmd_mbox_alloc(mlxp, &cmd->mlcmd_mbox_in, nblocks)) {
1006 			mlxcx_warn(mlxp, "failed to allocate %u blocks of "
1007 			    "input mailbox", nblocks);
1008 			return (B_FALSE);
1009 		}
1010 		cmd->mlcmd_nboxes_in = nblocks;
1011 	}
1012 
1013 	if (outlen > MLXCX_CMD_INLINE_OUTPUT_LEN) {
1014 		uint32_t need = outlen - MLXCX_CMD_INLINE_OUTPUT_LEN;
1015 		uint8_t nblocks;
1016 
1017 		if (need / MLXCX_CMD_MAILBOX_LEN + 1 > UINT8_MAX) {
1018 			mlxcx_warn(mlxp, "requested too many output blocks for "
1019 			    "%u byte output len", outlen);
1020 			return (B_FALSE);
1021 		}
1022 
1023 		nblocks = need / MLXCX_CMD_MAILBOX_LEN + 1;
1024 		if (!mlxcx_cmd_mbox_alloc(mlxp, &cmd->mlcmd_mbox_out,
1025 		    nblocks)) {
1026 			mlxcx_warn(mlxp, "failed to allocate %u blocks of "
1027 			    "output mailbox", nblocks);
1028 			return (B_FALSE);
1029 		}
1030 		cmd->mlcmd_nboxes_out = nblocks;
1031 	}
1032 
1033 	cmd->mlcmd_in = in;
1034 	cmd->mlcmd_inlen = inlen;
1035 	cmd->mlcmd_out = out;
1036 	cmd->mlcmd_outlen = outlen;
1037 	cmd->mlcmd_mlxp = mlxp;
1038 
1039 	/*
1040 	 * Now that all allocations have been done, all that remains is for us
1041 	 * to dispatch the request to process this to the taskq for it to be
1042 	 * processed.
1043 	 */
1044 	if (ddi_taskq_dispatch(mlxp->mlx_cmd.mcmd_taskq, mlxcx_cmd_taskq, cmd,
1045 	    DDI_SLEEP) != DDI_SUCCESS) {
1046 		mlxcx_warn(mlxp, "failed to submit command to taskq");
1047 		return (B_FALSE);
1048 	}
1049 
1050 	return (B_TRUE);
1051 }
1052 
1053 static void
mlxcx_cmd_wait(mlxcx_cmd_t * cmd)1054 mlxcx_cmd_wait(mlxcx_cmd_t *cmd)
1055 {
1056 	mutex_enter(&cmd->mlcmd_lock);
1057 	while (cmd->mlcmd_state == 0) {
1058 		cv_wait(&cmd->mlcmd_cv, &cmd->mlcmd_lock);
1059 	}
1060 	mutex_exit(&cmd->mlcmd_lock);
1061 }
1062 
1063 static boolean_t
mlxcx_cmd_evaluate(mlxcx_t * mlxp,mlxcx_cmd_t * cmd)1064 mlxcx_cmd_evaluate(mlxcx_t *mlxp, mlxcx_cmd_t *cmd)
1065 {
1066 	mlxcx_cmd_out_t *out;
1067 
1068 	if ((cmd->mlcmd_state & MLXCX_CMD_S_ERROR) != 0) {
1069 		mlxcx_warn(mlxp, "command %s (0x%x) failed due to an internal "
1070 		    "driver error",
1071 		    mlxcx_cmd_opcode_string(cmd->mlcmd_op),
1072 		    cmd->mlcmd_op);
1073 		return (B_FALSE);
1074 	}
1075 
1076 	if (cmd->mlcmd_status != 0) {
1077 		mlxcx_warn(mlxp, "command %s (0x%x) failed with command queue "
1078 		    "error 0x%x",
1079 		    mlxcx_cmd_opcode_string(cmd->mlcmd_op),
1080 		    cmd->mlcmd_op, cmd->mlcmd_status);
1081 		return (B_FALSE);
1082 	}
1083 
1084 	out = cmd->mlcmd_out;
1085 	if (out->mco_status != MLXCX_CMD_R_OK) {
1086 		mlxcx_warn(mlxp, "command %s 0x%x failed with status code %s "
1087 		    "(0x%x)", mlxcx_cmd_opcode_string(cmd->mlcmd_op),
1088 		    cmd->mlcmd_op, mlxcx_cmd_response_string(out->mco_status),
1089 		    out->mco_status);
1090 		return (B_FALSE);
1091 	}
1092 
1093 	return (B_TRUE);
1094 }
1095 
1096 boolean_t
mlxcx_cmd_disable_hca(mlxcx_t * mlxp)1097 mlxcx_cmd_disable_hca(mlxcx_t *mlxp)
1098 {
1099 	mlxcx_cmd_t cmd;
1100 	mlxcx_cmd_disable_hca_in_t in;
1101 	mlxcx_cmd_disable_hca_out_t out;
1102 	boolean_t ret;
1103 
1104 	bzero(&in, sizeof (in));
1105 	bzero(&out, sizeof (out));
1106 
1107 	mlxcx_cmd_init(mlxp, &cmd);
1108 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_disable_hca_head,
1109 	    MLXCX_OP_DISABLE_HCA, 0);
1110 	in.mlxi_disable_hca_func = MLXCX_FUNCTION_SELF;
1111 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1112 		mlxcx_cmd_fini(mlxp, &cmd);
1113 		return (B_FALSE);
1114 	}
1115 	mlxcx_cmd_wait(&cmd);
1116 
1117 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1118 	mlxcx_cmd_fini(mlxp, &cmd);
1119 	return (ret);
1120 }
1121 
1122 boolean_t
mlxcx_cmd_enable_hca(mlxcx_t * mlxp)1123 mlxcx_cmd_enable_hca(mlxcx_t *mlxp)
1124 {
1125 	mlxcx_cmd_t cmd;
1126 	mlxcx_cmd_enable_hca_in_t in;
1127 	mlxcx_cmd_enable_hca_out_t out;
1128 	boolean_t ret;
1129 
1130 	bzero(&in, sizeof (in));
1131 	bzero(&out, sizeof (out));
1132 
1133 	mlxcx_cmd_init(mlxp, &cmd);
1134 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_enable_hca_head,
1135 	    MLXCX_OP_ENABLE_HCA, 0);
1136 	in.mlxi_enable_hca_func = MLXCX_FUNCTION_SELF;
1137 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1138 		mlxcx_cmd_fini(mlxp, &cmd);
1139 		return (B_FALSE);
1140 	}
1141 	mlxcx_cmd_wait(&cmd);
1142 
1143 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1144 	mlxcx_cmd_fini(mlxp, &cmd);
1145 	return (ret);
1146 }
1147 
1148 boolean_t
mlxcx_cmd_query_issi(mlxcx_t * mlxp,uint32_t * issip)1149 mlxcx_cmd_query_issi(mlxcx_t *mlxp, uint32_t *issip)
1150 {
1151 	mlxcx_cmd_t cmd;
1152 	mlxcx_cmd_query_issi_in_t in;
1153 	mlxcx_cmd_query_issi_out_t out;
1154 	boolean_t ret;
1155 
1156 	bzero(&in, sizeof (in));
1157 	bzero(&out, sizeof (out));
1158 
1159 	mlxcx_cmd_init(mlxp, &cmd);
1160 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_issi_head,
1161 	    MLXCX_OP_QUERY_ISSI, 0);
1162 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1163 		mlxcx_cmd_fini(mlxp, &cmd);
1164 		return (B_FALSE);
1165 	}
1166 	mlxcx_cmd_wait(&cmd);
1167 
1168 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1169 	if (ret) {
1170 		*issip = out.mlxo_supported_issi;
1171 	} else if (cmd.mlcmd_status == 0 &&
1172 	    out.mlxo_query_issi_head.mco_status == MLXCX_CMD_R_BAD_OP) {
1173 		/*
1174 		 * The PRM says that if we get a bad operation, that means this
1175 		 * command isn't supported so it only supports version 1 of the
1176 		 * ISSI, which means bit zero should be set.
1177 		 */
1178 		ret = B_TRUE;
1179 		*issip = 1;
1180 	}
1181 	mlxcx_cmd_fini(mlxp, &cmd);
1182 	return (ret);
1183 }
1184 
1185 boolean_t
mlxcx_cmd_set_issi(mlxcx_t * mlxp,uint16_t issi)1186 mlxcx_cmd_set_issi(mlxcx_t *mlxp, uint16_t issi)
1187 {
1188 	mlxcx_cmd_t cmd;
1189 	mlxcx_cmd_set_issi_in_t in;
1190 	mlxcx_cmd_set_issi_out_t out;
1191 	boolean_t ret;
1192 
1193 	bzero(&in, sizeof (in));
1194 	bzero(&out, sizeof (out));
1195 
1196 	mlxcx_cmd_init(mlxp, &cmd);
1197 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_issi_head,
1198 	    MLXCX_OP_SET_ISSI, 0);
1199 	in.mlxi_set_issi_current = to_be16(issi);
1200 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1201 		mlxcx_cmd_fini(mlxp, &cmd);
1202 		return (B_FALSE);
1203 	}
1204 	mlxcx_cmd_wait(&cmd);
1205 
1206 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1207 	mlxcx_cmd_fini(mlxp, &cmd);
1208 	return (ret);
1209 }
1210 
1211 boolean_t
mlxcx_cmd_query_pages(mlxcx_t * mlxp,uint_t type,int32_t * npages)1212 mlxcx_cmd_query_pages(mlxcx_t *mlxp, uint_t type, int32_t *npages)
1213 {
1214 	mlxcx_cmd_t cmd;
1215 	mlxcx_cmd_query_pages_in_t in;
1216 	mlxcx_cmd_query_pages_out_t out;
1217 	boolean_t ret;
1218 
1219 	switch (type) {
1220 	case MLXCX_QUERY_PAGES_OPMOD_BOOT:
1221 	case MLXCX_QUERY_PAGES_OPMOD_INIT:
1222 	case MLXCX_QUERY_PAGES_OPMOD_REGULAR:
1223 		break;
1224 	default:
1225 		mlxcx_warn(mlxp, "!passed invalid type to query pages: %u",
1226 		    type);
1227 		return (B_FALSE);
1228 	}
1229 
1230 	bzero(&in, sizeof (in));
1231 	bzero(&out, sizeof (out));
1232 
1233 	mlxcx_cmd_init(mlxp, &cmd);
1234 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_pages_head,
1235 	    MLXCX_OP_QUERY_PAGES, type);
1236 	in.mlxi_query_pages_func = MLXCX_FUNCTION_SELF;
1237 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1238 		mlxcx_cmd_fini(mlxp, &cmd);
1239 		return (B_FALSE);
1240 	}
1241 	mlxcx_cmd_wait(&cmd);
1242 
1243 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1244 	if (ret) {
1245 		*npages = from_be32(out.mlxo_query_pages_npages);
1246 	}
1247 	mlxcx_cmd_fini(mlxp, &cmd);
1248 
1249 	return (ret);
1250 }
1251 
1252 boolean_t
mlxcx_cmd_give_pages(mlxcx_t * mlxp,uint_t type,int32_t npages,mlxcx_dev_page_t ** pages)1253 mlxcx_cmd_give_pages(mlxcx_t *mlxp, uint_t type, int32_t npages,
1254     mlxcx_dev_page_t **pages)
1255 {
1256 	mlxcx_cmd_t cmd;
1257 	mlxcx_cmd_manage_pages_in_t *in;
1258 	mlxcx_cmd_manage_pages_out_t out;
1259 	size_t insize, outsize;
1260 	boolean_t ret;
1261 	uint32_t i;
1262 	uint64_t pa;
1263 	const ddi_dma_cookie_t *ck;
1264 
1265 	switch (type) {
1266 	case MLXCX_MANAGE_PAGES_OPMOD_ALLOC_FAIL:
1267 		if (npages != 0) {
1268 			mlxcx_warn(mlxp, "passed non-zero number of pages (%d) "
1269 			    "but asked to fail page allocation", npages);
1270 			return (B_FALSE);
1271 		}
1272 		break;
1273 	case MLXCX_MANAGE_PAGES_OPMOD_GIVE_PAGES:
1274 		ASSERT3S(npages, <=, MLXCX_MANAGE_PAGES_MAX_PAGES);
1275 		if (npages <= 0) {
1276 			mlxcx_warn(mlxp, "passed invalid number of pages (%d) "
1277 			    "to give pages", npages);
1278 			return (B_FALSE);
1279 		}
1280 		break;
1281 	default:
1282 		mlxcx_warn(mlxp, "!passed invalid type to give pages: %u",
1283 		    type);
1284 		return (B_FALSE);
1285 	}
1286 
1287 	insize = offsetof(mlxcx_cmd_manage_pages_in_t, mlxi_manage_pages_pas) +
1288 	    npages * sizeof (uint64_t);
1289 	outsize = offsetof(mlxcx_cmd_manage_pages_out_t, mlxo_manage_pages_pas);
1290 
1291 	in = kmem_zalloc(insize, KM_SLEEP);
1292 	bzero(&out, sizeof (out));
1293 
1294 	mlxcx_cmd_init(mlxp, &cmd);
1295 	mlxcx_cmd_in_header_init(&cmd, &in->mlxi_manage_pages_head,
1296 	    MLXCX_OP_MANAGE_PAGES, type);
1297 	in->mlxi_manage_pages_func = MLXCX_FUNCTION_SELF;
1298 	in->mlxi_manage_pages_npages = to_be32(npages);
1299 	for (i = 0; i < npages; i++) {
1300 		ck = mlxcx_dma_cookie_one(&pages[i]->mxdp_dma);
1301 		pa = ck->dmac_laddress;
1302 		ASSERT3U(pa & 0xfff, ==, 0);
1303 		ASSERT3U(ck->dmac_size, ==, MLXCX_HW_PAGE_SIZE);
1304 		in->mlxi_manage_pages_pas[i] = to_be64(pa);
1305 	}
1306 
1307 	if ((ret = mlxcx_cmd_send(mlxp, &cmd, in, insize, &out, outsize))) {
1308 		mlxcx_cmd_wait(&cmd);
1309 		ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1310 	}
1311 
1312 	mlxcx_cmd_fini(mlxp, &cmd);
1313 
1314 	kmem_free(in, insize);
1315 	return (ret);
1316 }
1317 
1318 boolean_t
mlxcx_cmd_return_pages(mlxcx_t * mlxp,int32_t nreq,uint64_t * pas,int32_t * nret)1319 mlxcx_cmd_return_pages(mlxcx_t *mlxp, int32_t nreq, uint64_t *pas,
1320     int32_t *nret)
1321 {
1322 	mlxcx_cmd_t cmd;
1323 	mlxcx_cmd_manage_pages_in_t in;
1324 	mlxcx_cmd_manage_pages_out_t *out;
1325 	size_t insize, outsize;
1326 	boolean_t ret;
1327 	uint32_t i;
1328 
1329 	if (nreq <= 0) {
1330 		mlxcx_warn(mlxp, "passed invalid number of pages (%d) "
1331 		    "to return pages", nreq);
1332 		return (B_FALSE);
1333 	}
1334 
1335 	insize = offsetof(mlxcx_cmd_manage_pages_in_t, mlxi_manage_pages_pas);
1336 	outsize = offsetof(mlxcx_cmd_manage_pages_out_t,
1337 	    mlxo_manage_pages_pas) + nreq * sizeof (uint64_t);
1338 
1339 	bzero(&in, sizeof (in));
1340 	out = kmem_alloc(outsize, KM_SLEEP);
1341 
1342 	mlxcx_cmd_init(mlxp, &cmd);
1343 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_manage_pages_head,
1344 	    MLXCX_OP_MANAGE_PAGES, MLXCX_MANAGE_PAGES_OPMOD_RETURN_PAGES);
1345 	in.mlxi_manage_pages_func = MLXCX_FUNCTION_SELF;
1346 	in.mlxi_manage_pages_npages = to_be32(nreq);
1347 
1348 	if ((ret = mlxcx_cmd_send(mlxp, &cmd, &in, insize, out, outsize))) {
1349 		mlxcx_cmd_wait(&cmd);
1350 
1351 		ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1352 		if (ret) {
1353 			*nret = from_be32(out->mlxo_manage_pages_npages);
1354 			for (i = 0; i < *nret; i++) {
1355 				pas[i] =
1356 				    from_be64(out->mlxo_manage_pages_pas[i]);
1357 			}
1358 		}
1359 	}
1360 
1361 	mlxcx_cmd_fini(mlxp, &cmd);
1362 
1363 	kmem_free(out, outsize);
1364 	return (ret);
1365 }
1366 
1367 boolean_t
mlxcx_cmd_query_hca_cap(mlxcx_t * mlxp,mlxcx_hca_cap_type_t type,mlxcx_hca_cap_mode_t mode,mlxcx_hca_cap_t * capp)1368 mlxcx_cmd_query_hca_cap(mlxcx_t *mlxp, mlxcx_hca_cap_type_t type,
1369     mlxcx_hca_cap_mode_t mode, mlxcx_hca_cap_t *capp)
1370 {
1371 	mlxcx_cmd_t cmd;
1372 	mlxcx_cmd_query_hca_cap_in_t in;
1373 	mlxcx_cmd_query_hca_cap_out_t *out;
1374 	boolean_t ret;
1375 	uint16_t opmode;
1376 
1377 	bzero(&in, sizeof (in));
1378 	out = kmem_zalloc(sizeof (mlxcx_cmd_query_hca_cap_out_t), KM_SLEEP);
1379 	mlxcx_cmd_init(mlxp, &cmd);
1380 
1381 	opmode = type << 1 | mode;
1382 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_hca_cap_head,
1383 	    MLXCX_OP_QUERY_HCA_CAP, opmode);
1384 
1385 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), out, sizeof (*out))) {
1386 		mlxcx_cmd_fini(mlxp, &cmd);
1387 		kmem_free(out, sizeof (mlxcx_cmd_query_hca_cap_out_t));
1388 		return (B_FALSE);
1389 	}
1390 	mlxcx_cmd_wait(&cmd);
1391 
1392 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1393 	if (ret) {
1394 		capp->mhc_mode = mode;
1395 		capp->mhc_type = type;
1396 		ASSERT3U(sizeof (out->mlxo_query_hca_cap_data), ==,
1397 		    sizeof (capp->mhc_bulk));
1398 		bcopy(out->mlxo_query_hca_cap_data, capp->mhc_bulk,
1399 		    sizeof (capp->mhc_bulk));
1400 	}
1401 	mlxcx_cmd_fini(mlxp, &cmd);
1402 
1403 	kmem_free(out, sizeof (mlxcx_cmd_query_hca_cap_out_t));
1404 	return (B_TRUE);
1405 }
1406 
1407 boolean_t
mlxcx_cmd_init_hca(mlxcx_t * mlxp)1408 mlxcx_cmd_init_hca(mlxcx_t *mlxp)
1409 {
1410 	mlxcx_cmd_t cmd;
1411 	mlxcx_cmd_init_hca_in_t in;
1412 	mlxcx_cmd_init_hca_out_t out;
1413 	boolean_t ret;
1414 
1415 	bzero(&in, sizeof (in));
1416 	bzero(&out, sizeof (out));
1417 
1418 	mlxcx_cmd_init(mlxp, &cmd);
1419 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_init_hca_head,
1420 	    MLXCX_OP_INIT_HCA, 0);
1421 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1422 		mlxcx_cmd_fini(mlxp, &cmd);
1423 		return (B_FALSE);
1424 	}
1425 	mlxcx_cmd_wait(&cmd);
1426 
1427 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1428 	mlxcx_cmd_fini(mlxp, &cmd);
1429 	return (ret);
1430 }
1431 
1432 boolean_t
mlxcx_cmd_set_driver_version(mlxcx_t * mlxp,const char * version)1433 mlxcx_cmd_set_driver_version(mlxcx_t *mlxp, const char *version)
1434 {
1435 	mlxcx_cmd_t cmd;
1436 	mlxcx_cmd_set_driver_version_in_t in;
1437 	mlxcx_cmd_set_driver_version_out_t out;
1438 	boolean_t ret;
1439 
1440 	bzero(&in, sizeof (in));
1441 	bzero(&out, sizeof (out));
1442 
1443 	mlxcx_cmd_init(mlxp, &cmd);
1444 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_driver_version_head,
1445 	    MLXCX_OP_SET_DRIVER_VERSION, 0);
1446 	VERIFY3U(strlcpy(in.mlxi_set_driver_version_version, version,
1447 	    sizeof (in.mlxi_set_driver_version_version)), <=,
1448 	    sizeof (in.mlxi_set_driver_version_version));
1449 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1450 		mlxcx_cmd_fini(mlxp, &cmd);
1451 		return (B_FALSE);
1452 	}
1453 	mlxcx_cmd_wait(&cmd);
1454 
1455 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1456 	mlxcx_cmd_fini(mlxp, &cmd);
1457 	return (ret);
1458 }
1459 
1460 boolean_t
mlxcx_cmd_alloc_uar(mlxcx_t * mlxp,mlxcx_uar_t * mlup)1461 mlxcx_cmd_alloc_uar(mlxcx_t *mlxp, mlxcx_uar_t *mlup)
1462 {
1463 	mlxcx_cmd_t cmd;
1464 	mlxcx_cmd_alloc_uar_in_t in;
1465 	mlxcx_cmd_alloc_uar_out_t out;
1466 	boolean_t ret;
1467 	size_t i;
1468 
1469 	bzero(&in, sizeof (in));
1470 	bzero(&out, sizeof (out));
1471 
1472 	mlxcx_cmd_init(mlxp, &cmd);
1473 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_uar_head,
1474 	    MLXCX_OP_ALLOC_UAR, 0);
1475 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1476 		mlxcx_cmd_fini(mlxp, &cmd);
1477 		return (B_FALSE);
1478 	}
1479 	mlxcx_cmd_wait(&cmd);
1480 
1481 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1482 	if (ret) {
1483 		mlup->mlu_allocated = B_TRUE;
1484 		mlup->mlu_num = from_be24(out.mlxo_alloc_uar_uar);
1485 		VERIFY3U(mlup->mlu_num, >, 0);
1486 		mlup->mlu_base = mlup->mlu_num * MLXCX_HW_PAGE_SIZE;
1487 
1488 		for (i = 0; i < MLXCX_BF_PER_UAR; ++i) {
1489 			mlup->mlu_bf[i].mbf_even = mlup->mlu_base +
1490 			    MLXCX_BF_BASE + MLXCX_BF_SIZE * 2 * i;
1491 			mlup->mlu_bf[i].mbf_odd = mlup->mlu_bf[i].mbf_even +
1492 			    MLXCX_BF_SIZE;
1493 		}
1494 	}
1495 	mlxcx_cmd_fini(mlxp, &cmd);
1496 	return (ret);
1497 }
1498 
1499 boolean_t
mlxcx_cmd_dealloc_uar(mlxcx_t * mlxp,mlxcx_uar_t * mlup)1500 mlxcx_cmd_dealloc_uar(mlxcx_t *mlxp, mlxcx_uar_t *mlup)
1501 {
1502 	mlxcx_cmd_t cmd;
1503 	mlxcx_cmd_dealloc_uar_in_t in;
1504 	mlxcx_cmd_dealloc_uar_out_t out;
1505 	boolean_t ret;
1506 
1507 	bzero(&in, sizeof (in));
1508 	bzero(&out, sizeof (out));
1509 
1510 	mlxcx_cmd_init(mlxp, &cmd);
1511 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_uar_head,
1512 	    MLXCX_OP_DEALLOC_UAR, 0);
1513 	VERIFY(mlup->mlu_allocated);
1514 	in.mlxi_dealloc_uar_uar = to_be24(mlup->mlu_num);
1515 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1516 		mlxcx_cmd_fini(mlxp, &cmd);
1517 		return (B_FALSE);
1518 	}
1519 	mlxcx_cmd_wait(&cmd);
1520 
1521 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1522 	if (ret) {
1523 		mlup->mlu_allocated = B_FALSE;
1524 		mlup->mlu_num = 0;
1525 	}
1526 	mlxcx_cmd_fini(mlxp, &cmd);
1527 	return (ret);
1528 }
1529 
1530 boolean_t
mlxcx_cmd_alloc_pd(mlxcx_t * mlxp,mlxcx_pd_t * mlpd)1531 mlxcx_cmd_alloc_pd(mlxcx_t *mlxp, mlxcx_pd_t *mlpd)
1532 {
1533 	mlxcx_cmd_t cmd;
1534 	mlxcx_cmd_alloc_pd_in_t in;
1535 	mlxcx_cmd_alloc_pd_out_t out;
1536 	boolean_t ret;
1537 
1538 	bzero(&in, sizeof (in));
1539 	bzero(&out, sizeof (out));
1540 
1541 	mlxcx_cmd_init(mlxp, &cmd);
1542 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_pd_head,
1543 	    MLXCX_OP_ALLOC_PD, 0);
1544 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1545 		mlxcx_cmd_fini(mlxp, &cmd);
1546 		return (B_FALSE);
1547 	}
1548 	mlxcx_cmd_wait(&cmd);
1549 
1550 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1551 	if (ret) {
1552 		mlpd->mlpd_allocated = B_TRUE;
1553 		mlpd->mlpd_num = from_be24(out.mlxo_alloc_pd_pdn);
1554 	}
1555 	mlxcx_cmd_fini(mlxp, &cmd);
1556 	return (ret);
1557 }
1558 
1559 boolean_t
mlxcx_cmd_dealloc_pd(mlxcx_t * mlxp,mlxcx_pd_t * mlpd)1560 mlxcx_cmd_dealloc_pd(mlxcx_t *mlxp, mlxcx_pd_t *mlpd)
1561 {
1562 	mlxcx_cmd_t cmd;
1563 	mlxcx_cmd_dealloc_pd_in_t in;
1564 	mlxcx_cmd_dealloc_pd_out_t out;
1565 	boolean_t ret;
1566 
1567 	bzero(&in, sizeof (in));
1568 	bzero(&out, sizeof (out));
1569 
1570 	mlxcx_cmd_init(mlxp, &cmd);
1571 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_pd_head,
1572 	    MLXCX_OP_DEALLOC_PD, 0);
1573 	VERIFY(mlpd->mlpd_allocated);
1574 	in.mlxi_dealloc_pd_pdn = to_be24(mlpd->mlpd_num);
1575 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1576 		mlxcx_cmd_fini(mlxp, &cmd);
1577 		return (B_FALSE);
1578 	}
1579 	mlxcx_cmd_wait(&cmd);
1580 
1581 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1582 	if (ret) {
1583 		mlpd->mlpd_allocated = B_FALSE;
1584 		mlpd->mlpd_num = 0;
1585 	}
1586 	mlxcx_cmd_fini(mlxp, &cmd);
1587 	return (ret);
1588 }
1589 
1590 boolean_t
mlxcx_cmd_alloc_tdom(mlxcx_t * mlxp,mlxcx_tdom_t * mltd)1591 mlxcx_cmd_alloc_tdom(mlxcx_t *mlxp, mlxcx_tdom_t *mltd)
1592 {
1593 	mlxcx_cmd_t cmd;
1594 	mlxcx_cmd_alloc_tdom_in_t in;
1595 	mlxcx_cmd_alloc_tdom_out_t out;
1596 	boolean_t ret;
1597 
1598 	bzero(&in, sizeof (in));
1599 	bzero(&out, sizeof (out));
1600 
1601 	mlxcx_cmd_init(mlxp, &cmd);
1602 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_tdom_head,
1603 	    MLXCX_OP_ALLOC_TRANSPORT_DOMAIN, 0);
1604 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1605 		mlxcx_cmd_fini(mlxp, &cmd);
1606 		return (B_FALSE);
1607 	}
1608 	mlxcx_cmd_wait(&cmd);
1609 
1610 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1611 	if (ret) {
1612 		mltd->mltd_allocated = B_TRUE;
1613 		mltd->mltd_num = from_be24(out.mlxo_alloc_tdom_tdomn);
1614 	}
1615 	mlxcx_cmd_fini(mlxp, &cmd);
1616 	return (ret);
1617 }
1618 
1619 boolean_t
mlxcx_cmd_dealloc_tdom(mlxcx_t * mlxp,mlxcx_tdom_t * mltd)1620 mlxcx_cmd_dealloc_tdom(mlxcx_t *mlxp, mlxcx_tdom_t *mltd)
1621 {
1622 	mlxcx_cmd_t cmd;
1623 	mlxcx_cmd_dealloc_tdom_in_t in;
1624 	mlxcx_cmd_dealloc_tdom_out_t out;
1625 	boolean_t ret;
1626 
1627 	bzero(&in, sizeof (in));
1628 	bzero(&out, sizeof (out));
1629 
1630 	mlxcx_cmd_init(mlxp, &cmd);
1631 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_tdom_head,
1632 	    MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN, 0);
1633 	VERIFY(mltd->mltd_allocated);
1634 	in.mlxi_dealloc_tdom_tdomn = to_be24(mltd->mltd_num);
1635 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1636 		mlxcx_cmd_fini(mlxp, &cmd);
1637 		return (B_FALSE);
1638 	}
1639 	mlxcx_cmd_wait(&cmd);
1640 
1641 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1642 	if (ret) {
1643 		mltd->mltd_allocated = B_FALSE;
1644 		mltd->mltd_num = 0;
1645 	}
1646 	mlxcx_cmd_fini(mlxp, &cmd);
1647 	return (ret);
1648 }
1649 
1650 boolean_t
mlxcx_cmd_teardown_hca(mlxcx_t * mlxp)1651 mlxcx_cmd_teardown_hca(mlxcx_t *mlxp)
1652 {
1653 	mlxcx_cmd_t cmd;
1654 	mlxcx_cmd_teardown_hca_in_t in;
1655 	mlxcx_cmd_teardown_hca_out_t out;
1656 	boolean_t ret;
1657 
1658 	bzero(&in, sizeof (in));
1659 	bzero(&out, sizeof (out));
1660 
1661 	mlxcx_cmd_init(mlxp, &cmd);
1662 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_teardown_hca_head,
1663 	    MLXCX_OP_TEARDOWN_HCA, 0);
1664 	in.mlxi_teardown_hca_profile = to_be16(MLXCX_TEARDOWN_HCA_GRACEFUL);
1665 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1666 		mlxcx_cmd_fini(mlxp, &cmd);
1667 		return (B_FALSE);
1668 	}
1669 	mlxcx_cmd_wait(&cmd);
1670 
1671 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1672 	mlxcx_cmd_fini(mlxp, &cmd);
1673 	return (ret);
1674 }
1675 
1676 boolean_t
mlxcx_cmd_query_nic_vport_ctx(mlxcx_t * mlxp,mlxcx_port_t * mlp)1677 mlxcx_cmd_query_nic_vport_ctx(mlxcx_t *mlxp, mlxcx_port_t *mlp)
1678 {
1679 	mlxcx_cmd_t cmd;
1680 	mlxcx_cmd_query_nic_vport_ctx_in_t in;
1681 	mlxcx_cmd_query_nic_vport_ctx_out_t out;
1682 	boolean_t ret;
1683 	const mlxcx_nic_vport_ctx_t *ctx;
1684 
1685 	bzero(&in, sizeof (in));
1686 	bzero(&out, sizeof (out));
1687 
1688 	ASSERT(mutex_owned(&mlp->mlp_mtx));
1689 	mlxcx_cmd_init(mlxp, &cmd);
1690 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_nic_vport_ctx_head,
1691 	    MLXCX_OP_QUERY_NIC_VPORT_CONTEXT, MLXCX_VPORT_TYPE_VNIC);
1692 
1693 	in.mlxi_query_nic_vport_ctx_vport_number = to_be16(mlp->mlp_num);
1694 
1695 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
1696 		mlxcx_cmd_fini(mlxp, &cmd);
1697 		return (B_FALSE);
1698 	}
1699 	mlxcx_cmd_wait(&cmd);
1700 
1701 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1702 	if (ret) {
1703 		ctx = &out.mlxo_query_nic_vport_ctx_context;
1704 		mlp->mlp_guid = from_be64(ctx->mlnvc_port_guid);
1705 		mlp->mlp_mtu = from_be16(ctx->mlnvc_mtu);
1706 		bcopy(ctx->mlnvc_permanent_address, mlp->mlp_mac_address,
1707 		    sizeof (mlp->mlp_mac_address));
1708 		mlp->mlp_wqe_min_inline = get_bits64(ctx->mlnvc_flags,
1709 		    MLXCX_VPORT_CTX_MIN_WQE_INLINE);
1710 	}
1711 	mlxcx_cmd_fini(mlxp, &cmd);
1712 	return (ret);
1713 }
1714 
1715 static const char *
mlxcx_reg_name(mlxcx_register_id_t rid)1716 mlxcx_reg_name(mlxcx_register_id_t rid)
1717 {
1718 	switch (rid) {
1719 	case MLXCX_REG_PMTU:
1720 		return ("PMTU");
1721 	case MLXCX_REG_PAOS:
1722 		return ("PAOS");
1723 	case MLXCX_REG_PCAM:
1724 		return ("PCAM");
1725 	case MLXCX_REG_PTYS:
1726 		return ("PTYS");
1727 	case MLXCX_REG_MSGI:
1728 		return ("MSGI");
1729 	case MLXCX_REG_PMAOS:
1730 		return ("PMAOS");
1731 	case MLXCX_REG_MLCR:
1732 		return ("MLCR");
1733 	case MLXCX_REG_MCIA:
1734 		return ("MCIA");
1735 	case MLXCX_REG_PPCNT:
1736 		return ("PPCNT");
1737 	case MLXCX_REG_PPLM:
1738 		return ("PPLM");
1739 	case MLXCX_REG_MTCAP:
1740 		return ("MTCAP");
1741 	case MLXCX_REG_MTMP:
1742 		return ("MTMP");
1743 	default:
1744 		return ("???");
1745 	}
1746 }
1747 
1748 boolean_t
mlxcx_cmd_access_register(mlxcx_t * mlxp,mlxcx_cmd_reg_opmod_t opmod,mlxcx_register_id_t rid,mlxcx_register_data_t * data)1749 mlxcx_cmd_access_register(mlxcx_t *mlxp, mlxcx_cmd_reg_opmod_t opmod,
1750     mlxcx_register_id_t rid, mlxcx_register_data_t *data)
1751 {
1752 	mlxcx_cmd_t cmd;
1753 	mlxcx_cmd_access_register_in_t in;
1754 	mlxcx_cmd_access_register_out_t out;
1755 	boolean_t ret;
1756 	size_t dsize, insize, outsize;
1757 
1758 	bzero(&in, sizeof (in));
1759 	bzero(&out, sizeof (out));
1760 
1761 	mlxcx_cmd_init(mlxp, &cmd);
1762 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_access_register_head,
1763 	    MLXCX_OP_ACCESS_REG, opmod);
1764 
1765 	in.mlxi_access_register_register_id = to_be16(rid);
1766 
1767 	switch (rid) {
1768 	case MLXCX_REG_PMTU:
1769 		dsize = sizeof (mlxcx_reg_pmtu_t);
1770 		break;
1771 	case MLXCX_REG_PAOS:
1772 		dsize = sizeof (mlxcx_reg_paos_t);
1773 		break;
1774 	case MLXCX_REG_PCAM:
1775 		dsize = sizeof (mlxcx_reg_pcam_t);
1776 		break;
1777 	case MLXCX_REG_PTYS:
1778 		dsize = sizeof (mlxcx_reg_ptys_t);
1779 		break;
1780 	case MLXCX_REG_MLCR:
1781 		dsize = sizeof (mlxcx_reg_mlcr_t);
1782 		break;
1783 	case MLXCX_REG_PMAOS:
1784 		dsize = sizeof (mlxcx_reg_pmaos_t);
1785 		break;
1786 	case MLXCX_REG_MCIA:
1787 		dsize = sizeof (mlxcx_reg_mcia_t);
1788 		break;
1789 	case MLXCX_REG_PPCNT:
1790 		dsize = sizeof (mlxcx_reg_ppcnt_t);
1791 		break;
1792 	case MLXCX_REG_PPLM:
1793 		dsize = sizeof (mlxcx_reg_pplm_t);
1794 		break;
1795 	case MLXCX_REG_MTCAP:
1796 		dsize = sizeof (mlxcx_reg_mtcap_t);
1797 		break;
1798 	case MLXCX_REG_MTMP:
1799 		dsize = sizeof (mlxcx_reg_mtmp_t);
1800 		break;
1801 	default:
1802 		dsize = 0;
1803 		VERIFY(0);
1804 		return (B_FALSE);
1805 	}
1806 	insize = dsize + offsetof(mlxcx_cmd_access_register_in_t,
1807 	    mlxi_access_register_data);
1808 	outsize = dsize + offsetof(mlxcx_cmd_access_register_out_t,
1809 	    mlxo_access_register_data);
1810 
1811 	bcopy(data, &in.mlxi_access_register_data, dsize);
1812 
1813 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, outsize)) {
1814 		mlxcx_cmd_fini(mlxp, &cmd);
1815 		return (B_FALSE);
1816 	}
1817 	mlxcx_cmd_wait(&cmd);
1818 
1819 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
1820 	if (ret) {
1821 		bcopy(&out.mlxo_access_register_data, data, dsize);
1822 	} else {
1823 		mlxcx_warn(mlxp, "failed OP_ACCESS_REG was for register "
1824 		    "%04x (%s)", rid, mlxcx_reg_name(rid));
1825 	}
1826 	mlxcx_cmd_fini(mlxp, &cmd);
1827 	return (ret);
1828 }
1829 
1830 boolean_t
mlxcx_cmd_query_port_mtu(mlxcx_t * mlxp,mlxcx_port_t * mlp)1831 mlxcx_cmd_query_port_mtu(mlxcx_t *mlxp, mlxcx_port_t *mlp)
1832 {
1833 	mlxcx_register_data_t data;
1834 	boolean_t ret;
1835 
1836 	/*
1837 	 * Since we modify the port here we require that the caller is holding
1838 	 * the port mutex.
1839 	 */
1840 	ASSERT(mutex_owned(&mlp->mlp_mtx));
1841 	bzero(&data, sizeof (data));
1842 	data.mlrd_pmtu.mlrd_pmtu_local_port = mlp->mlp_num + 1;
1843 
1844 	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ,
1845 	    MLXCX_REG_PMTU, &data);
1846 
1847 	if (ret) {
1848 		mlp->mlp_mtu = from_be16(data.mlrd_pmtu.mlrd_pmtu_admin_mtu);
1849 		mlp->mlp_max_mtu = from_be16(data.mlrd_pmtu.mlrd_pmtu_max_mtu);
1850 	}
1851 
1852 	return (ret);
1853 }
1854 
1855 boolean_t
mlxcx_cmd_query_module_status(mlxcx_t * mlxp,uint_t id,mlxcx_module_status_t * pstatus,mlxcx_module_error_type_t * perr)1856 mlxcx_cmd_query_module_status(mlxcx_t *mlxp, uint_t id,
1857     mlxcx_module_status_t *pstatus, mlxcx_module_error_type_t *perr)
1858 {
1859 	mlxcx_register_data_t data;
1860 	boolean_t ret;
1861 
1862 	bzero(&data, sizeof (data));
1863 	ASSERT3U(id, <, 0xff);
1864 	data.mlrd_pmaos.mlrd_pmaos_module = (uint8_t)id;
1865 
1866 	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ,
1867 	    MLXCX_REG_PMAOS, &data);
1868 
1869 	if (ret) {
1870 		if (pstatus != NULL)
1871 			*pstatus = data.mlrd_pmaos.mlrd_pmaos_oper_status;
1872 		if (perr != NULL)
1873 			*perr = data.mlrd_pmaos.mlrd_pmaos_error_type;
1874 	}
1875 
1876 	return (ret);
1877 }
1878 
1879 boolean_t
mlxcx_cmd_set_port_mtu(mlxcx_t * mlxp,mlxcx_port_t * mlp)1880 mlxcx_cmd_set_port_mtu(mlxcx_t *mlxp, mlxcx_port_t *mlp)
1881 {
1882 	mlxcx_register_data_t data;
1883 	boolean_t ret;
1884 
1885 	ASSERT(mutex_owned(&mlp->mlp_mtx));
1886 	bzero(&data, sizeof (data));
1887 	data.mlrd_pmtu.mlrd_pmtu_local_port = mlp->mlp_num + 1;
1888 	data.mlrd_pmtu.mlrd_pmtu_admin_mtu = to_be16(mlp->mlp_mtu);
1889 
1890 	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE,
1891 	    MLXCX_REG_PMTU, &data);
1892 
1893 	return (ret);
1894 }
1895 
1896 boolean_t
mlxcx_cmd_set_port_led(mlxcx_t * mlxp,mlxcx_port_t * mlp,uint16_t sec)1897 mlxcx_cmd_set_port_led(mlxcx_t *mlxp, mlxcx_port_t *mlp, uint16_t sec)
1898 {
1899 	mlxcx_register_data_t data;
1900 	boolean_t ret;
1901 
1902 	ASSERT(mutex_owned(&mlp->mlp_mtx));
1903 	bzero(&data, sizeof (data));
1904 	data.mlrd_mlcr.mlrd_mlcr_local_port = mlp->mlp_num + 1;
1905 	set_bits8(&data.mlrd_mlcr.mlrd_mlcr_flags, MLXCX_MLCR_LED_TYPE,
1906 	    MLXCX_LED_TYPE_PORT);
1907 	data.mlrd_mlcr.mlrd_mlcr_beacon_duration = to_be16(sec);
1908 
1909 	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE,
1910 	    MLXCX_REG_MLCR, &data);
1911 
1912 	return (ret);
1913 }
1914 
1915 boolean_t
mlxcx_cmd_query_port_status(mlxcx_t * mlxp,mlxcx_port_t * mlp)1916 mlxcx_cmd_query_port_status(mlxcx_t *mlxp, mlxcx_port_t *mlp)
1917 {
1918 	mlxcx_register_data_t data;
1919 	boolean_t ret;
1920 
1921 	ASSERT(mutex_owned(&mlp->mlp_mtx));
1922 	bzero(&data, sizeof (data));
1923 	data.mlrd_paos.mlrd_paos_local_port = mlp->mlp_num + 1;
1924 
1925 	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ,
1926 	    MLXCX_REG_PAOS, &data);
1927 
1928 	if (ret) {
1929 		mlp->mlp_admin_status = data.mlrd_paos.mlrd_paos_admin_status;
1930 		mlp->mlp_oper_status = data.mlrd_paos.mlrd_paos_oper_status;
1931 	}
1932 
1933 	return (ret);
1934 }
1935 
1936 boolean_t
mlxcx_cmd_modify_port_status(mlxcx_t * mlxp,mlxcx_port_t * mlp,mlxcx_port_status_t status)1937 mlxcx_cmd_modify_port_status(mlxcx_t *mlxp, mlxcx_port_t *mlp,
1938     mlxcx_port_status_t status)
1939 {
1940 	mlxcx_register_data_t data;
1941 	boolean_t ret;
1942 
1943 	ASSERT(mutex_owned(&mlp->mlp_mtx));
1944 	bzero(&data, sizeof (data));
1945 	data.mlrd_paos.mlrd_paos_local_port = mlp->mlp_num + 1;
1946 	data.mlrd_paos.mlrd_paos_admin_status = status;
1947 	set_bit32(&data.mlrd_paos.mlrd_paos_flags, MLXCX_PAOS_ADMIN_ST_EN);
1948 
1949 	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE,
1950 	    MLXCX_REG_PAOS, &data);
1951 
1952 	return (ret);
1953 }
1954 
1955 boolean_t
mlxcx_cmd_query_port_speed(mlxcx_t * mlxp,mlxcx_port_t * mlp)1956 mlxcx_cmd_query_port_speed(mlxcx_t *mlxp, mlxcx_port_t *mlp)
1957 {
1958 	mlxcx_register_data_t data;
1959 	boolean_t ret;
1960 
1961 	ASSERT(mutex_owned(&mlp->mlp_mtx));
1962 	bzero(&data, sizeof (data));
1963 	data.mlrd_ptys.mlrd_ptys_local_port = mlp->mlp_num + 1;
1964 	set_bit8(&data.mlrd_ptys.mlrd_ptys_proto_mask,
1965 	    MLXCX_PTYS_PROTO_MASK_ETH);
1966 
1967 	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ,
1968 	    MLXCX_REG_PTYS, &data);
1969 
1970 	if (ret) {
1971 		if (get_bit8(data.mlrd_ptys.mlrd_ptys_autoneg_flags,
1972 		    MLXCX_AUTONEG_DISABLE)) {
1973 			mlp->mlp_autoneg = B_FALSE;
1974 		} else {
1975 			mlp->mlp_autoneg = B_TRUE;
1976 		}
1977 		mlp->mlp_max_proto =
1978 		    from_bits32(data.mlrd_ptys.mlrd_ptys_proto_cap);
1979 		mlp->mlp_admin_proto =
1980 		    from_bits32(data.mlrd_ptys.mlrd_ptys_proto_admin);
1981 		mlp->mlp_oper_proto =
1982 		    from_bits32(data.mlrd_ptys.mlrd_ptys_proto_oper);
1983 		if (mlxp->mlx_caps->mlc_ext_ptys) {
1984 			/*
1985 			 * Populate these bits only if we know the HW
1986 			 * supports them.  Otherwise keep them zeroed
1987 			 * per the above bzero() and use that zero-ness to
1988 			 * skip over them as need be.
1989 			 */
1990 			mlp->mlp_ext_max_proto = from_bits32(
1991 			    data.mlrd_ptys.mlrd_ptys_ext_proto_cap);
1992 			mlp->mlp_ext_admin_proto = from_bits32(
1993 			    data.mlrd_ptys.mlrd_ptys_ext_proto_admin);
1994 			mlp->mlp_ext_oper_proto = from_bits32(
1995 			    data.mlrd_ptys.mlrd_ptys_ext_proto_oper);
1996 		}
1997 	}
1998 
1999 	return (ret);
2000 }
2001 
2002 boolean_t
mlxcx_cmd_query_port_fec(mlxcx_t * mlxp,mlxcx_port_t * mlp)2003 mlxcx_cmd_query_port_fec(mlxcx_t *mlxp, mlxcx_port_t *mlp)
2004 {
2005 	mlxcx_register_data_t data;
2006 	boolean_t ret;
2007 
2008 	ASSERT(mutex_owned(&mlp->mlp_mtx));
2009 	bzero(&data, sizeof (data));
2010 	data.mlrd_pplm.mlrd_pplm_local_port = mlp->mlp_num + 1;
2011 
2012 	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ,
2013 	    MLXCX_REG_PPLM, &data);
2014 
2015 	if (ret) {
2016 		mlp->mlp_fec_active =
2017 		    from_be24(data.mlrd_pplm.mlrd_pplm_fec_mode_active);
2018 	}
2019 
2020 	return (ret);
2021 }
2022 
2023 boolean_t
mlxcx_cmd_modify_port_fec(mlxcx_t * mlxp,mlxcx_port_t * mlp,mlxcx_pplm_fec_caps_t fec)2024 mlxcx_cmd_modify_port_fec(mlxcx_t *mlxp, mlxcx_port_t *mlp,
2025     mlxcx_pplm_fec_caps_t fec)
2026 {
2027 	mlxcx_register_data_t data_in, data_out;
2028 	mlxcx_pplm_fec_caps_t caps;
2029 	mlxcx_reg_pplm_t *pplm_in, *pplm_out;
2030 	boolean_t ret;
2031 
2032 	ASSERT(mutex_owned(&mlp->mlp_mtx));
2033 	bzero(&data_in, sizeof (data_in));
2034 	pplm_in = &data_in.mlrd_pplm;
2035 	pplm_in->mlrd_pplm_local_port = mlp->mlp_num + 1;
2036 
2037 	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ,
2038 	    MLXCX_REG_PPLM, &data_in);
2039 
2040 	if (!ret)
2041 		return (B_FALSE);
2042 
2043 	bzero(&data_out, sizeof (data_out));
2044 	pplm_out = &data_out.mlrd_pplm;
2045 	pplm_out->mlrd_pplm_local_port = mlp->mlp_num + 1;
2046 
2047 	caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap,
2048 	    MLXCX_PPLM_CAP_56G);
2049 	set_bits32(&pplm_out->mlrd_pplm_fec_override_admin,
2050 	    MLXCX_PPLM_CAP_56G, fec & caps);
2051 
2052 	caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap,
2053 	    MLXCX_PPLM_CAP_100G);
2054 	set_bits32(&pplm_out->mlrd_pplm_fec_override_admin,
2055 	    MLXCX_PPLM_CAP_100G, fec & caps);
2056 
2057 	caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap,
2058 	    MLXCX_PPLM_CAP_50G);
2059 	set_bits32(&pplm_out->mlrd_pplm_fec_override_admin,
2060 	    MLXCX_PPLM_CAP_50G, fec & caps);
2061 
2062 	caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap,
2063 	    MLXCX_PPLM_CAP_25G);
2064 	set_bits32(&pplm_out->mlrd_pplm_fec_override_admin,
2065 	    MLXCX_PPLM_CAP_25G, fec & caps);
2066 
2067 	caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap,
2068 	    MLXCX_PPLM_CAP_10_40G);
2069 	set_bits32(&pplm_out->mlrd_pplm_fec_override_admin,
2070 	    MLXCX_PPLM_CAP_10_40G, fec & caps);
2071 
2072 	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE,
2073 	    MLXCX_REG_PPLM, &data_out);
2074 
2075 	return (ret);
2076 }
2077 
2078 boolean_t
mlxcx_cmd_modify_nic_vport_ctx(mlxcx_t * mlxp,mlxcx_port_t * mlp,mlxcx_modify_nic_vport_ctx_fields_t fields)2079 mlxcx_cmd_modify_nic_vport_ctx(mlxcx_t *mlxp, mlxcx_port_t *mlp,
2080     mlxcx_modify_nic_vport_ctx_fields_t fields)
2081 {
2082 	mlxcx_cmd_t cmd;
2083 	mlxcx_cmd_modify_nic_vport_ctx_in_t in;
2084 	mlxcx_cmd_modify_nic_vport_ctx_out_t out;
2085 	boolean_t ret;
2086 	mlxcx_nic_vport_ctx_t *ctx;
2087 
2088 	ASSERT(mutex_owned(&mlp->mlp_mtx));
2089 	bzero(&in, sizeof (in));
2090 	bzero(&out, sizeof (out));
2091 
2092 	mlxcx_cmd_init(mlxp, &cmd);
2093 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_nic_vport_ctx_head,
2094 	    MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT, MLXCX_VPORT_TYPE_VNIC);
2095 
2096 	in.mlxi_modify_nic_vport_ctx_vport_number = to_be16(mlp->mlp_num);
2097 	in.mlxi_modify_nic_vport_ctx_field_select = to_be32(fields);
2098 
2099 	ctx = &in.mlxi_modify_nic_vport_ctx_context;
2100 	if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_PROMISC) {
2101 		set_bit16(&ctx->mlnvc_promisc_list_type,
2102 		    MLXCX_VPORT_PROMISC_ALL);
2103 	}
2104 	if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_MTU) {
2105 		ctx->mlnvc_mtu = to_be16(mlp->mlp_mtu);
2106 	}
2107 
2108 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2109 		mlxcx_cmd_fini(mlxp, &cmd);
2110 		return (B_FALSE);
2111 	}
2112 	mlxcx_cmd_wait(&cmd);
2113 
2114 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2115 	if (ret) {
2116 		if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_PROMISC) {
2117 			mlp->mlp_flags |= MLXCX_PORT_VPORT_PROMISC;
2118 		}
2119 	}
2120 	mlxcx_cmd_fini(mlxp, &cmd);
2121 	return (ret);
2122 }
2123 
2124 boolean_t
mlxcx_cmd_create_eq(mlxcx_t * mlxp,mlxcx_event_queue_t * mleq)2125 mlxcx_cmd_create_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq)
2126 {
2127 	mlxcx_cmd_t cmd;
2128 	mlxcx_cmd_create_eq_in_t in;
2129 	mlxcx_cmd_create_eq_out_t out;
2130 	boolean_t ret;
2131 	mlxcx_eventq_ctx_t *ctx;
2132 	size_t rem, insize;
2133 	const ddi_dma_cookie_t *c;
2134 	uint64_t pa, npages;
2135 
2136 	bzero(&in, sizeof (in));
2137 	bzero(&out, sizeof (out));
2138 
2139 	ASSERT(mutex_owned(&mleq->mleq_mtx));
2140 	VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC);
2141 	VERIFY0(mleq->mleq_state & MLXCX_EQ_CREATED);
2142 
2143 	mlxcx_cmd_init(mlxp, &cmd);
2144 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_eq_head,
2145 	    MLXCX_OP_CREATE_EQ, 0);
2146 
2147 	ctx = &in.mlxi_create_eq_context;
2148 	ctx->mleqc_uar_page = to_be24(mleq->mleq_uar->mlu_num);
2149 	ctx->mleqc_log_eq_size = mleq->mleq_entshift;
2150 	ctx->mleqc_intr = mleq->mleq_intr_index;
2151 
2152 	in.mlxi_create_eq_event_bitmask = to_be64(mleq->mleq_events);
2153 
2154 	npages = 0;
2155 	c = NULL;
2156 	while ((c = mlxcx_dma_cookie_iter(&mleq->mleq_dma, c)) != NULL) {
2157 		pa = c->dmac_laddress;
2158 		rem = c->dmac_size;
2159 		while (rem > 0) {
2160 			ASSERT3U(pa & 0xfff, ==, 0);
2161 			ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE);
2162 			in.mlxi_create_eq_pas[npages++] = to_be64(pa);
2163 			rem -= MLXCX_HW_PAGE_SIZE;
2164 			pa += MLXCX_HW_PAGE_SIZE;
2165 		}
2166 	}
2167 	ASSERT3U(npages, <=, MLXCX_CREATE_QUEUE_MAX_PAGES);
2168 
2169 	insize = offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_pas) +
2170 	    sizeof (uint64_t) * npages;
2171 
2172 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) {
2173 		mlxcx_cmd_fini(mlxp, &cmd);
2174 		return (B_FALSE);
2175 	}
2176 	mlxcx_cmd_wait(&cmd);
2177 
2178 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2179 	if (ret) {
2180 		mleq->mleq_state |= MLXCX_EQ_CREATED;
2181 		mleq->mleq_num = out.mlxo_create_eq_eqn;
2182 	}
2183 	mlxcx_cmd_fini(mlxp, &cmd);
2184 	return (ret);
2185 }
2186 
2187 boolean_t
mlxcx_cmd_query_eq(mlxcx_t * mlxp,mlxcx_event_queue_t * mleq,mlxcx_eventq_ctx_t * ctxp)2188 mlxcx_cmd_query_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq,
2189     mlxcx_eventq_ctx_t *ctxp)
2190 {
2191 	mlxcx_cmd_t cmd;
2192 	mlxcx_cmd_query_eq_in_t in;
2193 	mlxcx_cmd_query_eq_out_t out;
2194 	boolean_t ret;
2195 
2196 	bzero(&in, sizeof (in));
2197 	bzero(&out, sizeof (out));
2198 
2199 	VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC);
2200 	VERIFY(mleq->mleq_state & MLXCX_EQ_CREATED);
2201 
2202 	mlxcx_cmd_init(mlxp, &cmd);
2203 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_eq_head,
2204 	    MLXCX_OP_QUERY_EQ, 0);
2205 
2206 	in.mlxi_query_eq_eqn = mleq->mleq_num;
2207 
2208 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2209 		mlxcx_cmd_fini(mlxp, &cmd);
2210 		return (B_FALSE);
2211 	}
2212 	mlxcx_cmd_wait(&cmd);
2213 
2214 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2215 	if (ret) {
2216 		bcopy(&out.mlxo_query_eq_context, ctxp,
2217 		    sizeof (mlxcx_eventq_ctx_t));
2218 	}
2219 	mlxcx_cmd_fini(mlxp, &cmd);
2220 	return (ret);
2221 }
2222 
2223 boolean_t
mlxcx_cmd_destroy_eq(mlxcx_t * mlxp,mlxcx_event_queue_t * mleq)2224 mlxcx_cmd_destroy_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq)
2225 {
2226 	mlxcx_cmd_t cmd;
2227 	mlxcx_cmd_destroy_eq_in_t in;
2228 	mlxcx_cmd_destroy_eq_out_t out;
2229 	boolean_t ret;
2230 
2231 	bzero(&in, sizeof (in));
2232 	bzero(&out, sizeof (out));
2233 
2234 	ASSERT(mutex_owned(&mleq->mleq_mtx));
2235 	VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC);
2236 	VERIFY(mleq->mleq_state & MLXCX_EQ_CREATED);
2237 
2238 	mlxcx_cmd_init(mlxp, &cmd);
2239 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_eq_head,
2240 	    MLXCX_OP_DESTROY_EQ, 0);
2241 
2242 	in.mlxi_destroy_eq_eqn = mleq->mleq_num;
2243 
2244 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2245 		mlxcx_cmd_fini(mlxp, &cmd);
2246 		return (B_FALSE);
2247 	}
2248 	mlxcx_cmd_wait(&cmd);
2249 
2250 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2251 	if (ret) {
2252 		mleq->mleq_state |= MLXCX_EQ_DESTROYED;
2253 	}
2254 	mlxcx_cmd_fini(mlxp, &cmd);
2255 	return (ret);
2256 }
2257 
2258 boolean_t
mlxcx_cmd_query_special_ctxs(mlxcx_t * mlxp)2259 mlxcx_cmd_query_special_ctxs(mlxcx_t *mlxp)
2260 {
2261 	mlxcx_cmd_t cmd;
2262 	mlxcx_cmd_query_special_ctxs_in_t in;
2263 	mlxcx_cmd_query_special_ctxs_out_t out;
2264 	boolean_t ret;
2265 
2266 	bzero(&in, sizeof (in));
2267 	bzero(&out, sizeof (out));
2268 
2269 	mlxcx_cmd_init(mlxp, &cmd);
2270 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_special_ctxs_head,
2271 	    MLXCX_OP_QUERY_SPECIAL_CONTEXTS, 0);
2272 
2273 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2274 		mlxcx_cmd_fini(mlxp, &cmd);
2275 		return (B_FALSE);
2276 	}
2277 	mlxcx_cmd_wait(&cmd);
2278 
2279 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2280 	if (ret) {
2281 		mlxp->mlx_rsvd_lkey = from_be32(
2282 		    out.mlxo_query_special_ctxs_resd_lkey);
2283 	}
2284 	mlxcx_cmd_fini(mlxp, &cmd);
2285 	return (ret);
2286 }
2287 
2288 boolean_t
mlxcx_cmd_create_cq(mlxcx_t * mlxp,mlxcx_completion_queue_t * mlcq)2289 mlxcx_cmd_create_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq)
2290 {
2291 	mlxcx_cmd_t cmd;
2292 	mlxcx_cmd_create_cq_in_t in;
2293 	mlxcx_cmd_create_cq_out_t out;
2294 	boolean_t ret;
2295 	mlxcx_completionq_ctx_t *ctx;
2296 	size_t rem, insize;
2297 	const ddi_dma_cookie_t *c;
2298 	uint64_t pa, npages;
2299 
2300 	bzero(&in, sizeof (in));
2301 	bzero(&out, sizeof (out));
2302 
2303 	ASSERT(mutex_owned(&mlcq->mlcq_mtx));
2304 	VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC);
2305 	VERIFY0(mlcq->mlcq_state & MLXCX_CQ_CREATED);
2306 
2307 	mlxcx_cmd_init(mlxp, &cmd);
2308 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_cq_head,
2309 	    MLXCX_OP_CREATE_CQ, 0);
2310 
2311 	ctx = &in.mlxi_create_cq_context;
2312 	ctx->mlcqc_uar_page = to_be24(mlcq->mlcq_uar->mlu_num);
2313 	ctx->mlcqc_log_cq_size = mlcq->mlcq_entshift;
2314 	ctx->mlcqc_eqn = mlcq->mlcq_eq->mleq_num;
2315 	ctx->mlcqc_cq_period = to_be16(mlcq->mlcq_cqemod_period_usec);
2316 	ctx->mlcqc_cq_max_count = to_be16(mlcq->mlcq_cqemod_count);
2317 
2318 	c = mlxcx_dma_cookie_one(&mlcq->mlcq_doorbell_dma);
2319 	ctx->mlcqc_dbr_addr = to_be64(c->dmac_laddress);
2320 	ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_completionq_doorbell_t));
2321 
2322 	npages = 0;
2323 	c = NULL;
2324 	while ((c = mlxcx_dma_cookie_iter(&mlcq->mlcq_dma, c)) != NULL) {
2325 		pa = c->dmac_laddress;
2326 		rem = c->dmac_size;
2327 		while (rem > 0) {
2328 			ASSERT3U(pa & 0xfff, ==, 0);
2329 			ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE);
2330 			in.mlxi_create_cq_pas[npages++] = to_be64(pa);
2331 			rem -= MLXCX_HW_PAGE_SIZE;
2332 			pa += MLXCX_HW_PAGE_SIZE;
2333 		}
2334 	}
2335 	ASSERT3U(npages, <=, MLXCX_CREATE_QUEUE_MAX_PAGES);
2336 
2337 	insize = offsetof(mlxcx_cmd_create_cq_in_t, mlxi_create_cq_pas) +
2338 	    sizeof (uint64_t) * npages;
2339 
2340 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) {
2341 		mlxcx_cmd_fini(mlxp, &cmd);
2342 		return (B_FALSE);
2343 	}
2344 	mlxcx_cmd_wait(&cmd);
2345 
2346 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2347 	if (ret) {
2348 		atomic_or_uint(&mlcq->mlcq_state, MLXCX_CQ_CREATED);
2349 		mlcq->mlcq_num = from_be24(out.mlxo_create_cq_cqn);
2350 	}
2351 	mlxcx_cmd_fini(mlxp, &cmd);
2352 	return (ret);
2353 }
2354 
2355 boolean_t
mlxcx_cmd_query_rq(mlxcx_t * mlxp,mlxcx_work_queue_t * mlwq,mlxcx_rq_ctx_t * ctxp)2356 mlxcx_cmd_query_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq,
2357     mlxcx_rq_ctx_t *ctxp)
2358 {
2359 	mlxcx_cmd_t cmd;
2360 	mlxcx_cmd_query_rq_in_t in;
2361 	mlxcx_cmd_query_rq_out_t out;
2362 	boolean_t ret;
2363 
2364 	bzero(&in, sizeof (in));
2365 	bzero(&out, sizeof (out));
2366 
2367 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);
2368 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);
2369 	ASSERT3S(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ);
2370 
2371 	mlxcx_cmd_init(mlxp, &cmd);
2372 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_rq_head,
2373 	    MLXCX_OP_QUERY_RQ, 0);
2374 
2375 	in.mlxi_query_rq_rqn = to_be24(mlwq->mlwq_num);
2376 
2377 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2378 		mlxcx_cmd_fini(mlxp, &cmd);
2379 		return (B_FALSE);
2380 	}
2381 	mlxcx_cmd_wait(&cmd);
2382 
2383 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2384 	if (ret) {
2385 		bcopy(&out.mlxo_query_rq_context, ctxp,
2386 		    sizeof (mlxcx_rq_ctx_t));
2387 	}
2388 	mlxcx_cmd_fini(mlxp, &cmd);
2389 	return (ret);
2390 }
2391 
2392 boolean_t
mlxcx_cmd_query_sq(mlxcx_t * mlxp,mlxcx_work_queue_t * mlwq,mlxcx_sq_ctx_t * ctxp)2393 mlxcx_cmd_query_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq,
2394     mlxcx_sq_ctx_t *ctxp)
2395 {
2396 	mlxcx_cmd_t cmd;
2397 	mlxcx_cmd_query_sq_in_t in;
2398 	mlxcx_cmd_query_sq_out_t out;
2399 	boolean_t ret;
2400 
2401 	bzero(&in, sizeof (in));
2402 	bzero(&out, sizeof (out));
2403 
2404 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);
2405 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);
2406 	ASSERT3S(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_SENDQ);
2407 
2408 	mlxcx_cmd_init(mlxp, &cmd);
2409 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_sq_head,
2410 	    MLXCX_OP_QUERY_SQ, 0);
2411 
2412 	in.mlxi_query_sq_sqn = to_be24(mlwq->mlwq_num);
2413 
2414 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2415 		mlxcx_cmd_fini(mlxp, &cmd);
2416 		return (B_FALSE);
2417 	}
2418 	mlxcx_cmd_wait(&cmd);
2419 
2420 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2421 	if (ret) {
2422 		bcopy(&out.mlxo_query_sq_context, ctxp,
2423 		    sizeof (mlxcx_sq_ctx_t));
2424 	}
2425 	mlxcx_cmd_fini(mlxp, &cmd);
2426 	return (ret);
2427 }
2428 
2429 boolean_t
mlxcx_cmd_query_cq(mlxcx_t * mlxp,mlxcx_completion_queue_t * mlcq,mlxcx_completionq_ctx_t * ctxp)2430 mlxcx_cmd_query_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq,
2431     mlxcx_completionq_ctx_t *ctxp)
2432 {
2433 	mlxcx_cmd_t cmd;
2434 	mlxcx_cmd_query_cq_in_t in;
2435 	mlxcx_cmd_query_cq_out_t out;
2436 	boolean_t ret;
2437 
2438 	bzero(&in, sizeof (in));
2439 	bzero(&out, sizeof (out));
2440 
2441 	VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC);
2442 	VERIFY(mlcq->mlcq_state & MLXCX_CQ_CREATED);
2443 
2444 	mlxcx_cmd_init(mlxp, &cmd);
2445 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_cq_head,
2446 	    MLXCX_OP_QUERY_CQ, 0);
2447 
2448 	in.mlxi_query_cq_cqn = to_be24(mlcq->mlcq_num);
2449 
2450 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2451 		mlxcx_cmd_fini(mlxp, &cmd);
2452 		return (B_FALSE);
2453 	}
2454 	mlxcx_cmd_wait(&cmd);
2455 
2456 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2457 	if (ret) {
2458 		bcopy(&out.mlxo_query_cq_context, ctxp,
2459 		    sizeof (mlxcx_completionq_ctx_t));
2460 	}
2461 	mlxcx_cmd_fini(mlxp, &cmd);
2462 	return (ret);
2463 }
2464 
2465 boolean_t
mlxcx_cmd_destroy_cq(mlxcx_t * mlxp,mlxcx_completion_queue_t * mlcq)2466 mlxcx_cmd_destroy_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq)
2467 {
2468 	mlxcx_cmd_t cmd;
2469 	mlxcx_cmd_destroy_cq_in_t in;
2470 	mlxcx_cmd_destroy_cq_out_t out;
2471 	boolean_t ret;
2472 
2473 	bzero(&in, sizeof (in));
2474 	bzero(&out, sizeof (out));
2475 
2476 	ASSERT(mutex_owned(&mlcq->mlcq_mtx));
2477 	VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC);
2478 	VERIFY(mlcq->mlcq_state & MLXCX_CQ_CREATED);
2479 
2480 	mlxcx_cmd_init(mlxp, &cmd);
2481 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_cq_head,
2482 	    MLXCX_OP_DESTROY_CQ, 0);
2483 
2484 	in.mlxi_destroy_cq_cqn = to_be24(mlcq->mlcq_num);
2485 
2486 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2487 		mlxcx_cmd_fini(mlxp, &cmd);
2488 		return (B_FALSE);
2489 	}
2490 	mlxcx_cmd_wait(&cmd);
2491 
2492 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2493 	if (ret) {
2494 		atomic_or_uint(&mlcq->mlcq_state, MLXCX_CQ_DESTROYED);
2495 	}
2496 	mlxcx_cmd_fini(mlxp, &cmd);
2497 	return (ret);
2498 }
2499 
2500 boolean_t
mlxcx_cmd_create_rq(mlxcx_t * mlxp,mlxcx_work_queue_t * mlwq)2501 mlxcx_cmd_create_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)
2502 {
2503 	mlxcx_cmd_t cmd;
2504 	mlxcx_cmd_create_rq_in_t in;
2505 	mlxcx_cmd_create_rq_out_t out;
2506 	boolean_t ret;
2507 	mlxcx_rq_ctx_t *ctx;
2508 	size_t rem, insize;
2509 	const ddi_dma_cookie_t *c;
2510 	uint64_t pa, npages;
2511 
2512 	bzero(&in, sizeof (in));
2513 	bzero(&out, sizeof (out));
2514 
2515 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));
2516 	VERIFY3U(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ);
2517 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);
2518 	VERIFY0(mlwq->mlwq_state & MLXCX_WQ_CREATED);
2519 
2520 	mlxcx_cmd_init(mlxp, &cmd);
2521 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_rq_head,
2522 	    MLXCX_OP_CREATE_RQ, 0);
2523 
2524 	ctx = &in.mlxi_create_rq_context;
2525 
2526 	set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_RLKEY);
2527 	set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_FLUSH_IN_ERROR);
2528 	set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_VLAN_STRIP_DISABLE);
2529 	ctx->mlrqc_cqn = to_be24(mlwq->mlwq_cq->mlcq_num);
2530 
2531 	set_bits32(&ctx->mlrqc_wq.mlwqc_flags, MLXCX_WORKQ_CTX_TYPE,
2532 	    MLXCX_WORKQ_TYPE_CYCLIC);
2533 	ctx->mlrqc_wq.mlwqc_pd = to_be24(mlwq->mlwq_pd->mlpd_num);
2534 	ctx->mlrqc_wq.mlwqc_log_wq_sz = mlwq->mlwq_entshift;
2535 	ctx->mlrqc_wq.mlwqc_log_wq_stride = MLXCX_RECVQ_STRIDE_SHIFT;
2536 
2537 	c = mlxcx_dma_cookie_one(&mlwq->mlwq_doorbell_dma);
2538 	ctx->mlrqc_wq.mlwqc_dbr_addr = to_be64(c->dmac_laddress);
2539 	ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_workq_doorbell_t));
2540 
2541 	npages = 0;
2542 	c = NULL;
2543 	while ((c = mlxcx_dma_cookie_iter(&mlwq->mlwq_dma, c)) != NULL) {
2544 		pa = c->dmac_laddress;
2545 		rem = c->dmac_size;
2546 		while (rem > 0) {
2547 			ASSERT3U(pa & 0xfff, ==, 0);
2548 			ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE);
2549 			ctx->mlrqc_wq.mlwqc_pas[npages++] = to_be64(pa);
2550 			rem -= MLXCX_HW_PAGE_SIZE;
2551 			pa += MLXCX_HW_PAGE_SIZE;
2552 		}
2553 	}
2554 	ASSERT3U(npages, <=, MLXCX_WORKQ_CTX_MAX_ADDRESSES);
2555 
2556 	insize = offsetof(mlxcx_cmd_create_rq_in_t, mlxi_create_rq_context) +
2557 	    offsetof(mlxcx_rq_ctx_t, mlrqc_wq) +
2558 	    offsetof(mlxcx_workq_ctx_t, mlwqc_pas) +
2559 	    sizeof (uint64_t) * npages;
2560 
2561 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) {
2562 		mlxcx_cmd_fini(mlxp, &cmd);
2563 		return (B_FALSE);
2564 	}
2565 	mlxcx_cmd_wait(&cmd);
2566 
2567 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2568 	if (ret) {
2569 		mlwq->mlwq_state |= MLXCX_WQ_CREATED;
2570 		mlwq->mlwq_num = from_be24(out.mlxo_create_rq_rqn);
2571 	}
2572 	mlxcx_cmd_fini(mlxp, &cmd);
2573 	return (ret);
2574 }
2575 
2576 boolean_t
mlxcx_cmd_start_rq(mlxcx_t * mlxp,mlxcx_work_queue_t * mlwq)2577 mlxcx_cmd_start_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)
2578 {
2579 	mlxcx_cmd_t cmd;
2580 	mlxcx_cmd_modify_rq_in_t in;
2581 	mlxcx_cmd_modify_rq_out_t out;
2582 	boolean_t ret;
2583 	ddi_fm_error_t err;
2584 
2585 	bzero(&in, sizeof (in));
2586 	bzero(&out, sizeof (out));
2587 
2588 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));
2589 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);
2590 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);
2591 	VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED);
2592 
2593 	/*
2594 	 * Before starting the queue, we have to be sure that it is
2595 	 * empty and the doorbell and counters are set to 0.
2596 	 */
2597 	ASSERT(mutex_owned(&mlwq->mlwq_cq->mlcq_mtx));
2598 	ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers));
2599 	ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers_b));
2600 
2601 	mlwq->mlwq_doorbell->mlwqd_recv_counter = to_be16(0);
2602 	MLXCX_DMA_SYNC(mlwq->mlwq_doorbell_dma, DDI_DMA_SYNC_FORDEV);
2603 	ddi_fm_dma_err_get(mlwq->mlwq_doorbell_dma.mxdb_dma_handle, &err,
2604 	    DDI_FME_VERSION);
2605 	if (err.fme_status != DDI_FM_OK)
2606 		return (B_FALSE);
2607 	mlwq->mlwq_pc = 0;
2608 
2609 	mlxcx_cmd_init(mlxp, &cmd);
2610 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_rq_head,
2611 	    MLXCX_OP_MODIFY_RQ, 0);
2612 
2613 	in.mlxi_modify_rq_rqn = to_be24(mlwq->mlwq_num);
2614 
2615 	/* From state */
2616 	set_bits8(&in.mlxi_modify_rq_state, MLXCX_CMD_MODIFY_RQ_STATE,
2617 	    MLXCX_RQ_STATE_RST);
2618 	/* To state */
2619 	set_bits32(&in.mlxi_modify_rq_context.mlrqc_flags, MLXCX_RQ_STATE,
2620 	    MLXCX_RQ_STATE_RDY);
2621 
2622 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2623 		mlxcx_cmd_fini(mlxp, &cmd);
2624 		return (B_FALSE);
2625 	}
2626 	mlxcx_cmd_wait(&cmd);
2627 
2628 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2629 	if (ret) {
2630 		mlwq->mlwq_state |= MLXCX_WQ_STARTED;
2631 	}
2632 	mlxcx_cmd_fini(mlxp, &cmd);
2633 	return (ret);
2634 }
2635 
2636 boolean_t
mlxcx_cmd_stop_rq(mlxcx_t * mlxp,mlxcx_work_queue_t * mlwq)2637 mlxcx_cmd_stop_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)
2638 {
2639 	mlxcx_cmd_t cmd;
2640 	mlxcx_cmd_modify_rq_in_t in;
2641 	mlxcx_cmd_modify_rq_out_t out;
2642 	boolean_t ret;
2643 
2644 	bzero(&in, sizeof (in));
2645 	bzero(&out, sizeof (out));
2646 
2647 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));
2648 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);
2649 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);
2650 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_STARTED);
2651 
2652 	mlxcx_cmd_init(mlxp, &cmd);
2653 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_rq_head,
2654 	    MLXCX_OP_MODIFY_RQ, 0);
2655 
2656 	in.mlxi_modify_rq_rqn = to_be24(mlwq->mlwq_num);
2657 
2658 	/* From state */
2659 	set_bits8(&in.mlxi_modify_rq_state, MLXCX_CMD_MODIFY_RQ_STATE,
2660 	    MLXCX_RQ_STATE_RDY);
2661 	/* To state */
2662 	set_bits32(&in.mlxi_modify_rq_context.mlrqc_flags, MLXCX_RQ_STATE,
2663 	    MLXCX_RQ_STATE_RST);
2664 
2665 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2666 		mlxcx_cmd_fini(mlxp, &cmd);
2667 		return (B_FALSE);
2668 	}
2669 	mlxcx_cmd_wait(&cmd);
2670 
2671 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2672 	if (ret) {
2673 		mlwq->mlwq_state &= ~MLXCX_WQ_STARTED;
2674 	}
2675 	mlxcx_cmd_fini(mlxp, &cmd);
2676 	return (ret);
2677 }
2678 
2679 boolean_t
mlxcx_cmd_destroy_rq(mlxcx_t * mlxp,mlxcx_work_queue_t * mlwq)2680 mlxcx_cmd_destroy_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)
2681 {
2682 	mlxcx_cmd_t cmd;
2683 	mlxcx_cmd_destroy_rq_in_t in;
2684 	mlxcx_cmd_destroy_rq_out_t out;
2685 	boolean_t ret;
2686 
2687 	bzero(&in, sizeof (in));
2688 	bzero(&out, sizeof (out));
2689 
2690 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));
2691 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);
2692 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);
2693 	VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED);
2694 
2695 	mlxcx_cmd_init(mlxp, &cmd);
2696 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_rq_head,
2697 	    MLXCX_OP_DESTROY_RQ, 0);
2698 
2699 	in.mlxi_destroy_rq_rqn = to_be24(mlwq->mlwq_num);
2700 
2701 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2702 		mlxcx_cmd_fini(mlxp, &cmd);
2703 		return (B_FALSE);
2704 	}
2705 	mlxcx_cmd_wait(&cmd);
2706 
2707 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2708 	if (ret) {
2709 		mlwq->mlwq_state |= MLXCX_WQ_DESTROYED;
2710 	}
2711 	mlxcx_cmd_fini(mlxp, &cmd);
2712 	return (ret);
2713 }
2714 
2715 boolean_t
mlxcx_cmd_create_tir(mlxcx_t * mlxp,mlxcx_tir_t * mltir)2716 mlxcx_cmd_create_tir(mlxcx_t *mlxp, mlxcx_tir_t *mltir)
2717 {
2718 	mlxcx_cmd_t cmd;
2719 	mlxcx_cmd_create_tir_in_t in;
2720 	mlxcx_cmd_create_tir_out_t out;
2721 	mlxcx_tir_ctx_t *ctx;
2722 	boolean_t ret;
2723 
2724 	bzero(&in, sizeof (in));
2725 	bzero(&out, sizeof (out));
2726 
2727 	VERIFY0(mltir->mltir_state & MLXCX_TIR_CREATED);
2728 
2729 	mlxcx_cmd_init(mlxp, &cmd);
2730 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_tir_head,
2731 	    MLXCX_OP_CREATE_TIR, 0);
2732 
2733 	ctx = &in.mlxi_create_tir_context;
2734 	ctx->mltirc_transport_domain = to_be24(mltir->mltir_tdom->mltd_num);
2735 	set_bits8(&ctx->mltirc_disp_type, MLXCX_TIR_CTX_DISP_TYPE,
2736 	    mltir->mltir_type);
2737 	switch (mltir->mltir_type) {
2738 	case MLXCX_TIR_INDIRECT:
2739 		VERIFY(mltir->mltir_rqtable != NULL);
2740 		VERIFY(mltir->mltir_rqtable->mlrqt_state & MLXCX_RQT_CREATED);
2741 		ctx->mltirc_indirect_table =
2742 		    to_be24(mltir->mltir_rqtable->mlrqt_num);
2743 		set_bits8(&ctx->mltirc_hash_lb, MLXCX_TIR_RX_HASH_FN,
2744 		    mltir->mltir_hash_fn);
2745 		bcopy(mltir->mltir_toeplitz_key,
2746 		    ctx->mltirc_rx_hash_toeplitz_key,
2747 		    sizeof (ctx->mltirc_rx_hash_toeplitz_key));
2748 		set_bits32(&ctx->mltirc_rx_hash_fields_outer,
2749 		    MLXCX_RX_HASH_L3_TYPE, mltir->mltir_l3_type);
2750 		set_bits32(&ctx->mltirc_rx_hash_fields_outer,
2751 		    MLXCX_RX_HASH_L4_TYPE, mltir->mltir_l4_type);
2752 		set_bits32(&ctx->mltirc_rx_hash_fields_outer,
2753 		    MLXCX_RX_HASH_FIELDS, mltir->mltir_hash_fields);
2754 		break;
2755 	case MLXCX_TIR_DIRECT:
2756 		VERIFY(mltir->mltir_rq != NULL);
2757 		VERIFY(mltir->mltir_rq->mlwq_state & MLXCX_WQ_CREATED);
2758 		ctx->mltirc_inline_rqn = to_be24(mltir->mltir_rq->mlwq_num);
2759 		break;
2760 	default:
2761 		VERIFY(0);
2762 	}
2763 
2764 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2765 		mlxcx_cmd_fini(mlxp, &cmd);
2766 		return (B_FALSE);
2767 	}
2768 	mlxcx_cmd_wait(&cmd);
2769 
2770 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2771 	if (ret) {
2772 		mltir->mltir_state |= MLXCX_TIR_CREATED;
2773 		mltir->mltir_num = from_be24(out.mlxo_create_tir_tirn);
2774 	}
2775 	mlxcx_cmd_fini(mlxp, &cmd);
2776 	return (ret);
2777 }
2778 
2779 boolean_t
mlxcx_cmd_destroy_tir(mlxcx_t * mlxp,mlxcx_tir_t * mltir)2780 mlxcx_cmd_destroy_tir(mlxcx_t *mlxp, mlxcx_tir_t *mltir)
2781 {
2782 	mlxcx_cmd_t cmd;
2783 	mlxcx_cmd_destroy_tir_in_t in;
2784 	mlxcx_cmd_destroy_tir_out_t out;
2785 	boolean_t ret;
2786 
2787 	bzero(&in, sizeof (in));
2788 	bzero(&out, sizeof (out));
2789 
2790 	VERIFY(mltir->mltir_state & MLXCX_TIR_CREATED);
2791 	VERIFY0(mltir->mltir_state & MLXCX_TIR_DESTROYED);
2792 
2793 	mlxcx_cmd_init(mlxp, &cmd);
2794 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_tir_head,
2795 	    MLXCX_OP_DESTROY_TIR, 0);
2796 
2797 	in.mlxi_destroy_tir_tirn = to_be24(mltir->mltir_num);
2798 
2799 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2800 		mlxcx_cmd_fini(mlxp, &cmd);
2801 		return (B_FALSE);
2802 	}
2803 	mlxcx_cmd_wait(&cmd);
2804 
2805 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2806 	if (ret) {
2807 		mltir->mltir_state |= MLXCX_TIR_DESTROYED;
2808 	}
2809 	mlxcx_cmd_fini(mlxp, &cmd);
2810 	return (ret);
2811 }
2812 
2813 boolean_t
mlxcx_cmd_create_tis(mlxcx_t * mlxp,mlxcx_tis_t * mltis)2814 mlxcx_cmd_create_tis(mlxcx_t *mlxp, mlxcx_tis_t *mltis)
2815 {
2816 	mlxcx_cmd_t cmd;
2817 	mlxcx_cmd_create_tis_in_t in;
2818 	mlxcx_cmd_create_tis_out_t out;
2819 	mlxcx_tis_ctx_t *ctx;
2820 	boolean_t ret;
2821 
2822 	bzero(&in, sizeof (in));
2823 	bzero(&out, sizeof (out));
2824 
2825 	VERIFY0(mltis->mltis_state & MLXCX_TIS_CREATED);
2826 
2827 	mlxcx_cmd_init(mlxp, &cmd);
2828 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_tis_head,
2829 	    MLXCX_OP_CREATE_TIS, 0);
2830 
2831 	ctx = &in.mlxi_create_tis_context;
2832 	ctx->mltisc_transport_domain = to_be24(mltis->mltis_tdom->mltd_num);
2833 
2834 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2835 		mlxcx_cmd_fini(mlxp, &cmd);
2836 		return (B_FALSE);
2837 	}
2838 	mlxcx_cmd_wait(&cmd);
2839 
2840 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2841 	if (ret) {
2842 		mltis->mltis_state |= MLXCX_TIS_CREATED;
2843 		mltis->mltis_num = from_be24(out.mlxo_create_tis_tisn);
2844 	}
2845 	mlxcx_cmd_fini(mlxp, &cmd);
2846 	return (ret);
2847 }
2848 
2849 boolean_t
mlxcx_cmd_destroy_tis(mlxcx_t * mlxp,mlxcx_tis_t * mltis)2850 mlxcx_cmd_destroy_tis(mlxcx_t *mlxp, mlxcx_tis_t *mltis)
2851 {
2852 	mlxcx_cmd_t cmd;
2853 	mlxcx_cmd_destroy_tis_in_t in;
2854 	mlxcx_cmd_destroy_tis_out_t out;
2855 	boolean_t ret;
2856 
2857 	bzero(&in, sizeof (in));
2858 	bzero(&out, sizeof (out));
2859 
2860 	VERIFY(mltis->mltis_state & MLXCX_TIR_CREATED);
2861 	VERIFY0(mltis->mltis_state & MLXCX_TIR_DESTROYED);
2862 
2863 	mlxcx_cmd_init(mlxp, &cmd);
2864 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_tis_head,
2865 	    MLXCX_OP_DESTROY_TIS, 0);
2866 
2867 	in.mlxi_destroy_tis_tisn = to_be24(mltis->mltis_num);
2868 
2869 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2870 		mlxcx_cmd_fini(mlxp, &cmd);
2871 		return (B_FALSE);
2872 	}
2873 	mlxcx_cmd_wait(&cmd);
2874 
2875 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2876 	if (ret) {
2877 		mltis->mltis_state |= MLXCX_TIS_DESTROYED;
2878 	}
2879 	mlxcx_cmd_fini(mlxp, &cmd);
2880 	return (ret);
2881 }
2882 
2883 boolean_t
mlxcx_cmd_create_flow_table(mlxcx_t * mlxp,mlxcx_flow_table_t * mlft)2884 mlxcx_cmd_create_flow_table(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft)
2885 {
2886 	mlxcx_cmd_t cmd;
2887 	mlxcx_cmd_create_flow_table_in_t in;
2888 	mlxcx_cmd_create_flow_table_out_t out;
2889 	mlxcx_flow_table_ctx_t *ctx;
2890 	boolean_t ret;
2891 
2892 	bzero(&in, sizeof (in));
2893 	bzero(&out, sizeof (out));
2894 
2895 	ASSERT(mutex_owned(&mlft->mlft_mtx));
2896 	VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED);
2897 
2898 	mlxcx_cmd_init(mlxp, &cmd);
2899 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_flow_table_head,
2900 	    MLXCX_OP_CREATE_FLOW_TABLE, 0);
2901 
2902 	in.mlxi_create_flow_table_vport_number =
2903 	    to_be16(mlft->mlft_port->mlp_num);
2904 	in.mlxi_create_flow_table_table_type = mlft->mlft_type;
2905 	ctx = &in.mlxi_create_flow_table_context;
2906 	ctx->mlftc_log_size = mlft->mlft_entshift;
2907 	ctx->mlftc_level = mlft->mlft_level;
2908 
2909 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2910 		mlxcx_cmd_fini(mlxp, &cmd);
2911 		return (B_FALSE);
2912 	}
2913 	mlxcx_cmd_wait(&cmd);
2914 
2915 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2916 	if (ret) {
2917 		mlft->mlft_num = from_be24(out.mlxo_create_flow_table_table_id);
2918 		mlft->mlft_state |= MLXCX_FLOW_TABLE_CREATED;
2919 	}
2920 	mlxcx_cmd_fini(mlxp, &cmd);
2921 	return (ret);
2922 }
2923 
2924 boolean_t
mlxcx_cmd_destroy_flow_table(mlxcx_t * mlxp,mlxcx_flow_table_t * mlft)2925 mlxcx_cmd_destroy_flow_table(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft)
2926 {
2927 	mlxcx_cmd_t cmd;
2928 	mlxcx_cmd_destroy_flow_table_in_t in;
2929 	mlxcx_cmd_destroy_flow_table_out_t out;
2930 	boolean_t ret;
2931 
2932 	bzero(&in, sizeof (in));
2933 	bzero(&out, sizeof (out));
2934 
2935 	ASSERT(mutex_owned(&mlft->mlft_mtx));
2936 	VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED);
2937 	VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED);
2938 
2939 	mlxcx_cmd_init(mlxp, &cmd);
2940 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_flow_table_head,
2941 	    MLXCX_OP_DESTROY_FLOW_TABLE, 0);
2942 
2943 	in.mlxi_destroy_flow_table_vport_number =
2944 	    to_be16(mlft->mlft_port->mlp_num);
2945 	in.mlxi_destroy_flow_table_table_type = mlft->mlft_type;
2946 	in.mlxi_destroy_flow_table_table_id = to_be24(mlft->mlft_num);
2947 
2948 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2949 		mlxcx_cmd_fini(mlxp, &cmd);
2950 		return (B_FALSE);
2951 	}
2952 	mlxcx_cmd_wait(&cmd);
2953 
2954 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2955 	if (ret) {
2956 		mlft->mlft_state |= MLXCX_FLOW_TABLE_DESTROYED;
2957 	}
2958 	mlxcx_cmd_fini(mlxp, &cmd);
2959 	return (ret);
2960 }
2961 
2962 boolean_t
mlxcx_cmd_set_flow_table_root(mlxcx_t * mlxp,mlxcx_flow_table_t * mlft)2963 mlxcx_cmd_set_flow_table_root(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft)
2964 {
2965 	mlxcx_cmd_t cmd;
2966 	mlxcx_cmd_set_flow_table_root_in_t in;
2967 	mlxcx_cmd_set_flow_table_root_out_t out;
2968 	boolean_t ret;
2969 
2970 	bzero(&in, sizeof (in));
2971 	bzero(&out, sizeof (out));
2972 
2973 	ASSERT(mutex_owned(&mlft->mlft_mtx));
2974 	VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED);
2975 	VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED);
2976 
2977 	mlxcx_cmd_init(mlxp, &cmd);
2978 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_flow_table_root_head,
2979 	    MLXCX_OP_SET_FLOW_TABLE_ROOT, 0);
2980 
2981 	in.mlxi_set_flow_table_root_vport_number =
2982 	    to_be16(mlft->mlft_port->mlp_num);
2983 	in.mlxi_set_flow_table_root_table_type = mlft->mlft_type;
2984 	in.mlxi_set_flow_table_root_table_id = to_be24(mlft->mlft_num);
2985 
2986 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
2987 		mlxcx_cmd_fini(mlxp, &cmd);
2988 		return (B_FALSE);
2989 	}
2990 	mlxcx_cmd_wait(&cmd);
2991 
2992 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
2993 	if (ret) {
2994 		mlft->mlft_state |= MLXCX_FLOW_TABLE_ROOT;
2995 	}
2996 	mlxcx_cmd_fini(mlxp, &cmd);
2997 	return (ret);
2998 }
2999 
3000 boolean_t
mlxcx_cmd_create_flow_group(mlxcx_t * mlxp,mlxcx_flow_group_t * mlfg)3001 mlxcx_cmd_create_flow_group(mlxcx_t *mlxp, mlxcx_flow_group_t *mlfg)
3002 {
3003 	mlxcx_cmd_t cmd;
3004 	mlxcx_cmd_create_flow_group_in_t in;
3005 	mlxcx_cmd_create_flow_group_out_t out;
3006 	boolean_t ret;
3007 	const mlxcx_flow_table_t *mlft;
3008 	mlxcx_flow_header_match_t *hdrs;
3009 	mlxcx_flow_params_match_t *params;
3010 
3011 	bzero(&in, sizeof (in));
3012 	bzero(&out, sizeof (out));
3013 
3014 	mlft = mlfg->mlfg_table;
3015 	ASSERT(mutex_owned(&mlft->mlft_mtx));
3016 	VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED);
3017 	VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED);
3018 	VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED);
3019 
3020 	mlxcx_cmd_init(mlxp, &cmd);
3021 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_flow_group_head,
3022 	    MLXCX_OP_CREATE_FLOW_GROUP, 0);
3023 
3024 	in.mlxi_create_flow_group_vport_number =
3025 	    to_be16(mlft->mlft_port->mlp_num);
3026 	in.mlxi_create_flow_group_table_type = mlft->mlft_type;
3027 	in.mlxi_create_flow_group_table_id = to_be24(mlft->mlft_num);
3028 	in.mlxi_create_flow_group_start_flow_index =
3029 	    to_be32(mlfg->mlfg_start_idx);
3030 	in.mlxi_create_flow_group_end_flow_index =
3031 	    to_be32(mlfg->mlfg_start_idx + (mlfg->mlfg_size - 1));
3032 
3033 	hdrs = &in.mlxi_create_flow_group_match_criteria.mlfm_outer_headers;
3034 	params = &in.mlxi_create_flow_group_match_criteria.mlfm_misc_parameters;
3035 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SMAC) {
3036 		in.mlxi_create_flow_group_match_criteria_en |=
3037 		    MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS;
3038 		(void) memset(&hdrs->mlfh_smac, 0xff, sizeof (hdrs->mlfh_smac));
3039 	}
3040 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DMAC) {
3041 		in.mlxi_create_flow_group_match_criteria_en |=
3042 		    MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS;
3043 		(void) memset(&hdrs->mlfh_dmac, 0xff, sizeof (hdrs->mlfh_dmac));
3044 	}
3045 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN) {
3046 		in.mlxi_create_flow_group_match_criteria_en |=
3047 		    MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS;
3048 		set_bit24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_CVLAN_TAG);
3049 		set_bit24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_SVLAN_TAG);
3050 	}
3051 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VID) {
3052 		ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN);
3053 		set_bits16(&hdrs->mlfh_first_vid_flags,
3054 		    MLXCX_FLOW_HDR_FIRST_VID, UINT16_MAX);
3055 	}
3056 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER) {
3057 		in.mlxi_create_flow_group_match_criteria_en |=
3058 		    MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS;
3059 		set_bits24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_IP_VERSION,
3060 		    UINT32_MAX);
3061 	}
3062 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) {
3063 		ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER);
3064 		(void) memset(&hdrs->mlfh_src_ip, 0xff,
3065 		    sizeof (hdrs->mlfh_src_ip));
3066 	}
3067 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) {
3068 		ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER);
3069 		(void) memset(&hdrs->mlfh_src_ip, 0xff,
3070 		    sizeof (hdrs->mlfh_dst_ip));
3071 	}
3072 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_PROTO) {
3073 		in.mlxi_create_flow_group_match_criteria_en |=
3074 		    MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS;
3075 		hdrs->mlfh_ip_protocol = UINT8_MAX;
3076 	}
3077 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) {
3078 		ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER);
3079 		(void) memset(&hdrs->mlfh_src_ip, 0xff,
3080 		    sizeof (hdrs->mlfh_src_ip));
3081 	}
3082 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) {
3083 		ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER);
3084 		(void) memset(&hdrs->mlfh_src_ip, 0xff,
3085 		    sizeof (hdrs->mlfh_dst_ip));
3086 	}
3087 
3088 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SQN) {
3089 		in.mlxi_create_flow_group_match_criteria_en |=
3090 		    MLXCX_FLOW_GROUP_MATCH_MISC_PARAMS;
3091 		params->mlfp_source_sqn = to_be24(UINT32_MAX);
3092 	}
3093 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VXLAN) {
3094 		in.mlxi_create_flow_group_match_criteria_en |=
3095 		    MLXCX_FLOW_GROUP_MATCH_MISC_PARAMS;
3096 		params->mlfp_vxlan_vni = to_be24(UINT32_MAX);
3097 	}
3098 
3099 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
3100 		mlxcx_cmd_fini(mlxp, &cmd);
3101 		return (B_FALSE);
3102 	}
3103 	mlxcx_cmd_wait(&cmd);
3104 
3105 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
3106 	if (ret) {
3107 		mlfg->mlfg_state |= MLXCX_FLOW_GROUP_CREATED;
3108 		mlfg->mlfg_num = from_be24(out.mlxo_create_flow_group_group_id);
3109 	}
3110 	mlxcx_cmd_fini(mlxp, &cmd);
3111 	return (ret);
3112 }
3113 
3114 boolean_t
mlxcx_cmd_destroy_flow_group(mlxcx_t * mlxp,mlxcx_flow_group_t * mlfg)3115 mlxcx_cmd_destroy_flow_group(mlxcx_t *mlxp, mlxcx_flow_group_t *mlfg)
3116 {
3117 	mlxcx_cmd_t cmd;
3118 	mlxcx_cmd_destroy_flow_group_in_t in;
3119 	mlxcx_cmd_destroy_flow_group_out_t out;
3120 	boolean_t ret;
3121 	const mlxcx_flow_table_t *mlft;
3122 
3123 	bzero(&in, sizeof (in));
3124 	bzero(&out, sizeof (out));
3125 
3126 	mlft = mlfg->mlfg_table;
3127 	ASSERT(mutex_owned(&mlft->mlft_mtx));
3128 	VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED);
3129 	VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED);
3130 	VERIFY(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED);
3131 	VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_DESTROYED);
3132 
3133 	mlxcx_cmd_init(mlxp, &cmd);
3134 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_flow_group_head,
3135 	    MLXCX_OP_DESTROY_FLOW_GROUP, 0);
3136 
3137 	in.mlxi_destroy_flow_group_vport_number =
3138 	    to_be16(mlft->mlft_port->mlp_num);
3139 	in.mlxi_destroy_flow_group_table_type = mlft->mlft_type;
3140 	in.mlxi_destroy_flow_group_table_id = to_be24(mlft->mlft_num);
3141 	in.mlxi_destroy_flow_group_group_id = to_be32(mlfg->mlfg_num);
3142 
3143 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
3144 		mlxcx_cmd_fini(mlxp, &cmd);
3145 		return (B_FALSE);
3146 	}
3147 	mlxcx_cmd_wait(&cmd);
3148 
3149 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
3150 	if (ret) {
3151 		mlfg->mlfg_state |= MLXCX_FLOW_GROUP_DESTROYED;
3152 	}
3153 	mlxcx_cmd_fini(mlxp, &cmd);
3154 	return (ret);
3155 }
3156 
3157 boolean_t
mlxcx_cmd_set_flow_table_entry(mlxcx_t * mlxp,mlxcx_flow_entry_t * mlfe)3158 mlxcx_cmd_set_flow_table_entry(mlxcx_t *mlxp, mlxcx_flow_entry_t *mlfe)
3159 {
3160 	mlxcx_cmd_t cmd;
3161 	mlxcx_cmd_set_flow_table_entry_in_t in;
3162 	mlxcx_cmd_set_flow_table_entry_out_t out;
3163 	boolean_t ret;
3164 	size_t insize;
3165 	mlxcx_flow_entry_ctx_t *ctx;
3166 	const mlxcx_flow_table_t *mlft;
3167 	mlxcx_flow_group_t *mlfg;
3168 	mlxcx_flow_dest_t *d;
3169 	uint_t i;
3170 	mlxcx_flow_header_match_t *hdrs;
3171 	mlxcx_flow_params_match_t *params;
3172 	mlxcx_cmd_set_flow_table_entry_opmod_t opmod;
3173 
3174 	bzero(&in, sizeof (in));
3175 	bzero(&out, sizeof (out));
3176 
3177 	mlft = mlfe->mlfe_table;
3178 	ASSERT(mutex_owned(&mlft->mlft_mtx));
3179 	VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED);
3180 	VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED);
3181 
3182 	mlfg = mlfe->mlfe_group;
3183 	VERIFY(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED);
3184 	VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_DESTROYED);
3185 
3186 	opmod = MLXCX_CMD_FLOW_ENTRY_SET_NEW;
3187 	if (mlfe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED) {
3188 		ASSERT(mlfe->mlfe_state & MLXCX_FLOW_ENTRY_DIRTY);
3189 		opmod = MLXCX_CMD_FLOW_ENTRY_MODIFY;
3190 	}
3191 
3192 	mlxcx_cmd_init(mlxp, &cmd);
3193 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_flow_table_entry_head,
3194 	    MLXCX_OP_SET_FLOW_TABLE_ENTRY, opmod);
3195 
3196 	in.mlxi_set_flow_table_entry_vport_number =
3197 	    to_be16(mlft->mlft_port->mlp_num);
3198 	in.mlxi_set_flow_table_entry_table_type = mlft->mlft_type;
3199 	in.mlxi_set_flow_table_entry_table_id = to_be24(mlft->mlft_num);
3200 	in.mlxi_set_flow_table_entry_flow_index = to_be32(mlfe->mlfe_index);
3201 
3202 	if (mlfe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED) {
3203 		set_bit8(&in.mlxi_set_flow_table_entry_modify_bitmask,
3204 		    MLXCX_CMD_FLOW_ENTRY_SET_ACTION);
3205 		set_bit8(&in.mlxi_set_flow_table_entry_modify_bitmask,
3206 		    MLXCX_CMD_FLOW_ENTRY_SET_DESTINATION);
3207 	}
3208 
3209 	ctx = &in.mlxi_set_flow_table_entry_context;
3210 	ctx->mlfec_group_id = to_be32(mlfg->mlfg_num);
3211 
3212 	insize = offsetof(mlxcx_cmd_set_flow_table_entry_in_t,
3213 	    mlxi_set_flow_table_entry_context) +
3214 	    offsetof(mlxcx_flow_entry_ctx_t, mlfec_destination);
3215 
3216 	ctx->mlfec_action = to_be16(mlfe->mlfe_action);
3217 
3218 	switch (mlfe->mlfe_action) {
3219 	case MLXCX_FLOW_ACTION_ALLOW:
3220 	case MLXCX_FLOW_ACTION_DROP:
3221 		break;
3222 	case MLXCX_FLOW_ACTION_FORWARD:
3223 		ASSERT3U(mlfe->mlfe_ndest, <=, MLXCX_FLOW_MAX_DESTINATIONS);
3224 		ASSERT3U(mlfe->mlfe_ndest, <=,
3225 		    mlxp->mlx_caps->mlc_max_rx_fe_dest);
3226 		ctx->mlfec_destination_list_size = to_be24(mlfe->mlfe_ndest);
3227 		for (i = 0; i < mlfe->mlfe_ndest; ++i) {
3228 			insize += sizeof (mlxcx_flow_dest_t);
3229 			d = &ctx->mlfec_destination[i];
3230 			if (mlfe->mlfe_dest[i].mlfed_tir != NULL) {
3231 				d->mlfd_destination_type = MLXCX_FLOW_DEST_TIR;
3232 				d->mlfd_destination_id = to_be24(
3233 				    mlfe->mlfe_dest[i].mlfed_tir->mltir_num);
3234 			} else if (mlfe->mlfe_dest[i].mlfed_flow != NULL) {
3235 				d->mlfd_destination_type =
3236 				    MLXCX_FLOW_DEST_FLOW_TABLE;
3237 				d->mlfd_destination_id = to_be24(
3238 				    mlfe->mlfe_dest[i].mlfed_flow->mlft_num);
3239 			} else {
3240 				/* Invalid flow entry destination */
3241 				VERIFY(0);
3242 			}
3243 		}
3244 		break;
3245 	case MLXCX_FLOW_ACTION_COUNT:
3246 		/* We don't support count actions yet. */
3247 		VERIFY(0);
3248 		break;
3249 	case MLXCX_FLOW_ACTION_ENCAP:
3250 	case MLXCX_FLOW_ACTION_DECAP:
3251 		/* We don't support encap/decap actions yet. */
3252 		VERIFY(0);
3253 		break;
3254 	}
3255 
3256 	hdrs = &ctx->mlfec_match_value.mlfm_outer_headers;
3257 	params = &ctx->mlfec_match_value.mlfm_misc_parameters;
3258 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SMAC) {
3259 		bcopy(mlfe->mlfe_smac, hdrs->mlfh_smac,
3260 		    sizeof (hdrs->mlfh_smac));
3261 	}
3262 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DMAC) {
3263 		bcopy(mlfe->mlfe_dmac, hdrs->mlfh_dmac,
3264 		    sizeof (hdrs->mlfh_dmac));
3265 	}
3266 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN) {
3267 		switch (mlfe->mlfe_vlan_type) {
3268 		case MLXCX_VLAN_TYPE_CVLAN:
3269 			set_bit24(&hdrs->mlfh_tcp_ip_flags,
3270 			    MLXCX_FLOW_HDR_CVLAN_TAG);
3271 			break;
3272 		case MLXCX_VLAN_TYPE_SVLAN:
3273 			set_bit24(&hdrs->mlfh_tcp_ip_flags,
3274 			    MLXCX_FLOW_HDR_SVLAN_TAG);
3275 			break;
3276 		default:
3277 			break;
3278 		}
3279 	}
3280 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VID) {
3281 		ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN);
3282 		set_bits16(&hdrs->mlfh_first_vid_flags,
3283 		    MLXCX_FLOW_HDR_FIRST_VID, mlfe->mlfe_vid);
3284 	}
3285 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER) {
3286 		set_bits24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_IP_VERSION,
3287 		    mlfe->mlfe_ip_version);
3288 	}
3289 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) {
3290 		ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER);
3291 		bcopy(mlfe->mlfe_srcip, hdrs->mlfh_src_ip,
3292 		    sizeof (hdrs->mlfh_src_ip));
3293 	}
3294 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) {
3295 		ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER);
3296 		bcopy(mlfe->mlfe_dstip, hdrs->mlfh_src_ip,
3297 		    sizeof (hdrs->mlfh_dst_ip));
3298 	}
3299 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_PROTO) {
3300 		hdrs->mlfh_ip_protocol = mlfe->mlfe_ip_proto;
3301 	}
3302 
3303 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SQN) {
3304 		params->mlfp_source_sqn = to_be24(mlfe->mlfe_sqn);
3305 	}
3306 	if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VXLAN) {
3307 		params->mlfp_vxlan_vni = to_be24(mlfe->mlfe_vxlan_vni);
3308 	}
3309 
3310 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) {
3311 		mlxcx_cmd_fini(mlxp, &cmd);
3312 		return (B_FALSE);
3313 	}
3314 	mlxcx_cmd_wait(&cmd);
3315 
3316 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
3317 	if (ret) {
3318 		mlfe->mlfe_state |= MLXCX_FLOW_ENTRY_CREATED;
3319 		mlfe->mlfe_state &= ~MLXCX_FLOW_ENTRY_DIRTY;
3320 		mlfg->mlfg_state |= MLXCX_FLOW_GROUP_BUSY;
3321 	}
3322 	mlxcx_cmd_fini(mlxp, &cmd);
3323 	return (ret);
3324 }
3325 
3326 boolean_t
mlxcx_cmd_delete_flow_table_entry(mlxcx_t * mlxp,mlxcx_flow_entry_t * mlfe)3327 mlxcx_cmd_delete_flow_table_entry(mlxcx_t *mlxp, mlxcx_flow_entry_t *mlfe)
3328 {
3329 	mlxcx_cmd_t cmd;
3330 	mlxcx_cmd_delete_flow_table_entry_in_t in;
3331 	mlxcx_cmd_delete_flow_table_entry_out_t out;
3332 	boolean_t ret;
3333 	const mlxcx_flow_table_t *mlft;
3334 
3335 	bzero(&in, sizeof (in));
3336 	bzero(&out, sizeof (out));
3337 
3338 	mlft = mlfe->mlfe_table;
3339 	ASSERT(mutex_owned(&mlft->mlft_mtx));
3340 	VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED);
3341 	VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED);
3342 
3343 	mlxcx_cmd_init(mlxp, &cmd);
3344 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_delete_flow_table_entry_head,
3345 	    MLXCX_OP_DELETE_FLOW_TABLE_ENTRY, 0);
3346 
3347 	in.mlxi_delete_flow_table_entry_vport_number =
3348 	    to_be16(mlft->mlft_port->mlp_num);
3349 	in.mlxi_delete_flow_table_entry_table_type = mlft->mlft_type;
3350 	in.mlxi_delete_flow_table_entry_table_id = to_be24(mlft->mlft_num);
3351 	in.mlxi_delete_flow_table_entry_flow_index = to_be32(mlfe->mlfe_index);
3352 
3353 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
3354 		mlxcx_cmd_fini(mlxp, &cmd);
3355 		return (B_FALSE);
3356 	}
3357 	mlxcx_cmd_wait(&cmd);
3358 
3359 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
3360 	if (ret) {
3361 		/*
3362 		 * Note that flow entries have a different lifecycle to most
3363 		 * other things we create -- we have to be able to re-use them
3364 		 * after they have been deleted, since they exist at a fixed
3365 		 * position in their flow table.
3366 		 *
3367 		 * So we clear the CREATED bit here for them to let us call
3368 		 * create_flow_table_entry() on the same entry again later.
3369 		 */
3370 		mlfe->mlfe_state &= ~MLXCX_FLOW_ENTRY_CREATED;
3371 		mlfe->mlfe_state |= MLXCX_FLOW_ENTRY_DELETED;
3372 	}
3373 	mlxcx_cmd_fini(mlxp, &cmd);
3374 	return (ret);
3375 }
3376 
3377 boolean_t
mlxcx_cmd_create_sq(mlxcx_t * mlxp,mlxcx_work_queue_t * mlwq)3378 mlxcx_cmd_create_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)
3379 {
3380 	mlxcx_cmd_t cmd;
3381 	mlxcx_cmd_create_sq_in_t in;
3382 	mlxcx_cmd_create_sq_out_t out;
3383 	boolean_t ret;
3384 	mlxcx_sq_ctx_t *ctx;
3385 	size_t rem, insize;
3386 	const ddi_dma_cookie_t *c;
3387 	uint64_t pa, npages;
3388 
3389 	bzero(&in, sizeof (in));
3390 	bzero(&out, sizeof (out));
3391 
3392 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));
3393 	VERIFY3U(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_SENDQ);
3394 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);
3395 	VERIFY0(mlwq->mlwq_state & MLXCX_WQ_CREATED);
3396 
3397 	mlxcx_cmd_init(mlxp, &cmd);
3398 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_sq_head,
3399 	    MLXCX_OP_CREATE_SQ, 0);
3400 
3401 	ctx = &in.mlxi_create_sq_context;
3402 
3403 	set_bit32(&ctx->mlsqc_flags, MLXCX_SQ_FLAGS_RLKEY);
3404 	set_bit32(&ctx->mlsqc_flags, MLXCX_SQ_FLAGS_FLUSH_IN_ERROR);
3405 	set_bits32(&ctx->mlsqc_flags, MLXCX_SQ_MIN_WQE_INLINE,
3406 	    mlwq->mlwq_inline_mode);
3407 	ctx->mlsqc_cqn = to_be24(mlwq->mlwq_cq->mlcq_num);
3408 
3409 	VERIFY(mlwq->mlwq_tis != NULL);
3410 	ctx->mlsqc_tis_lst_sz = to_be16(1);
3411 	ctx->mlsqc_tis_num = to_be24(mlwq->mlwq_tis->mltis_num);
3412 
3413 	set_bits32(&ctx->mlsqc_wq.mlwqc_flags, MLXCX_WORKQ_CTX_TYPE,
3414 	    MLXCX_WORKQ_TYPE_CYCLIC);
3415 	ctx->mlsqc_wq.mlwqc_pd = to_be24(mlwq->mlwq_pd->mlpd_num);
3416 	ctx->mlsqc_wq.mlwqc_uar_page = to_be24(mlwq->mlwq_uar->mlu_num);
3417 	ctx->mlsqc_wq.mlwqc_log_wq_sz = mlwq->mlwq_entshift;
3418 	ctx->mlsqc_wq.mlwqc_log_wq_stride = MLXCX_SENDQ_STRIDE_SHIFT;
3419 
3420 	c = mlxcx_dma_cookie_one(&mlwq->mlwq_doorbell_dma);
3421 	ctx->mlsqc_wq.mlwqc_dbr_addr = to_be64(c->dmac_laddress);
3422 	ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_workq_doorbell_t));
3423 
3424 	npages = 0;
3425 	c = NULL;
3426 	while ((c = mlxcx_dma_cookie_iter(&mlwq->mlwq_dma, c)) != NULL) {
3427 		pa = c->dmac_laddress;
3428 		rem = c->dmac_size;
3429 		while (rem > 0) {
3430 			ASSERT3U(pa & 0xfff, ==, 0);
3431 			ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE);
3432 			ctx->mlsqc_wq.mlwqc_pas[npages++] = to_be64(pa);
3433 			rem -= MLXCX_HW_PAGE_SIZE;
3434 			pa += MLXCX_HW_PAGE_SIZE;
3435 		}
3436 	}
3437 	ASSERT3U(npages, <=, MLXCX_WORKQ_CTX_MAX_ADDRESSES);
3438 
3439 	insize = offsetof(mlxcx_cmd_create_sq_in_t, mlxi_create_sq_context) +
3440 	    offsetof(mlxcx_sq_ctx_t, mlsqc_wq) +
3441 	    offsetof(mlxcx_workq_ctx_t, mlwqc_pas) +
3442 	    sizeof (uint64_t) * npages;
3443 
3444 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) {
3445 		mlxcx_cmd_fini(mlxp, &cmd);
3446 		return (B_FALSE);
3447 	}
3448 	mlxcx_cmd_wait(&cmd);
3449 
3450 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
3451 	if (ret) {
3452 		mlwq->mlwq_state |= MLXCX_WQ_CREATED;
3453 		mlwq->mlwq_num = from_be24(out.mlxo_create_sq_sqn);
3454 	}
3455 	mlxcx_cmd_fini(mlxp, &cmd);
3456 	return (ret);
3457 }
3458 
3459 boolean_t
mlxcx_cmd_start_sq(mlxcx_t * mlxp,mlxcx_work_queue_t * mlwq)3460 mlxcx_cmd_start_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)
3461 {
3462 	mlxcx_cmd_t cmd;
3463 	mlxcx_cmd_modify_sq_in_t in;
3464 	mlxcx_cmd_modify_sq_out_t out;
3465 	boolean_t ret;
3466 	ddi_fm_error_t err;
3467 
3468 	bzero(&in, sizeof (in));
3469 	bzero(&out, sizeof (out));
3470 
3471 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));
3472 	ASSERT(mlwq->mlwq_cq != NULL);
3473 
3474 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);
3475 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);
3476 	VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED);
3477 
3478 	/*
3479 	 * Before starting the queue, we have to be sure that it is
3480 	 * empty and the doorbell and counters are set to 0.
3481 	 */
3482 	ASSERT(mutex_owned(&mlwq->mlwq_cq->mlcq_mtx));
3483 	ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers));
3484 	ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers_b));
3485 
3486 	mlwq->mlwq_doorbell->mlwqd_recv_counter = to_be16(0);
3487 	MLXCX_DMA_SYNC(mlwq->mlwq_doorbell_dma, DDI_DMA_SYNC_FORDEV);
3488 	ddi_fm_dma_err_get(mlwq->mlwq_doorbell_dma.mxdb_dma_handle, &err,
3489 	    DDI_FME_VERSION);
3490 	if (err.fme_status != DDI_FM_OK)
3491 		return (B_FALSE);
3492 	mlwq->mlwq_pc = 0;
3493 
3494 	mlxcx_cmd_init(mlxp, &cmd);
3495 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_sq_head,
3496 	    MLXCX_OP_MODIFY_SQ, 0);
3497 
3498 	in.mlxi_modify_sq_sqn = to_be24(mlwq->mlwq_num);
3499 
3500 	/* From state */
3501 	set_bits8(&in.mlxi_modify_sq_state, MLXCX_CMD_MODIFY_SQ_STATE,
3502 	    MLXCX_SQ_STATE_RST);
3503 	/* To state */
3504 	set_bits32(&in.mlxi_modify_sq_context.mlsqc_flags, MLXCX_SQ_STATE,
3505 	    MLXCX_SQ_STATE_RDY);
3506 
3507 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
3508 		mlxcx_cmd_fini(mlxp, &cmd);
3509 		return (B_FALSE);
3510 	}
3511 	mlxcx_cmd_wait(&cmd);
3512 
3513 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
3514 	if (ret) {
3515 		mlwq->mlwq_state |= MLXCX_WQ_STARTED;
3516 	}
3517 	mlxcx_cmd_fini(mlxp, &cmd);
3518 	return (ret);
3519 }
3520 
3521 boolean_t
mlxcx_cmd_stop_sq(mlxcx_t * mlxp,mlxcx_work_queue_t * mlwq)3522 mlxcx_cmd_stop_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)
3523 {
3524 	mlxcx_cmd_t cmd;
3525 	mlxcx_cmd_modify_sq_in_t in;
3526 	mlxcx_cmd_modify_sq_out_t out;
3527 	boolean_t ret;
3528 
3529 	bzero(&in, sizeof (in));
3530 	bzero(&out, sizeof (out));
3531 
3532 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));
3533 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);
3534 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);
3535 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_STARTED);
3536 
3537 	mlxcx_cmd_init(mlxp, &cmd);
3538 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_sq_head,
3539 	    MLXCX_OP_MODIFY_SQ, 0);
3540 
3541 	in.mlxi_modify_sq_sqn = to_be24(mlwq->mlwq_num);
3542 
3543 	/* From state */
3544 	set_bits8(&in.mlxi_modify_sq_state, MLXCX_CMD_MODIFY_SQ_STATE,
3545 	    MLXCX_SQ_STATE_RDY);
3546 	/* To state */
3547 	set_bits32(&in.mlxi_modify_sq_context.mlsqc_flags, MLXCX_SQ_STATE,
3548 	    MLXCX_SQ_STATE_RST);
3549 
3550 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
3551 		mlxcx_cmd_fini(mlxp, &cmd);
3552 		return (B_FALSE);
3553 	}
3554 	mlxcx_cmd_wait(&cmd);
3555 
3556 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
3557 	if (ret) {
3558 		mlwq->mlwq_state &= ~MLXCX_WQ_STARTED;
3559 	}
3560 	mlxcx_cmd_fini(mlxp, &cmd);
3561 	return (ret);
3562 }
3563 
3564 boolean_t
mlxcx_cmd_destroy_sq(mlxcx_t * mlxp,mlxcx_work_queue_t * mlwq)3565 mlxcx_cmd_destroy_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)
3566 {
3567 	mlxcx_cmd_t cmd;
3568 	mlxcx_cmd_destroy_sq_in_t in;
3569 	mlxcx_cmd_destroy_sq_out_t out;
3570 	boolean_t ret;
3571 
3572 	bzero(&in, sizeof (in));
3573 	bzero(&out, sizeof (out));
3574 
3575 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));
3576 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);
3577 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);
3578 	VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED);
3579 
3580 	mlxcx_cmd_init(mlxp, &cmd);
3581 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_sq_head,
3582 	    MLXCX_OP_DESTROY_SQ, 0);
3583 
3584 	in.mlxi_destroy_sq_sqn = to_be24(mlwq->mlwq_num);
3585 
3586 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
3587 		mlxcx_cmd_fini(mlxp, &cmd);
3588 		return (B_FALSE);
3589 	}
3590 	mlxcx_cmd_wait(&cmd);
3591 
3592 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
3593 	if (ret) {
3594 		mlwq->mlwq_state |= MLXCX_WQ_DESTROYED;
3595 	}
3596 	mlxcx_cmd_fini(mlxp, &cmd);
3597 	return (ret);
3598 }
3599 
3600 boolean_t
mlxcx_cmd_create_rqt(mlxcx_t * mlxp,mlxcx_rqtable_t * mlrqt)3601 mlxcx_cmd_create_rqt(mlxcx_t *mlxp, mlxcx_rqtable_t *mlrqt)
3602 {
3603 	mlxcx_cmd_t cmd;
3604 	mlxcx_cmd_create_rqt_in_t in;
3605 	mlxcx_cmd_create_rqt_out_t out;
3606 	mlxcx_rqtable_ctx_t *ctx;
3607 	boolean_t ret;
3608 	uint_t i;
3609 
3610 	bzero(&in, sizeof (in));
3611 	bzero(&out, sizeof (out));
3612 
3613 	VERIFY0(mlrqt->mlrqt_state & MLXCX_RQT_CREATED);
3614 
3615 	mlxcx_cmd_init(mlxp, &cmd);
3616 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_rqt_head,
3617 	    MLXCX_OP_CREATE_RQT, 0);
3618 
3619 	ctx = &in.mlxi_create_rqt_context;
3620 	ASSERT3U(mlrqt->mlrqt_max, <=, MLXCX_RQT_MAX_RQ_REFS);
3621 	ASSERT3U(mlrqt->mlrqt_max, <=, mlxp->mlx_caps->mlc_max_rqt_size);
3622 	ctx->mlrqtc_max_size = to_be16(mlrqt->mlrqt_max);
3623 	ctx->mlrqtc_actual_size = to_be16(mlrqt->mlrqt_used);
3624 	for (i = 0; i < mlrqt->mlrqt_used; ++i) {
3625 		ctx->mlrqtc_rqref[i].mlrqtr_rqn = to_be24(
3626 		    mlrqt->mlrqt_rq[i]->mlwq_num);
3627 	}
3628 
3629 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
3630 		mlxcx_cmd_fini(mlxp, &cmd);
3631 		return (B_FALSE);
3632 	}
3633 	mlxcx_cmd_wait(&cmd);
3634 
3635 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
3636 	if (ret) {
3637 		mlrqt->mlrqt_num = from_be24(out.mlxo_create_rqt_rqtn);
3638 		mlrqt->mlrqt_state |= MLXCX_RQT_CREATED;
3639 		mlrqt->mlrqt_state &= ~MLXCX_RQT_DIRTY;
3640 	}
3641 	mlxcx_cmd_fini(mlxp, &cmd);
3642 	return (ret);
3643 }
3644 
3645 boolean_t
mlxcx_cmd_destroy_rqt(mlxcx_t * mlxp,mlxcx_rqtable_t * mlrqt)3646 mlxcx_cmd_destroy_rqt(mlxcx_t *mlxp, mlxcx_rqtable_t *mlrqt)
3647 {
3648 	mlxcx_cmd_t cmd;
3649 	mlxcx_cmd_destroy_rqt_in_t in;
3650 	mlxcx_cmd_destroy_rqt_out_t out;
3651 	boolean_t ret;
3652 
3653 	bzero(&in, sizeof (in));
3654 	bzero(&out, sizeof (out));
3655 
3656 	VERIFY(mlrqt->mlrqt_state & MLXCX_RQT_CREATED);
3657 	VERIFY0(mlrqt->mlrqt_state & MLXCX_RQT_DESTROYED);
3658 
3659 	mlxcx_cmd_init(mlxp, &cmd);
3660 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_rqt_head,
3661 	    MLXCX_OP_DESTROY_RQT, 0);
3662 
3663 	in.mlxi_destroy_rqt_rqtn = to_be24(mlrqt->mlrqt_num);
3664 
3665 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
3666 		mlxcx_cmd_fini(mlxp, &cmd);
3667 		return (B_FALSE);
3668 	}
3669 	mlxcx_cmd_wait(&cmd);
3670 
3671 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
3672 	if (ret) {
3673 		mlrqt->mlrqt_state |= MLXCX_RQT_DESTROYED;
3674 	}
3675 	mlxcx_cmd_fini(mlxp, &cmd);
3676 	return (ret);
3677 }
3678 
3679 boolean_t
mlxcx_cmd_set_int_mod(mlxcx_t * mlxp,uint_t intr,uint_t min_delay)3680 mlxcx_cmd_set_int_mod(mlxcx_t *mlxp, uint_t intr, uint_t min_delay)
3681 {
3682 	mlxcx_cmd_t cmd;
3683 	mlxcx_cmd_config_int_mod_in_t in;
3684 	mlxcx_cmd_config_int_mod_out_t out;
3685 	boolean_t ret;
3686 
3687 	bzero(&in, sizeof (in));
3688 	bzero(&out, sizeof (out));
3689 
3690 	mlxcx_cmd_init(mlxp, &cmd);
3691 	mlxcx_cmd_in_header_init(&cmd, &in.mlxi_config_int_mod_head,
3692 	    MLXCX_OP_CONFIG_INT_MODERATION, MLXCX_CMD_CONFIG_INT_MOD_WRITE);
3693 
3694 	in.mlxi_config_int_mod_int_vector = to_be16(intr);
3695 	in.mlxi_config_int_mod_min_delay = to_be16(min_delay);
3696 
3697 	if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) {
3698 		mlxcx_cmd_fini(mlxp, &cmd);
3699 		return (B_FALSE);
3700 	}
3701 	mlxcx_cmd_wait(&cmd);
3702 
3703 	ret = mlxcx_cmd_evaluate(mlxp, &cmd);
3704 	mlxcx_cmd_fini(mlxp, &cmd);
3705 	return (ret);
3706 }
3707 
3708 /*
3709  * CTASSERTs here are for the structs in mlxcx_reg.h, to check they match
3710  * against offsets from the PRM.
3711  *
3712  * They're not in the header file, to avoid them being used by multiple .c
3713  * files.
3714  */
3715 
3716 CTASSERT(offsetof(mlxcx_eventq_ent_t, mleqe_unknown_data) == 0x20);
3717 CTASSERT(offsetof(mlxcx_eventq_ent_t, mleqe_signature) == 0x3c + 2);
3718 CTASSERT(sizeof (mlxcx_eventq_ent_t) == 64);
3719 
3720 CTASSERT(offsetof(mlxcx_completionq_error_ent_t, mlcqee_byte_cnt) == 0x2C);
3721 CTASSERT(offsetof(mlxcx_completionq_error_ent_t, mlcqee_wqe_opcode) == 0x38);
3722 
3723 CTASSERT(sizeof (mlxcx_completionq_error_ent_t) ==
3724     sizeof (mlxcx_completionq_ent_t));
3725 CTASSERT(sizeof (mlxcx_wqe_control_seg_t) == (1 << 4));
3726 
3727 CTASSERT(offsetof(mlxcx_wqe_eth_seg_t, mles_inline_headers) == 0x0e);
3728 CTASSERT(sizeof (mlxcx_wqe_eth_seg_t) == (1 << 5));
3729 
3730 CTASSERT(sizeof (mlxcx_wqe_data_seg_t) == (1 << 4));
3731 
3732 CTASSERT(sizeof (mlxcx_sendq_ent_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT));
3733 
3734 CTASSERT(sizeof (mlxcx_sendq_bf_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT));
3735 
3736 CTASSERT(sizeof (mlxcx_sendq_extra_ent_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT));
3737 
3738 CTASSERT(sizeof (mlxcx_recvq_ent_t) == (1 << MLXCX_RECVQ_STRIDE_SHIFT));
3739 
3740 CTASSERT(offsetof(mlxcx_workq_ctx_t, mlwqc_dbr_addr) == 0x10);
3741 CTASSERT(offsetof(mlxcx_workq_ctx_t, mlwqc_pas) == 0xc0);
3742 
3743 CTASSERT(offsetof(mlxcx_rq_ctx_t, mlrqc_cqn) == 0x09);
3744 CTASSERT(offsetof(mlxcx_rq_ctx_t, mlrqc_wq) == 0x30);
3745 
3746 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_cqn) == 0x09);
3747 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_tis_lst_sz) == 0x20);
3748 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_tis_num) == 0x2d);
3749 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_wq) == 0x30);
3750 
3751 CTASSERT(sizeof (mlxcx_tis_ctx_t) == 0xa0);
3752 CTASSERT(offsetof(mlxcx_tis_ctx_t, mltisc_transport_domain) == 0x25);
3753 
3754 CTASSERT(offsetof(mlxcx_rqtable_ctx_t, mlrqtc_max_size) == 0x16);
3755 CTASSERT(offsetof(mlxcx_rqtable_ctx_t, mlrqtc_rqref) == 0xF0);
3756 
3757 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_event_bitmask) ==
3758     0x58);
3759 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_pas) == 0x110);
3760 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_context) == 0x10);
3761 
3762 CTASSERT(offsetof(mlxcx_cmd_create_tir_in_t, mlxi_create_tir_context) == 0x20);
3763 
3764 CTASSERT(offsetof(mlxcx_cmd_create_tis_in_t, mlxi_create_tis_context) == 0x20);
3765 
3766 CTASSERT(offsetof(mlxcx_cmd_query_special_ctxs_out_t,
3767     mlxo_query_special_ctxs_resd_lkey) == 0x0c);
3768 
3769 CTASSERT(offsetof(mlxcx_cmd_query_cq_out_t, mlxo_query_cq_context) == 0x10);
3770 CTASSERT(offsetof(mlxcx_cmd_query_cq_out_t, mlxo_query_cq_pas) == 0x110);
3771 
3772 CTASSERT(offsetof(mlxcx_cmd_query_rq_out_t, mlxo_query_rq_context) == 0x20);
3773 
3774 CTASSERT(offsetof(mlxcx_cmd_create_sq_in_t, mlxi_create_sq_context) == 0x20);
3775 
3776 CTASSERT(offsetof(mlxcx_cmd_modify_sq_in_t, mlxi_modify_sq_context) == 0x20);
3777 
3778 CTASSERT(offsetof(mlxcx_cmd_query_sq_out_t, mlxo_query_sq_context) == 0x20);
3779 
3780 CTASSERT(offsetof(mlxcx_cmd_create_rqt_in_t, mlxi_create_rqt_context) == 0x20);
3781 
3782 CTASSERT(offsetof(mlxcx_reg_pmtu_t, mlrd_pmtu_oper_mtu) == 0x0C);
3783 
3784 CTASSERT(sizeof (mlxcx_reg_ptys_t) == 64);
3785 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_ext_proto_cap) == 0x08);
3786 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_cap) == 0x0c);
3787 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_ext_proto_admin) == 0x14);
3788 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_admin) == 0x18);
3789 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_partner_advert) == 0x30);
3790 
3791 CTASSERT(offsetof(mlxcx_reg_mcia_t, mlrd_mcia_data) == 0x10);
3792 
3793 CTASSERT(offsetof(mlxcx_ppcnt_ieee_802_3_t,
3794     mlppc_ieee_802_3_in_range_len_err) == 0x50);
3795 CTASSERT(offsetof(mlxcx_ppcnt_ieee_802_3_t,
3796     mlppc_ieee_802_3_pause_tx) == 0x90);
3797 
3798 CTASSERT(sizeof (mlxcx_reg_ppcnt_t) == 256);
3799 CTASSERT(offsetof(mlxcx_reg_ppcnt_t, mlrd_ppcnt_data) == 0x08);
3800 
3801 CTASSERT(offsetof(mlxcx_cmd_access_register_in_t,
3802     mlxi_access_register_argument) == 0x0C);
3803 CTASSERT(offsetof(mlxcx_cmd_access_register_in_t,
3804     mlxi_access_register_data) == 0x10);
3805 
3806 CTASSERT(offsetof(mlxcx_cmd_access_register_out_t,
3807     mlxo_access_register_data) == 0x10);
3808 
3809 CTASSERT(sizeof (mlxcx_cmd_set_flow_table_root_in_t) == 64);
3810 CTASSERT(offsetof(mlxcx_cmd_set_flow_table_root_in_t,
3811     mlxi_set_flow_table_root_table_id) == 0x15);
3812 CTASSERT(offsetof(mlxcx_cmd_set_flow_table_root_in_t,
3813     mlxi_set_flow_table_root_esw_owner_vhca_id_valid) == 0x1C);
3814