1 /* 2 * Huawei HiNIC PCI Express Linux driver 3 * Copyright(c) 2017 Huawei Technologies Co., Ltd 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * for more details. 13 * 14 */ 15 16 #include <linux/kernel.h> 17 #include <linux/types.h> 18 #include <linux/errno.h> 19 #include <linux/pci.h> 20 #include <linux/device.h> 21 #include <linux/slab.h> 22 #include <linux/vmalloc.h> 23 #include <linux/spinlock.h> 24 #include <linux/sizes.h> 25 #include <linux/atomic.h> 26 #include <linux/log2.h> 27 #include <linux/io.h> 28 #include <linux/completion.h> 29 #include <linux/err.h> 30 #include <asm/byteorder.h> 31 #include <asm/barrier.h> 32 33 #include "hinic_common.h" 34 #include "hinic_hw_if.h" 35 #include "hinic_hw_eqs.h" 36 #include "hinic_hw_mgmt.h" 37 #include "hinic_hw_wqe.h" 38 #include "hinic_hw_wq.h" 39 #include "hinic_hw_cmdq.h" 40 #include "hinic_hw_io.h" 41 #include "hinic_hw_dev.h" 42 43 #define CMDQ_CEQE_TYPE_SHIFT 0 44 45 #define CMDQ_CEQE_TYPE_MASK 0x7 46 47 #define CMDQ_CEQE_GET(val, member) \ 48 (((val) >> CMDQ_CEQE_##member##_SHIFT) \ 49 & CMDQ_CEQE_##member##_MASK) 50 51 #define CMDQ_WQE_ERRCODE_VAL_SHIFT 20 52 53 #define CMDQ_WQE_ERRCODE_VAL_MASK 0xF 54 55 #define CMDQ_WQE_ERRCODE_GET(val, member) \ 56 (((val) >> CMDQ_WQE_ERRCODE_##member##_SHIFT) \ 57 & CMDQ_WQE_ERRCODE_##member##_MASK) 58 59 #define CMDQ_DB_PI_OFF(pi) (((u16)LOWER_8_BITS(pi)) << 3) 60 61 #define CMDQ_DB_ADDR(db_base, pi) ((db_base) + CMDQ_DB_PI_OFF(pi)) 62 63 #define CMDQ_WQE_HEADER(wqe) ((struct hinic_cmdq_header *)(wqe)) 64 65 #define CMDQ_WQE_COMPLETED(ctrl_info) \ 66 HINIC_CMDQ_CTRL_GET(ctrl_info, HW_BUSY_BIT) 67 68 #define FIRST_DATA_TO_WRITE_LAST sizeof(u64) 69 70 #define CMDQ_DB_OFF SZ_2K 71 72 #define CMDQ_WQEBB_SIZE 64 73 #define CMDQ_WQE_SIZE 64 74 #define CMDQ_DEPTH SZ_4K 75 76 #define CMDQ_WQ_PAGE_SIZE SZ_4K 77 78 #define WQE_LCMD_SIZE 64 79 #define WQE_SCMD_SIZE 64 80 81 #define COMPLETE_LEN 3 82 83 #define CMDQ_TIMEOUT 1000 84 85 #define CMDQ_PFN(addr, page_size) ((addr) >> (ilog2(page_size))) 86 87 #define cmdq_to_cmdqs(cmdq) container_of((cmdq) - (cmdq)->cmdq_type, \ 88 struct hinic_cmdqs, cmdq[0]) 89 90 #define cmdqs_to_func_to_io(cmdqs) container_of(cmdqs, \ 91 struct hinic_func_to_io, \ 92 cmdqs) 93 94 enum cmdq_wqe_type { 95 WQE_LCMD_TYPE = 0, 96 WQE_SCMD_TYPE = 1, 97 }; 98 99 enum completion_format { 100 COMPLETE_DIRECT = 0, 101 COMPLETE_SGE = 1, 102 }; 103 104 enum data_format { 105 DATA_SGE = 0, 106 DATA_DIRECT = 1, 107 }; 108 109 enum bufdesc_len { 110 BUFDESC_LCMD_LEN = 2, /* 16 bytes - 2(8 byte unit) */ 111 BUFDESC_SCMD_LEN = 3, /* 24 bytes - 3(8 byte unit) */ 112 }; 113 114 enum ctrl_sect_len { 115 CTRL_SECT_LEN = 1, /* 4 bytes (ctrl) - 1(8 byte unit) */ 116 CTRL_DIRECT_SECT_LEN = 2, /* 12 bytes (ctrl + rsvd) - 2(8 byte unit) */ 117 }; 118 119 enum cmdq_scmd_type { 120 CMDQ_SET_ARM_CMD = 2, 121 }; 122 123 enum cmdq_cmd_type { 124 CMDQ_CMD_SYNC_DIRECT_RESP = 0, 125 CMDQ_CMD_SYNC_SGE_RESP = 1, 126 }; 127 128 enum completion_request { 129 NO_CEQ = 0, 130 CEQ_SET = 1, 131 }; 132 133 /** 134 * hinic_alloc_cmdq_buf - alloc buffer for sending command 135 * @cmdqs: the cmdqs 136 * @cmdq_buf: the buffer returned in this struct 137 * 138 * Return 0 - Success, negative - Failure 139 **/ 140 int hinic_alloc_cmdq_buf(struct hinic_cmdqs *cmdqs, 141 struct hinic_cmdq_buf *cmdq_buf) 142 { 143 struct hinic_hwif *hwif = cmdqs->hwif; 144 struct pci_dev *pdev = hwif->pdev; 145 146 cmdq_buf->buf = dma_pool_alloc(cmdqs->cmdq_buf_pool, GFP_KERNEL, 147 &cmdq_buf->dma_addr); 148 if (!cmdq_buf->buf) { 149 dev_err(&pdev->dev, "Failed to allocate cmd from the pool\n"); 150 return -ENOMEM; 151 } 152 153 return 0; 154 } 155 156 /** 157 * hinic_free_cmdq_buf - free buffer 158 * @cmdqs: the cmdqs 159 * @cmdq_buf: the buffer to free that is in this struct 160 **/ 161 void hinic_free_cmdq_buf(struct hinic_cmdqs *cmdqs, 162 struct hinic_cmdq_buf *cmdq_buf) 163 { 164 dma_pool_free(cmdqs->cmdq_buf_pool, cmdq_buf->buf, cmdq_buf->dma_addr); 165 } 166 167 static unsigned int cmdq_wqe_size_from_bdlen(enum bufdesc_len len) 168 { 169 unsigned int wqe_size = 0; 170 171 switch (len) { 172 case BUFDESC_LCMD_LEN: 173 wqe_size = WQE_LCMD_SIZE; 174 break; 175 case BUFDESC_SCMD_LEN: 176 wqe_size = WQE_SCMD_SIZE; 177 break; 178 } 179 180 return wqe_size; 181 } 182 183 static void cmdq_set_sge_completion(struct hinic_cmdq_completion *completion, 184 struct hinic_cmdq_buf *buf_out) 185 { 186 struct hinic_sge_resp *sge_resp = &completion->sge_resp; 187 188 hinic_set_sge(&sge_resp->sge, buf_out->dma_addr, buf_out->size); 189 } 190 191 static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped, 192 enum hinic_cmd_ack_type ack_type, 193 enum hinic_mod_type mod, u8 cmd, u16 prod_idx, 194 enum completion_format complete_format, 195 enum data_format data_format, 196 enum bufdesc_len buf_len) 197 { 198 struct hinic_cmdq_wqe_lcmd *wqe_lcmd; 199 struct hinic_cmdq_wqe_scmd *wqe_scmd; 200 enum ctrl_sect_len ctrl_len; 201 struct hinic_ctrl *ctrl; 202 u32 saved_data; 203 204 if (data_format == DATA_SGE) { 205 wqe_lcmd = &wqe->wqe_lcmd; 206 207 wqe_lcmd->status.status_info = 0; 208 ctrl = &wqe_lcmd->ctrl; 209 ctrl_len = CTRL_SECT_LEN; 210 } else { 211 wqe_scmd = &wqe->direct_wqe.wqe_scmd; 212 213 wqe_scmd->status.status_info = 0; 214 ctrl = &wqe_scmd->ctrl; 215 ctrl_len = CTRL_DIRECT_SECT_LEN; 216 } 217 218 ctrl->ctrl_info = HINIC_CMDQ_CTRL_SET(prod_idx, PI) | 219 HINIC_CMDQ_CTRL_SET(cmd, CMD) | 220 HINIC_CMDQ_CTRL_SET(mod, MOD) | 221 HINIC_CMDQ_CTRL_SET(ack_type, ACK_TYPE); 222 223 CMDQ_WQE_HEADER(wqe)->header_info = 224 HINIC_CMDQ_WQE_HEADER_SET(buf_len, BUFDESC_LEN) | 225 HINIC_CMDQ_WQE_HEADER_SET(complete_format, COMPLETE_FMT) | 226 HINIC_CMDQ_WQE_HEADER_SET(data_format, DATA_FMT) | 227 HINIC_CMDQ_WQE_HEADER_SET(CEQ_SET, COMPLETE_REQ) | 228 HINIC_CMDQ_WQE_HEADER_SET(COMPLETE_LEN, COMPLETE_SECT_LEN) | 229 HINIC_CMDQ_WQE_HEADER_SET(ctrl_len, CTRL_LEN) | 230 HINIC_CMDQ_WQE_HEADER_SET(wrapped, TOGGLED_WRAPPED); 231 232 saved_data = CMDQ_WQE_HEADER(wqe)->saved_data; 233 saved_data = HINIC_SAVED_DATA_CLEAR(saved_data, ARM); 234 235 if ((cmd == CMDQ_SET_ARM_CMD) && (mod == HINIC_MOD_COMM)) 236 CMDQ_WQE_HEADER(wqe)->saved_data |= 237 HINIC_SAVED_DATA_SET(1, ARM); 238 else 239 CMDQ_WQE_HEADER(wqe)->saved_data = saved_data; 240 } 241 242 static void cmdq_set_lcmd_bufdesc(struct hinic_cmdq_wqe_lcmd *wqe_lcmd, 243 struct hinic_cmdq_buf *buf_in) 244 { 245 hinic_set_sge(&wqe_lcmd->buf_desc.sge, buf_in->dma_addr, buf_in->size); 246 } 247 248 static void cmdq_set_direct_wqe_data(struct hinic_cmdq_direct_wqe *wqe, 249 void *buf_in, u32 in_size) 250 { 251 struct hinic_cmdq_wqe_scmd *wqe_scmd = &wqe->wqe_scmd; 252 253 wqe_scmd->buf_desc.buf_len = in_size; 254 memcpy(wqe_scmd->buf_desc.data, buf_in, in_size); 255 } 256 257 static void cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe *wqe, 258 enum cmdq_cmd_type cmd_type, 259 struct hinic_cmdq_buf *buf_in, 260 struct hinic_cmdq_buf *buf_out, int wrapped, 261 enum hinic_cmd_ack_type ack_type, 262 enum hinic_mod_type mod, u8 cmd, u16 prod_idx) 263 { 264 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd; 265 enum completion_format complete_format; 266 267 switch (cmd_type) { 268 case CMDQ_CMD_SYNC_SGE_RESP: 269 complete_format = COMPLETE_SGE; 270 cmdq_set_sge_completion(&wqe_lcmd->completion, buf_out); 271 break; 272 case CMDQ_CMD_SYNC_DIRECT_RESP: 273 complete_format = COMPLETE_DIRECT; 274 wqe_lcmd->completion.direct_resp = 0; 275 break; 276 } 277 278 cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, 279 prod_idx, complete_format, DATA_SGE, 280 BUFDESC_LCMD_LEN); 281 282 cmdq_set_lcmd_bufdesc(wqe_lcmd, buf_in); 283 } 284 285 static void cmdq_set_direct_wqe(struct hinic_cmdq_wqe *wqe, 286 enum cmdq_cmd_type cmd_type, 287 void *buf_in, u16 in_size, 288 struct hinic_cmdq_buf *buf_out, int wrapped, 289 enum hinic_cmd_ack_type ack_type, 290 enum hinic_mod_type mod, u8 cmd, u16 prod_idx) 291 { 292 struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; 293 enum completion_format complete_format; 294 struct hinic_cmdq_wqe_scmd *wqe_scmd; 295 296 wqe_scmd = &direct_wqe->wqe_scmd; 297 298 switch (cmd_type) { 299 case CMDQ_CMD_SYNC_SGE_RESP: 300 complete_format = COMPLETE_SGE; 301 cmdq_set_sge_completion(&wqe_scmd->completion, buf_out); 302 break; 303 case CMDQ_CMD_SYNC_DIRECT_RESP: 304 complete_format = COMPLETE_DIRECT; 305 wqe_scmd->completion.direct_resp = 0; 306 break; 307 } 308 309 cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, prod_idx, 310 complete_format, DATA_DIRECT, BUFDESC_SCMD_LEN); 311 312 cmdq_set_direct_wqe_data(direct_wqe, buf_in, in_size); 313 } 314 315 static void cmdq_wqe_fill(void *dst, void *src) 316 { 317 memcpy(dst + FIRST_DATA_TO_WRITE_LAST, src + FIRST_DATA_TO_WRITE_LAST, 318 CMDQ_WQE_SIZE - FIRST_DATA_TO_WRITE_LAST); 319 320 wmb(); /* The first 8 bytes should be written last */ 321 322 *(u64 *)dst = *(u64 *)src; 323 } 324 325 static void cmdq_fill_db(u32 *db_info, 326 enum hinic_cmdq_type cmdq_type, u16 prod_idx) 327 { 328 *db_info = HINIC_CMDQ_DB_INFO_SET(UPPER_8_BITS(prod_idx), HI_PROD_IDX) | 329 HINIC_CMDQ_DB_INFO_SET(HINIC_CTRL_PATH, PATH) | 330 HINIC_CMDQ_DB_INFO_SET(cmdq_type, CMDQ_TYPE) | 331 HINIC_CMDQ_DB_INFO_SET(HINIC_DB_CMDQ_TYPE, DB_TYPE); 332 } 333 334 static void cmdq_set_db(struct hinic_cmdq *cmdq, 335 enum hinic_cmdq_type cmdq_type, u16 prod_idx) 336 { 337 u32 db_info; 338 339 cmdq_fill_db(&db_info, cmdq_type, prod_idx); 340 341 /* The data that is written to HW should be in Big Endian Format */ 342 db_info = cpu_to_be32(db_info); 343 344 wmb(); /* write all before the doorbell */ 345 346 writel(db_info, CMDQ_DB_ADDR(cmdq->db_base, prod_idx)); 347 } 348 349 static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq, 350 enum hinic_mod_type mod, u8 cmd, 351 struct hinic_cmdq_buf *buf_in, 352 u64 *resp) 353 { 354 struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe; 355 u16 curr_prod_idx, next_prod_idx; 356 int errcode, wrapped, num_wqebbs; 357 struct hinic_wq *wq = cmdq->wq; 358 struct hinic_hw_wqe *hw_wqe; 359 struct completion done; 360 361 /* Keep doorbell index correct. bh - for tasklet(ceq). */ 362 spin_lock_bh(&cmdq->cmdq_lock); 363 364 /* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/ 365 hw_wqe = hinic_get_wqe(wq, WQE_LCMD_SIZE, &curr_prod_idx); 366 if (IS_ERR(hw_wqe)) { 367 spin_unlock_bh(&cmdq->cmdq_lock); 368 return -EBUSY; 369 } 370 371 curr_cmdq_wqe = &hw_wqe->cmdq_wqe; 372 373 wrapped = cmdq->wrapped; 374 375 num_wqebbs = ALIGN(WQE_LCMD_SIZE, wq->wqebb_size) / wq->wqebb_size; 376 next_prod_idx = curr_prod_idx + num_wqebbs; 377 if (next_prod_idx >= wq->q_depth) { 378 cmdq->wrapped = !cmdq->wrapped; 379 next_prod_idx -= wq->q_depth; 380 } 381 382 cmdq->errcode[curr_prod_idx] = &errcode; 383 384 init_completion(&done); 385 cmdq->done[curr_prod_idx] = &done; 386 387 cmdq_set_lcmd_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in, NULL, 388 wrapped, HINIC_CMD_ACK_TYPE_CMDQ, mod, cmd, 389 curr_prod_idx); 390 391 /* The data that is written to HW should be in Big Endian Format */ 392 hinic_cpu_to_be32(&cmdq_wqe, WQE_LCMD_SIZE); 393 394 /* CMDQ WQE is not shadow, therefore wqe will be written to wq */ 395 cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe); 396 397 cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx); 398 399 spin_unlock_bh(&cmdq->cmdq_lock); 400 401 if (!wait_for_completion_timeout(&done, CMDQ_TIMEOUT)) { 402 spin_lock_bh(&cmdq->cmdq_lock); 403 404 if (cmdq->errcode[curr_prod_idx] == &errcode) 405 cmdq->errcode[curr_prod_idx] = NULL; 406 407 if (cmdq->done[curr_prod_idx] == &done) 408 cmdq->done[curr_prod_idx] = NULL; 409 410 spin_unlock_bh(&cmdq->cmdq_lock); 411 412 return -ETIMEDOUT; 413 } 414 415 smp_rmb(); /* read error code after completion */ 416 417 if (resp) { 418 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &curr_cmdq_wqe->wqe_lcmd; 419 420 *resp = cpu_to_be64(wqe_lcmd->completion.direct_resp); 421 } 422 423 if (errcode != 0) 424 return -EFAULT; 425 426 return 0; 427 } 428 429 static int cmdq_set_arm_bit(struct hinic_cmdq *cmdq, void *buf_in, 430 u16 in_size) 431 { 432 struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe; 433 u16 curr_prod_idx, next_prod_idx; 434 struct hinic_wq *wq = cmdq->wq; 435 struct hinic_hw_wqe *hw_wqe; 436 int wrapped, num_wqebbs; 437 438 /* Keep doorbell index correct */ 439 spin_lock(&cmdq->cmdq_lock); 440 441 /* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/ 442 hw_wqe = hinic_get_wqe(wq, WQE_SCMD_SIZE, &curr_prod_idx); 443 if (IS_ERR(hw_wqe)) { 444 spin_unlock(&cmdq->cmdq_lock); 445 return -EBUSY; 446 } 447 448 curr_cmdq_wqe = &hw_wqe->cmdq_wqe; 449 450 wrapped = cmdq->wrapped; 451 452 num_wqebbs = ALIGN(WQE_SCMD_SIZE, wq->wqebb_size) / wq->wqebb_size; 453 next_prod_idx = curr_prod_idx + num_wqebbs; 454 if (next_prod_idx >= wq->q_depth) { 455 cmdq->wrapped = !cmdq->wrapped; 456 next_prod_idx -= wq->q_depth; 457 } 458 459 cmdq_set_direct_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in, 460 in_size, NULL, wrapped, HINIC_CMD_ACK_TYPE_CMDQ, 461 HINIC_MOD_COMM, CMDQ_SET_ARM_CMD, curr_prod_idx); 462 463 /* The data that is written to HW should be in Big Endian Format */ 464 hinic_cpu_to_be32(&cmdq_wqe, WQE_SCMD_SIZE); 465 466 /* cmdq wqe is not shadow, therefore wqe will be written to wq */ 467 cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe); 468 469 cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx); 470 471 spin_unlock(&cmdq->cmdq_lock); 472 return 0; 473 } 474 475 static int cmdq_params_valid(struct hinic_cmdq_buf *buf_in) 476 { 477 if (buf_in->size > HINIC_CMDQ_MAX_DATA_SIZE) 478 return -EINVAL; 479 480 return 0; 481 } 482 483 /** 484 * hinic_cmdq_direct_resp - send command with direct data as resp 485 * @cmdqs: the cmdqs 486 * @mod: module on the card that will handle the command 487 * @cmd: the command 488 * @buf_in: the buffer for the command 489 * @resp: the response to return 490 * 491 * Return 0 - Success, negative - Failure 492 **/ 493 int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs, 494 enum hinic_mod_type mod, u8 cmd, 495 struct hinic_cmdq_buf *buf_in, u64 *resp) 496 { 497 struct hinic_hwif *hwif = cmdqs->hwif; 498 struct pci_dev *pdev = hwif->pdev; 499 int err; 500 501 err = cmdq_params_valid(buf_in); 502 if (err) { 503 dev_err(&pdev->dev, "Invalid CMDQ parameters\n"); 504 return err; 505 } 506 507 return cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC_CMDQ_SYNC], 508 mod, cmd, buf_in, resp); 509 } 510 511 /** 512 * hinic_set_arm_bit - set arm bit for enable interrupt again 513 * @cmdqs: the cmdqs 514 * @q_type: type of queue to set the arm bit for 515 * @q_id: the queue number 516 * 517 * Return 0 - Success, negative - Failure 518 **/ 519 int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs, 520 enum hinic_set_arm_qtype q_type, u32 q_id) 521 { 522 struct hinic_cmdq *cmdq = &cmdqs->cmdq[HINIC_CMDQ_SYNC]; 523 struct hinic_hwif *hwif = cmdqs->hwif; 524 struct pci_dev *pdev = hwif->pdev; 525 struct hinic_cmdq_arm_bit arm_bit; 526 int err; 527 528 arm_bit.q_type = q_type; 529 arm_bit.q_id = q_id; 530 531 err = cmdq_set_arm_bit(cmdq, &arm_bit, sizeof(arm_bit)); 532 if (err) { 533 dev_err(&pdev->dev, "Failed to set arm for qid %d\n", q_id); 534 return err; 535 } 536 537 return 0; 538 } 539 540 static void clear_wqe_complete_bit(struct hinic_cmdq *cmdq, 541 struct hinic_cmdq_wqe *wqe) 542 { 543 u32 header_info = be32_to_cpu(CMDQ_WQE_HEADER(wqe)->header_info); 544 unsigned int bufdesc_len, wqe_size; 545 struct hinic_ctrl *ctrl; 546 547 bufdesc_len = HINIC_CMDQ_WQE_HEADER_GET(header_info, BUFDESC_LEN); 548 wqe_size = cmdq_wqe_size_from_bdlen(bufdesc_len); 549 if (wqe_size == WQE_LCMD_SIZE) { 550 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd; 551 552 ctrl = &wqe_lcmd->ctrl; 553 } else { 554 struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; 555 struct hinic_cmdq_wqe_scmd *wqe_scmd; 556 557 wqe_scmd = &direct_wqe->wqe_scmd; 558 ctrl = &wqe_scmd->ctrl; 559 } 560 561 /* clear HW busy bit */ 562 ctrl->ctrl_info = 0; 563 564 wmb(); /* verify wqe is clear */ 565 } 566 567 /** 568 * cmdq_arm_ceq_handler - cmdq completion event handler for arm command 569 * @cmdq: the cmdq of the arm command 570 * @wqe: the wqe of the arm command 571 * 572 * Return 0 - Success, negative - Failure 573 **/ 574 static int cmdq_arm_ceq_handler(struct hinic_cmdq *cmdq, 575 struct hinic_cmdq_wqe *wqe) 576 { 577 struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; 578 struct hinic_cmdq_wqe_scmd *wqe_scmd; 579 struct hinic_ctrl *ctrl; 580 u32 ctrl_info; 581 582 wqe_scmd = &direct_wqe->wqe_scmd; 583 ctrl = &wqe_scmd->ctrl; 584 ctrl_info = be32_to_cpu(ctrl->ctrl_info); 585 586 /* HW should toggle the HW BUSY BIT */ 587 if (!CMDQ_WQE_COMPLETED(ctrl_info)) 588 return -EBUSY; 589 590 clear_wqe_complete_bit(cmdq, wqe); 591 592 hinic_put_wqe(cmdq->wq, WQE_SCMD_SIZE); 593 return 0; 594 } 595 596 static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx, 597 int errcode) 598 { 599 if (cmdq->errcode[prod_idx]) 600 *cmdq->errcode[prod_idx] = errcode; 601 } 602 603 /** 604 * cmdq_arm_ceq_handler - cmdq completion event handler for sync command 605 * @cmdq: the cmdq of the command 606 * @cons_idx: the consumer index to update the error code for 607 * @errcode: the error code 608 **/ 609 static void cmdq_sync_cmd_handler(struct hinic_cmdq *cmdq, u16 cons_idx, 610 int errcode) 611 { 612 u16 prod_idx = cons_idx; 613 614 spin_lock(&cmdq->cmdq_lock); 615 cmdq_update_errcode(cmdq, prod_idx, errcode); 616 617 wmb(); /* write all before update for the command request */ 618 619 if (cmdq->done[prod_idx]) 620 complete(cmdq->done[prod_idx]); 621 spin_unlock(&cmdq->cmdq_lock); 622 } 623 624 static int cmdq_cmd_ceq_handler(struct hinic_cmdq *cmdq, u16 ci, 625 struct hinic_cmdq_wqe *cmdq_wqe) 626 { 627 struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &cmdq_wqe->wqe_lcmd; 628 struct hinic_status *status = &wqe_lcmd->status; 629 struct hinic_ctrl *ctrl = &wqe_lcmd->ctrl; 630 int errcode; 631 632 if (!CMDQ_WQE_COMPLETED(be32_to_cpu(ctrl->ctrl_info))) 633 return -EBUSY; 634 635 errcode = CMDQ_WQE_ERRCODE_GET(be32_to_cpu(status->status_info), VAL); 636 637 cmdq_sync_cmd_handler(cmdq, ci, errcode); 638 639 clear_wqe_complete_bit(cmdq, cmdq_wqe); 640 hinic_put_wqe(cmdq->wq, WQE_LCMD_SIZE); 641 return 0; 642 } 643 644 /** 645 * cmdq_ceq_handler - cmdq completion event handler 646 * @handle: private data for the handler(cmdqs) 647 * @ceqe_data: ceq element data 648 **/ 649 static void cmdq_ceq_handler(void *handle, u32 ceqe_data) 650 { 651 enum hinic_cmdq_type cmdq_type = CMDQ_CEQE_GET(ceqe_data, TYPE); 652 struct hinic_cmdqs *cmdqs = (struct hinic_cmdqs *)handle; 653 struct hinic_cmdq *cmdq = &cmdqs->cmdq[cmdq_type]; 654 struct hinic_cmdq_header *header; 655 struct hinic_hw_wqe *hw_wqe; 656 int err, set_arm = 0; 657 u32 saved_data; 658 u16 ci; 659 660 /* Read the smallest wqe size for getting wqe size */ 661 while ((hw_wqe = hinic_read_wqe(cmdq->wq, WQE_SCMD_SIZE, &ci))) { 662 if (IS_ERR(hw_wqe)) 663 break; 664 665 header = CMDQ_WQE_HEADER(&hw_wqe->cmdq_wqe); 666 saved_data = be32_to_cpu(header->saved_data); 667 668 if (HINIC_SAVED_DATA_GET(saved_data, ARM)) { 669 /* arm_bit was set until here */ 670 set_arm = 0; 671 672 if (cmdq_arm_ceq_handler(cmdq, &hw_wqe->cmdq_wqe)) 673 break; 674 } else { 675 set_arm = 1; 676 677 hw_wqe = hinic_read_wqe(cmdq->wq, WQE_LCMD_SIZE, &ci); 678 if (IS_ERR(hw_wqe)) 679 break; 680 681 if (cmdq_cmd_ceq_handler(cmdq, ci, &hw_wqe->cmdq_wqe)) 682 break; 683 } 684 } 685 686 if (set_arm) { 687 struct hinic_hwif *hwif = cmdqs->hwif; 688 struct pci_dev *pdev = hwif->pdev; 689 690 err = hinic_set_arm_bit(cmdqs, HINIC_SET_ARM_CMDQ, cmdq_type); 691 if (err) 692 dev_err(&pdev->dev, "Failed to set arm for CMDQ\n"); 693 } 694 } 695 696 /** 697 * cmdq_init_queue_ctxt - init the queue ctxt of a cmdq 698 * @cmdq_ctxt: cmdq ctxt to initialize 699 * @cmdq: the cmdq 700 * @cmdq_pages: the memory of the queue 701 **/ 702 static void cmdq_init_queue_ctxt(struct hinic_cmdq_ctxt *cmdq_ctxt, 703 struct hinic_cmdq *cmdq, 704 struct hinic_cmdq_pages *cmdq_pages) 705 { 706 struct hinic_cmdq_ctxt_info *ctxt_info = &cmdq_ctxt->ctxt_info; 707 u64 wq_first_page_paddr, cmdq_first_block_paddr, pfn; 708 struct hinic_cmdqs *cmdqs = cmdq_to_cmdqs(cmdq); 709 struct hinic_wq *wq = cmdq->wq; 710 711 /* The data in the HW is in Big Endian Format */ 712 wq_first_page_paddr = be64_to_cpu(*wq->block_vaddr); 713 714 pfn = CMDQ_PFN(wq_first_page_paddr, wq->wq_page_size); 715 716 ctxt_info->curr_wqe_page_pfn = 717 HINIC_CMDQ_CTXT_PAGE_INFO_SET(pfn, CURR_WQE_PAGE_PFN) | 718 HINIC_CMDQ_CTXT_PAGE_INFO_SET(HINIC_CEQ_ID_CMDQ, EQ_ID) | 719 HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_ARM) | 720 HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_EN) | 721 HINIC_CMDQ_CTXT_PAGE_INFO_SET(cmdq->wrapped, WRAPPED); 722 723 /* block PFN - Read Modify Write */ 724 cmdq_first_block_paddr = cmdq_pages->page_paddr; 725 726 pfn = CMDQ_PFN(cmdq_first_block_paddr, wq->wq_page_size); 727 728 ctxt_info->wq_block_pfn = 729 HINIC_CMDQ_CTXT_BLOCK_INFO_SET(pfn, WQ_BLOCK_PFN) | 730 HINIC_CMDQ_CTXT_BLOCK_INFO_SET(atomic_read(&wq->cons_idx), CI); 731 732 cmdq_ctxt->func_idx = HINIC_HWIF_FUNC_IDX(cmdqs->hwif); 733 cmdq_ctxt->cmdq_type = cmdq->cmdq_type; 734 } 735 736 /** 737 * init_cmdq - initialize cmdq 738 * @cmdq: the cmdq 739 * @wq: the wq attaced to the cmdq 740 * @q_type: the cmdq type of the cmdq 741 * @db_area: doorbell area for the cmdq 742 * 743 * Return 0 - Success, negative - Failure 744 **/ 745 static int init_cmdq(struct hinic_cmdq *cmdq, struct hinic_wq *wq, 746 enum hinic_cmdq_type q_type, void __iomem *db_area) 747 { 748 int err; 749 750 cmdq->wq = wq; 751 cmdq->cmdq_type = q_type; 752 cmdq->wrapped = 1; 753 754 spin_lock_init(&cmdq->cmdq_lock); 755 756 cmdq->done = vzalloc(array_size(sizeof(*cmdq->done), wq->q_depth)); 757 if (!cmdq->done) 758 return -ENOMEM; 759 760 cmdq->errcode = vzalloc(array_size(sizeof(*cmdq->errcode), 761 wq->q_depth)); 762 if (!cmdq->errcode) { 763 err = -ENOMEM; 764 goto err_errcode; 765 } 766 767 cmdq->db_base = db_area + CMDQ_DB_OFF; 768 return 0; 769 770 err_errcode: 771 vfree(cmdq->done); 772 return err; 773 } 774 775 /** 776 * free_cmdq - Free cmdq 777 * @cmdq: the cmdq to free 778 **/ 779 static void free_cmdq(struct hinic_cmdq *cmdq) 780 { 781 vfree(cmdq->errcode); 782 vfree(cmdq->done); 783 } 784 785 /** 786 * init_cmdqs_ctxt - write the cmdq ctxt to HW after init all cmdq 787 * @hwdev: the NIC HW device 788 * @cmdqs: cmdqs to write the ctxts for 789 * &db_area: db_area for all the cmdqs 790 * 791 * Return 0 - Success, negative - Failure 792 **/ 793 static int init_cmdqs_ctxt(struct hinic_hwdev *hwdev, 794 struct hinic_cmdqs *cmdqs, void __iomem **db_area) 795 { 796 struct hinic_hwif *hwif = hwdev->hwif; 797 enum hinic_cmdq_type type, cmdq_type; 798 struct hinic_cmdq_ctxt *cmdq_ctxts; 799 struct pci_dev *pdev = hwif->pdev; 800 struct hinic_pfhwdev *pfhwdev; 801 size_t cmdq_ctxts_size; 802 int err; 803 804 if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) { 805 dev_err(&pdev->dev, "Unsupported PCI function type\n"); 806 return -EINVAL; 807 } 808 809 cmdq_ctxts_size = HINIC_MAX_CMDQ_TYPES * sizeof(*cmdq_ctxts); 810 cmdq_ctxts = devm_kzalloc(&pdev->dev, cmdq_ctxts_size, GFP_KERNEL); 811 if (!cmdq_ctxts) 812 return -ENOMEM; 813 814 pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); 815 816 cmdq_type = HINIC_CMDQ_SYNC; 817 for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) { 818 err = init_cmdq(&cmdqs->cmdq[cmdq_type], 819 &cmdqs->saved_wqs[cmdq_type], cmdq_type, 820 db_area[cmdq_type]); 821 if (err) { 822 dev_err(&pdev->dev, "Failed to initialize cmdq\n"); 823 goto err_init_cmdq; 824 } 825 826 cmdq_init_queue_ctxt(&cmdq_ctxts[cmdq_type], 827 &cmdqs->cmdq[cmdq_type], 828 &cmdqs->cmdq_pages); 829 } 830 831 /* Write the CMDQ ctxts */ 832 cmdq_type = HINIC_CMDQ_SYNC; 833 for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) { 834 err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, 835 HINIC_COMM_CMD_CMDQ_CTXT_SET, 836 &cmdq_ctxts[cmdq_type], 837 sizeof(cmdq_ctxts[cmdq_type]), 838 NULL, NULL, HINIC_MGMT_MSG_SYNC); 839 if (err) { 840 dev_err(&pdev->dev, "Failed to set CMDQ CTXT type = %d\n", 841 cmdq_type); 842 goto err_write_cmdq_ctxt; 843 } 844 } 845 846 devm_kfree(&pdev->dev, cmdq_ctxts); 847 return 0; 848 849 err_write_cmdq_ctxt: 850 cmdq_type = HINIC_MAX_CMDQ_TYPES; 851 852 err_init_cmdq: 853 for (type = HINIC_CMDQ_SYNC; type < cmdq_type; type++) 854 free_cmdq(&cmdqs->cmdq[type]); 855 856 devm_kfree(&pdev->dev, cmdq_ctxts); 857 return err; 858 } 859 860 /** 861 * hinic_init_cmdqs - init all cmdqs 862 * @cmdqs: cmdqs to init 863 * @hwif: HW interface for accessing cmdqs 864 * @db_area: doorbell areas for all the cmdqs 865 * 866 * Return 0 - Success, negative - Failure 867 **/ 868 int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif, 869 void __iomem **db_area) 870 { 871 struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs); 872 struct pci_dev *pdev = hwif->pdev; 873 struct hinic_hwdev *hwdev; 874 size_t saved_wqs_size; 875 u16 max_wqe_size; 876 int err; 877 878 cmdqs->hwif = hwif; 879 cmdqs->cmdq_buf_pool = dma_pool_create("hinic_cmdq", &pdev->dev, 880 HINIC_CMDQ_BUF_SIZE, 881 HINIC_CMDQ_BUF_SIZE, 0); 882 if (!cmdqs->cmdq_buf_pool) 883 return -ENOMEM; 884 885 saved_wqs_size = HINIC_MAX_CMDQ_TYPES * sizeof(struct hinic_wq); 886 cmdqs->saved_wqs = devm_kzalloc(&pdev->dev, saved_wqs_size, GFP_KERNEL); 887 if (!cmdqs->saved_wqs) { 888 err = -ENOMEM; 889 goto err_saved_wqs; 890 } 891 892 max_wqe_size = WQE_LCMD_SIZE; 893 err = hinic_wqs_cmdq_alloc(&cmdqs->cmdq_pages, cmdqs->saved_wqs, hwif, 894 HINIC_MAX_CMDQ_TYPES, CMDQ_WQEBB_SIZE, 895 CMDQ_WQ_PAGE_SIZE, CMDQ_DEPTH, max_wqe_size); 896 if (err) { 897 dev_err(&pdev->dev, "Failed to allocate CMDQ wqs\n"); 898 goto err_cmdq_wqs; 899 } 900 901 hwdev = container_of(func_to_io, struct hinic_hwdev, func_to_io); 902 err = init_cmdqs_ctxt(hwdev, cmdqs, db_area); 903 if (err) { 904 dev_err(&pdev->dev, "Failed to write cmdq ctxt\n"); 905 goto err_cmdq_ctxt; 906 } 907 908 hinic_ceq_register_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ, cmdqs, 909 cmdq_ceq_handler); 910 return 0; 911 912 err_cmdq_ctxt: 913 hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs, 914 HINIC_MAX_CMDQ_TYPES); 915 916 err_cmdq_wqs: 917 devm_kfree(&pdev->dev, cmdqs->saved_wqs); 918 919 err_saved_wqs: 920 dma_pool_destroy(cmdqs->cmdq_buf_pool); 921 return err; 922 } 923 924 /** 925 * hinic_free_cmdqs - free all cmdqs 926 * @cmdqs: cmdqs to free 927 **/ 928 void hinic_free_cmdqs(struct hinic_cmdqs *cmdqs) 929 { 930 struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs); 931 struct hinic_hwif *hwif = cmdqs->hwif; 932 struct pci_dev *pdev = hwif->pdev; 933 enum hinic_cmdq_type cmdq_type; 934 935 hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ); 936 937 cmdq_type = HINIC_CMDQ_SYNC; 938 for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) 939 free_cmdq(&cmdqs->cmdq[cmdq_type]); 940 941 hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs, 942 HINIC_MAX_CMDQ_TYPES); 943 944 devm_kfree(&pdev->dev, cmdqs->saved_wqs); 945 946 dma_pool_destroy(cmdqs->cmdq_buf_pool); 947 } 948