1 /* 2 * CXL Flash Device Driver 3 * 4 * Written by: Manoj N. Kumar <manoj@linux.vnet.ibm.com>, IBM Corporation 5 * Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation 6 * 7 * Copyright (C) 2015 IBM Corporation 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; either version 12 * 2 of the License, or (at your option) any later version. 13 */ 14 15 #include <linux/delay.h> 16 #include <linux/list.h> 17 #include <linux/module.h> 18 #include <linux/pci.h> 19 20 #include <asm/unaligned.h> 21 22 #include <misc/cxl.h> 23 24 #include <scsi/scsi_cmnd.h> 25 #include <scsi/scsi_host.h> 26 #include <uapi/scsi/cxlflash_ioctl.h> 27 28 #include "main.h" 29 #include "sislite.h" 30 #include "common.h" 31 32 MODULE_DESCRIPTION(CXLFLASH_ADAPTER_NAME); 33 MODULE_AUTHOR("Manoj N. Kumar <manoj@linux.vnet.ibm.com>"); 34 MODULE_AUTHOR("Matthew R. Ochs <mrochs@linux.vnet.ibm.com>"); 35 MODULE_LICENSE("GPL"); 36 37 static struct class *cxlflash_class; 38 static u32 cxlflash_major; 39 static DECLARE_BITMAP(cxlflash_minor, CXLFLASH_MAX_ADAPTERS); 40 41 /** 42 * process_cmd_err() - command error handler 43 * @cmd: AFU command that experienced the error. 44 * @scp: SCSI command associated with the AFU command in error. 45 * 46 * Translates error bits from AFU command to SCSI command results. 47 */ 48 static void process_cmd_err(struct afu_cmd *cmd, struct scsi_cmnd *scp) 49 { 50 struct afu *afu = cmd->parent; 51 struct cxlflash_cfg *cfg = afu->parent; 52 struct device *dev = &cfg->dev->dev; 53 struct sisl_ioarcb *ioarcb; 54 struct sisl_ioasa *ioasa; 55 u32 resid; 56 57 if (unlikely(!cmd)) 58 return; 59 60 ioarcb = &(cmd->rcb); 61 ioasa = &(cmd->sa); 62 63 if (ioasa->rc.flags & SISL_RC_FLAGS_UNDERRUN) { 64 resid = ioasa->resid; 65 scsi_set_resid(scp, resid); 66 dev_dbg(dev, "%s: cmd underrun cmd = %p scp = %p, resid = %d\n", 67 __func__, cmd, scp, resid); 68 } 69 70 if (ioasa->rc.flags & SISL_RC_FLAGS_OVERRUN) { 71 dev_dbg(dev, "%s: cmd underrun cmd = %p scp = %p\n", 72 __func__, cmd, scp); 73 scp->result = (DID_ERROR << 16); 74 } 75 76 dev_dbg(dev, "%s: cmd failed afu_rc=%02x scsi_rc=%02x fc_rc=%02x " 77 "afu_extra=%02x scsi_extra=%02x fc_extra=%02x\n", __func__, 78 ioasa->rc.afu_rc, ioasa->rc.scsi_rc, ioasa->rc.fc_rc, 79 ioasa->afu_extra, ioasa->scsi_extra, ioasa->fc_extra); 80 81 if (ioasa->rc.scsi_rc) { 82 /* We have a SCSI status */ 83 if (ioasa->rc.flags & SISL_RC_FLAGS_SENSE_VALID) { 84 memcpy(scp->sense_buffer, ioasa->sense_data, 85 SISL_SENSE_DATA_LEN); 86 scp->result = ioasa->rc.scsi_rc; 87 } else 88 scp->result = ioasa->rc.scsi_rc | (DID_ERROR << 16); 89 } 90 91 /* 92 * We encountered an error. Set scp->result based on nature 93 * of error. 94 */ 95 if (ioasa->rc.fc_rc) { 96 /* We have an FC status */ 97 switch (ioasa->rc.fc_rc) { 98 case SISL_FC_RC_LINKDOWN: 99 scp->result = (DID_REQUEUE << 16); 100 break; 101 case SISL_FC_RC_RESID: 102 /* This indicates an FCP resid underrun */ 103 if (!(ioasa->rc.flags & SISL_RC_FLAGS_OVERRUN)) { 104 /* If the SISL_RC_FLAGS_OVERRUN flag was set, 105 * then we will handle this error else where. 106 * If not then we must handle it here. 107 * This is probably an AFU bug. 108 */ 109 scp->result = (DID_ERROR << 16); 110 } 111 break; 112 case SISL_FC_RC_RESIDERR: 113 /* Resid mismatch between adapter and device */ 114 case SISL_FC_RC_TGTABORT: 115 case SISL_FC_RC_ABORTOK: 116 case SISL_FC_RC_ABORTFAIL: 117 case SISL_FC_RC_NOLOGI: 118 case SISL_FC_RC_ABORTPEND: 119 case SISL_FC_RC_WRABORTPEND: 120 case SISL_FC_RC_NOEXP: 121 case SISL_FC_RC_INUSE: 122 scp->result = (DID_ERROR << 16); 123 break; 124 } 125 } 126 127 if (ioasa->rc.afu_rc) { 128 /* We have an AFU error */ 129 switch (ioasa->rc.afu_rc) { 130 case SISL_AFU_RC_NO_CHANNELS: 131 scp->result = (DID_NO_CONNECT << 16); 132 break; 133 case SISL_AFU_RC_DATA_DMA_ERR: 134 switch (ioasa->afu_extra) { 135 case SISL_AFU_DMA_ERR_PAGE_IN: 136 /* Retry */ 137 scp->result = (DID_IMM_RETRY << 16); 138 break; 139 case SISL_AFU_DMA_ERR_INVALID_EA: 140 default: 141 scp->result = (DID_ERROR << 16); 142 } 143 break; 144 case SISL_AFU_RC_OUT_OF_DATA_BUFS: 145 /* Retry */ 146 scp->result = (DID_ALLOC_FAILURE << 16); 147 break; 148 default: 149 scp->result = (DID_ERROR << 16); 150 } 151 } 152 } 153 154 /** 155 * cmd_complete() - command completion handler 156 * @cmd: AFU command that has completed. 157 * 158 * For SCSI commands this routine prepares and submits commands that have 159 * either completed or timed out to the SCSI stack. For internal commands 160 * (TMF or AFU), this routine simply notifies the originator that the 161 * command has completed. 162 */ 163 static void cmd_complete(struct afu_cmd *cmd) 164 { 165 struct scsi_cmnd *scp; 166 ulong lock_flags; 167 struct afu *afu = cmd->parent; 168 struct cxlflash_cfg *cfg = afu->parent; 169 struct device *dev = &cfg->dev->dev; 170 struct hwq *hwq = get_hwq(afu, cmd->hwq_index); 171 172 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); 173 list_del(&cmd->list); 174 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); 175 176 if (cmd->scp) { 177 scp = cmd->scp; 178 if (unlikely(cmd->sa.ioasc)) 179 process_cmd_err(cmd, scp); 180 else 181 scp->result = (DID_OK << 16); 182 183 dev_dbg_ratelimited(dev, "%s:scp=%p result=%08x ioasc=%08x\n", 184 __func__, scp, scp->result, cmd->sa.ioasc); 185 scp->scsi_done(scp); 186 } else if (cmd->cmd_tmf) { 187 spin_lock_irqsave(&cfg->tmf_slock, lock_flags); 188 cfg->tmf_active = false; 189 wake_up_all_locked(&cfg->tmf_waitq); 190 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags); 191 } else 192 complete(&cmd->cevent); 193 } 194 195 /** 196 * flush_pending_cmds() - flush all pending commands on this hardware queue 197 * @hwq: Hardware queue to flush. 198 * 199 * The hardware send queue lock associated with this hardware queue must be 200 * held when calling this routine. 201 */ 202 static void flush_pending_cmds(struct hwq *hwq) 203 { 204 struct cxlflash_cfg *cfg = hwq->afu->parent; 205 struct afu_cmd *cmd, *tmp; 206 struct scsi_cmnd *scp; 207 ulong lock_flags; 208 209 list_for_each_entry_safe(cmd, tmp, &hwq->pending_cmds, list) { 210 /* Bypass command when on a doneq, cmd_complete() will handle */ 211 if (!list_empty(&cmd->queue)) 212 continue; 213 214 list_del(&cmd->list); 215 216 if (cmd->scp) { 217 scp = cmd->scp; 218 scp->result = (DID_IMM_RETRY << 16); 219 scp->scsi_done(scp); 220 } else { 221 cmd->cmd_aborted = true; 222 223 if (cmd->cmd_tmf) { 224 spin_lock_irqsave(&cfg->tmf_slock, lock_flags); 225 cfg->tmf_active = false; 226 wake_up_all_locked(&cfg->tmf_waitq); 227 spin_unlock_irqrestore(&cfg->tmf_slock, 228 lock_flags); 229 } else 230 complete(&cmd->cevent); 231 } 232 } 233 } 234 235 /** 236 * context_reset() - reset context via specified register 237 * @hwq: Hardware queue owning the context to be reset. 238 * @reset_reg: MMIO register to perform reset. 239 * 240 * When the reset is successful, the SISLite specification guarantees that 241 * the AFU has aborted all currently pending I/O. Accordingly, these commands 242 * must be flushed. 243 * 244 * Return: 0 on success, -errno on failure 245 */ 246 static int context_reset(struct hwq *hwq, __be64 __iomem *reset_reg) 247 { 248 struct cxlflash_cfg *cfg = hwq->afu->parent; 249 struct device *dev = &cfg->dev->dev; 250 int rc = -ETIMEDOUT; 251 int nretry = 0; 252 u64 val = 0x1; 253 ulong lock_flags; 254 255 dev_dbg(dev, "%s: hwq=%p\n", __func__, hwq); 256 257 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); 258 259 writeq_be(val, reset_reg); 260 do { 261 val = readq_be(reset_reg); 262 if ((val & 0x1) == 0x0) { 263 rc = 0; 264 break; 265 } 266 267 /* Double delay each time */ 268 udelay(1 << nretry); 269 } while (nretry++ < MC_ROOM_RETRY_CNT); 270 271 if (!rc) 272 flush_pending_cmds(hwq); 273 274 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); 275 276 dev_dbg(dev, "%s: returning rc=%d, val=%016llx nretry=%d\n", 277 __func__, rc, val, nretry); 278 return rc; 279 } 280 281 /** 282 * context_reset_ioarrin() - reset context via IOARRIN register 283 * @hwq: Hardware queue owning the context to be reset. 284 * 285 * Return: 0 on success, -errno on failure 286 */ 287 static int context_reset_ioarrin(struct hwq *hwq) 288 { 289 return context_reset(hwq, &hwq->host_map->ioarrin); 290 } 291 292 /** 293 * context_reset_sq() - reset context via SQ_CONTEXT_RESET register 294 * @hwq: Hardware queue owning the context to be reset. 295 * 296 * Return: 0 on success, -errno on failure 297 */ 298 static int context_reset_sq(struct hwq *hwq) 299 { 300 return context_reset(hwq, &hwq->host_map->sq_ctx_reset); 301 } 302 303 /** 304 * send_cmd_ioarrin() - sends an AFU command via IOARRIN register 305 * @afu: AFU associated with the host. 306 * @cmd: AFU command to send. 307 * 308 * Return: 309 * 0 on success, SCSI_MLQUEUE_HOST_BUSY on failure 310 */ 311 static int send_cmd_ioarrin(struct afu *afu, struct afu_cmd *cmd) 312 { 313 struct cxlflash_cfg *cfg = afu->parent; 314 struct device *dev = &cfg->dev->dev; 315 struct hwq *hwq = get_hwq(afu, cmd->hwq_index); 316 int rc = 0; 317 s64 room; 318 ulong lock_flags; 319 320 /* 321 * To avoid the performance penalty of MMIO, spread the update of 322 * 'room' over multiple commands. 323 */ 324 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); 325 if (--hwq->room < 0) { 326 room = readq_be(&hwq->host_map->cmd_room); 327 if (room <= 0) { 328 dev_dbg_ratelimited(dev, "%s: no cmd_room to send " 329 "0x%02X, room=0x%016llX\n", 330 __func__, cmd->rcb.cdb[0], room); 331 hwq->room = 0; 332 rc = SCSI_MLQUEUE_HOST_BUSY; 333 goto out; 334 } 335 hwq->room = room - 1; 336 } 337 338 list_add(&cmd->list, &hwq->pending_cmds); 339 writeq_be((u64)&cmd->rcb, &hwq->host_map->ioarrin); 340 out: 341 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); 342 dev_dbg(dev, "%s: cmd=%p len=%u ea=%016llx rc=%d\n", __func__, 343 cmd, cmd->rcb.data_len, cmd->rcb.data_ea, rc); 344 return rc; 345 } 346 347 /** 348 * send_cmd_sq() - sends an AFU command via SQ ring 349 * @afu: AFU associated with the host. 350 * @cmd: AFU command to send. 351 * 352 * Return: 353 * 0 on success, SCSI_MLQUEUE_HOST_BUSY on failure 354 */ 355 static int send_cmd_sq(struct afu *afu, struct afu_cmd *cmd) 356 { 357 struct cxlflash_cfg *cfg = afu->parent; 358 struct device *dev = &cfg->dev->dev; 359 struct hwq *hwq = get_hwq(afu, cmd->hwq_index); 360 int rc = 0; 361 int newval; 362 ulong lock_flags; 363 364 newval = atomic_dec_if_positive(&hwq->hsq_credits); 365 if (newval <= 0) { 366 rc = SCSI_MLQUEUE_HOST_BUSY; 367 goto out; 368 } 369 370 cmd->rcb.ioasa = &cmd->sa; 371 372 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); 373 374 *hwq->hsq_curr = cmd->rcb; 375 if (hwq->hsq_curr < hwq->hsq_end) 376 hwq->hsq_curr++; 377 else 378 hwq->hsq_curr = hwq->hsq_start; 379 380 list_add(&cmd->list, &hwq->pending_cmds); 381 writeq_be((u64)hwq->hsq_curr, &hwq->host_map->sq_tail); 382 383 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); 384 out: 385 dev_dbg(dev, "%s: cmd=%p len=%u ea=%016llx ioasa=%p rc=%d curr=%p " 386 "head=%016llx tail=%016llx\n", __func__, cmd, cmd->rcb.data_len, 387 cmd->rcb.data_ea, cmd->rcb.ioasa, rc, hwq->hsq_curr, 388 readq_be(&hwq->host_map->sq_head), 389 readq_be(&hwq->host_map->sq_tail)); 390 return rc; 391 } 392 393 /** 394 * wait_resp() - polls for a response or timeout to a sent AFU command 395 * @afu: AFU associated with the host. 396 * @cmd: AFU command that was sent. 397 * 398 * Return: 0 on success, -errno on failure 399 */ 400 static int wait_resp(struct afu *afu, struct afu_cmd *cmd) 401 { 402 struct cxlflash_cfg *cfg = afu->parent; 403 struct device *dev = &cfg->dev->dev; 404 int rc = 0; 405 ulong timeout = msecs_to_jiffies(cmd->rcb.timeout * 2 * 1000); 406 407 timeout = wait_for_completion_timeout(&cmd->cevent, timeout); 408 if (!timeout) 409 rc = -ETIMEDOUT; 410 411 if (cmd->cmd_aborted) 412 rc = -EAGAIN; 413 414 if (unlikely(cmd->sa.ioasc != 0)) { 415 dev_err(dev, "%s: cmd %02x failed, ioasc=%08x\n", 416 __func__, cmd->rcb.cdb[0], cmd->sa.ioasc); 417 rc = -EIO; 418 } 419 420 return rc; 421 } 422 423 /** 424 * cmd_to_target_hwq() - selects a target hardware queue for a SCSI command 425 * @host: SCSI host associated with device. 426 * @scp: SCSI command to send. 427 * @afu: SCSI command to send. 428 * 429 * Hashes a command based upon the hardware queue mode. 430 * 431 * Return: Trusted index of target hardware queue 432 */ 433 static u32 cmd_to_target_hwq(struct Scsi_Host *host, struct scsi_cmnd *scp, 434 struct afu *afu) 435 { 436 u32 tag; 437 u32 hwq = 0; 438 439 if (afu->num_hwqs == 1) 440 return 0; 441 442 switch (afu->hwq_mode) { 443 case HWQ_MODE_RR: 444 hwq = afu->hwq_rr_count++ % afu->num_hwqs; 445 break; 446 case HWQ_MODE_TAG: 447 tag = blk_mq_unique_tag(scp->request); 448 hwq = blk_mq_unique_tag_to_hwq(tag); 449 break; 450 case HWQ_MODE_CPU: 451 hwq = smp_processor_id() % afu->num_hwqs; 452 break; 453 default: 454 WARN_ON_ONCE(1); 455 } 456 457 return hwq; 458 } 459 460 /** 461 * send_tmf() - sends a Task Management Function (TMF) 462 * @cfg: Internal structure associated with the host. 463 * @sdev: SCSI device destined for TMF. 464 * @tmfcmd: TMF command to send. 465 * 466 * Return: 467 * 0 on success, SCSI_MLQUEUE_HOST_BUSY or -errno on failure 468 */ 469 static int send_tmf(struct cxlflash_cfg *cfg, struct scsi_device *sdev, 470 u64 tmfcmd) 471 { 472 struct afu *afu = cfg->afu; 473 struct afu_cmd *cmd = NULL; 474 struct device *dev = &cfg->dev->dev; 475 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ); 476 char *buf = NULL; 477 ulong lock_flags; 478 int rc = 0; 479 ulong to; 480 481 buf = kzalloc(sizeof(*cmd) + __alignof__(*cmd) - 1, GFP_KERNEL); 482 if (unlikely(!buf)) { 483 dev_err(dev, "%s: no memory for command\n", __func__); 484 rc = -ENOMEM; 485 goto out; 486 } 487 488 cmd = (struct afu_cmd *)PTR_ALIGN(buf, __alignof__(*cmd)); 489 INIT_LIST_HEAD(&cmd->queue); 490 491 /* When Task Management Function is active do not send another */ 492 spin_lock_irqsave(&cfg->tmf_slock, lock_flags); 493 if (cfg->tmf_active) 494 wait_event_interruptible_lock_irq(cfg->tmf_waitq, 495 !cfg->tmf_active, 496 cfg->tmf_slock); 497 cfg->tmf_active = true; 498 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags); 499 500 cmd->parent = afu; 501 cmd->cmd_tmf = true; 502 cmd->hwq_index = hwq->index; 503 504 cmd->rcb.ctx_id = hwq->ctx_hndl; 505 cmd->rcb.msi = SISL_MSI_RRQ_UPDATED; 506 cmd->rcb.port_sel = CHAN2PORTMASK(sdev->channel); 507 cmd->rcb.lun_id = lun_to_lunid(sdev->lun); 508 cmd->rcb.req_flags = (SISL_REQ_FLAGS_PORT_LUN_ID | 509 SISL_REQ_FLAGS_SUP_UNDERRUN | 510 SISL_REQ_FLAGS_TMF_CMD); 511 memcpy(cmd->rcb.cdb, &tmfcmd, sizeof(tmfcmd)); 512 513 rc = afu->send_cmd(afu, cmd); 514 if (unlikely(rc)) { 515 spin_lock_irqsave(&cfg->tmf_slock, lock_flags); 516 cfg->tmf_active = false; 517 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags); 518 goto out; 519 } 520 521 spin_lock_irqsave(&cfg->tmf_slock, lock_flags); 522 to = msecs_to_jiffies(5000); 523 to = wait_event_interruptible_lock_irq_timeout(cfg->tmf_waitq, 524 !cfg->tmf_active, 525 cfg->tmf_slock, 526 to); 527 if (!to) { 528 dev_err(dev, "%s: TMF timed out\n", __func__); 529 rc = -ETIMEDOUT; 530 } else if (cmd->cmd_aborted) { 531 dev_err(dev, "%s: TMF aborted\n", __func__); 532 rc = -EAGAIN; 533 } else if (cmd->sa.ioasc) { 534 dev_err(dev, "%s: TMF failed ioasc=%08x\n", 535 __func__, cmd->sa.ioasc); 536 rc = -EIO; 537 } 538 cfg->tmf_active = false; 539 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags); 540 out: 541 kfree(buf); 542 return rc; 543 } 544 545 /** 546 * cxlflash_driver_info() - information handler for this host driver 547 * @host: SCSI host associated with device. 548 * 549 * Return: A string describing the device. 550 */ 551 static const char *cxlflash_driver_info(struct Scsi_Host *host) 552 { 553 return CXLFLASH_ADAPTER_NAME; 554 } 555 556 /** 557 * cxlflash_queuecommand() - sends a mid-layer request 558 * @host: SCSI host associated with device. 559 * @scp: SCSI command to send. 560 * 561 * Return: 0 on success, SCSI_MLQUEUE_HOST_BUSY on failure 562 */ 563 static int cxlflash_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scp) 564 { 565 struct cxlflash_cfg *cfg = shost_priv(host); 566 struct afu *afu = cfg->afu; 567 struct device *dev = &cfg->dev->dev; 568 struct afu_cmd *cmd = sc_to_afuci(scp); 569 struct scatterlist *sg = scsi_sglist(scp); 570 int hwq_index = cmd_to_target_hwq(host, scp, afu); 571 struct hwq *hwq = get_hwq(afu, hwq_index); 572 u16 req_flags = SISL_REQ_FLAGS_SUP_UNDERRUN; 573 ulong lock_flags; 574 int rc = 0; 575 576 dev_dbg_ratelimited(dev, "%s: (scp=%p) %d/%d/%d/%llu " 577 "cdb=(%08x-%08x-%08x-%08x)\n", 578 __func__, scp, host->host_no, scp->device->channel, 579 scp->device->id, scp->device->lun, 580 get_unaligned_be32(&((u32 *)scp->cmnd)[0]), 581 get_unaligned_be32(&((u32 *)scp->cmnd)[1]), 582 get_unaligned_be32(&((u32 *)scp->cmnd)[2]), 583 get_unaligned_be32(&((u32 *)scp->cmnd)[3])); 584 585 /* 586 * If a Task Management Function is active, wait for it to complete 587 * before continuing with regular commands. 588 */ 589 spin_lock_irqsave(&cfg->tmf_slock, lock_flags); 590 if (cfg->tmf_active) { 591 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags); 592 rc = SCSI_MLQUEUE_HOST_BUSY; 593 goto out; 594 } 595 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags); 596 597 switch (cfg->state) { 598 case STATE_PROBING: 599 case STATE_PROBED: 600 case STATE_RESET: 601 dev_dbg_ratelimited(dev, "%s: device is in reset\n", __func__); 602 rc = SCSI_MLQUEUE_HOST_BUSY; 603 goto out; 604 case STATE_FAILTERM: 605 dev_dbg_ratelimited(dev, "%s: device has failed\n", __func__); 606 scp->result = (DID_NO_CONNECT << 16); 607 scp->scsi_done(scp); 608 rc = 0; 609 goto out; 610 default: 611 break; 612 } 613 614 if (likely(sg)) { 615 cmd->rcb.data_len = sg->length; 616 cmd->rcb.data_ea = (uintptr_t)sg_virt(sg); 617 } 618 619 cmd->scp = scp; 620 cmd->parent = afu; 621 cmd->hwq_index = hwq_index; 622 623 cmd->rcb.ctx_id = hwq->ctx_hndl; 624 cmd->rcb.msi = SISL_MSI_RRQ_UPDATED; 625 cmd->rcb.port_sel = CHAN2PORTMASK(scp->device->channel); 626 cmd->rcb.lun_id = lun_to_lunid(scp->device->lun); 627 628 if (scp->sc_data_direction == DMA_TO_DEVICE) 629 req_flags |= SISL_REQ_FLAGS_HOST_WRITE; 630 631 cmd->rcb.req_flags = req_flags; 632 memcpy(cmd->rcb.cdb, scp->cmnd, sizeof(cmd->rcb.cdb)); 633 634 rc = afu->send_cmd(afu, cmd); 635 out: 636 return rc; 637 } 638 639 /** 640 * cxlflash_wait_for_pci_err_recovery() - wait for error recovery during probe 641 * @cfg: Internal structure associated with the host. 642 */ 643 static void cxlflash_wait_for_pci_err_recovery(struct cxlflash_cfg *cfg) 644 { 645 struct pci_dev *pdev = cfg->dev; 646 647 if (pci_channel_offline(pdev)) 648 wait_event_timeout(cfg->reset_waitq, 649 !pci_channel_offline(pdev), 650 CXLFLASH_PCI_ERROR_RECOVERY_TIMEOUT); 651 } 652 653 /** 654 * free_mem() - free memory associated with the AFU 655 * @cfg: Internal structure associated with the host. 656 */ 657 static void free_mem(struct cxlflash_cfg *cfg) 658 { 659 struct afu *afu = cfg->afu; 660 661 if (cfg->afu) { 662 free_pages((ulong)afu, get_order(sizeof(struct afu))); 663 cfg->afu = NULL; 664 } 665 } 666 667 /** 668 * cxlflash_reset_sync() - synchronizing point for asynchronous resets 669 * @cfg: Internal structure associated with the host. 670 */ 671 static void cxlflash_reset_sync(struct cxlflash_cfg *cfg) 672 { 673 if (cfg->async_reset_cookie == 0) 674 return; 675 676 /* Wait until all async calls prior to this cookie have completed */ 677 async_synchronize_cookie(cfg->async_reset_cookie + 1); 678 cfg->async_reset_cookie = 0; 679 } 680 681 /** 682 * stop_afu() - stops the AFU command timers and unmaps the MMIO space 683 * @cfg: Internal structure associated with the host. 684 * 685 * Safe to call with AFU in a partially allocated/initialized state. 686 * 687 * Cancels scheduled worker threads, waits for any active internal AFU 688 * commands to timeout, disables IRQ polling and then unmaps the MMIO space. 689 */ 690 static void stop_afu(struct cxlflash_cfg *cfg) 691 { 692 struct afu *afu = cfg->afu; 693 struct hwq *hwq; 694 int i; 695 696 cancel_work_sync(&cfg->work_q); 697 if (!current_is_async()) 698 cxlflash_reset_sync(cfg); 699 700 if (likely(afu)) { 701 while (atomic_read(&afu->cmds_active)) 702 ssleep(1); 703 704 if (afu_is_irqpoll_enabled(afu)) { 705 for (i = 0; i < afu->num_hwqs; i++) { 706 hwq = get_hwq(afu, i); 707 708 irq_poll_disable(&hwq->irqpoll); 709 } 710 } 711 712 if (likely(afu->afu_map)) { 713 cxl_psa_unmap((void __iomem *)afu->afu_map); 714 afu->afu_map = NULL; 715 } 716 } 717 } 718 719 /** 720 * term_intr() - disables all AFU interrupts 721 * @cfg: Internal structure associated with the host. 722 * @level: Depth of allocation, where to begin waterfall tear down. 723 * @index: Index of the hardware queue. 724 * 725 * Safe to call with AFU/MC in partially allocated/initialized state. 726 */ 727 static void term_intr(struct cxlflash_cfg *cfg, enum undo_level level, 728 u32 index) 729 { 730 struct afu *afu = cfg->afu; 731 struct device *dev = &cfg->dev->dev; 732 struct hwq *hwq; 733 734 if (!afu) { 735 dev_err(dev, "%s: returning with NULL afu\n", __func__); 736 return; 737 } 738 739 hwq = get_hwq(afu, index); 740 741 if (!hwq->ctx) { 742 dev_err(dev, "%s: returning with NULL MC\n", __func__); 743 return; 744 } 745 746 switch (level) { 747 case UNMAP_THREE: 748 /* SISL_MSI_ASYNC_ERROR is setup only for the primary HWQ */ 749 if (index == PRIMARY_HWQ) 750 cxl_unmap_afu_irq(hwq->ctx, 3, hwq); 751 case UNMAP_TWO: 752 cxl_unmap_afu_irq(hwq->ctx, 2, hwq); 753 case UNMAP_ONE: 754 cxl_unmap_afu_irq(hwq->ctx, 1, hwq); 755 case FREE_IRQ: 756 cxl_free_afu_irqs(hwq->ctx); 757 /* fall through */ 758 case UNDO_NOOP: 759 /* No action required */ 760 break; 761 } 762 } 763 764 /** 765 * term_mc() - terminates the master context 766 * @cfg: Internal structure associated with the host. 767 * @index: Index of the hardware queue. 768 * 769 * Safe to call with AFU/MC in partially allocated/initialized state. 770 */ 771 static void term_mc(struct cxlflash_cfg *cfg, u32 index) 772 { 773 struct afu *afu = cfg->afu; 774 struct device *dev = &cfg->dev->dev; 775 struct hwq *hwq; 776 ulong lock_flags; 777 778 if (!afu) { 779 dev_err(dev, "%s: returning with NULL afu\n", __func__); 780 return; 781 } 782 783 hwq = get_hwq(afu, index); 784 785 if (!hwq->ctx) { 786 dev_err(dev, "%s: returning with NULL MC\n", __func__); 787 return; 788 } 789 790 WARN_ON(cxl_stop_context(hwq->ctx)); 791 if (index != PRIMARY_HWQ) 792 WARN_ON(cxl_release_context(hwq->ctx)); 793 hwq->ctx = NULL; 794 795 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); 796 flush_pending_cmds(hwq); 797 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); 798 } 799 800 /** 801 * term_afu() - terminates the AFU 802 * @cfg: Internal structure associated with the host. 803 * 804 * Safe to call with AFU/MC in partially allocated/initialized state. 805 */ 806 static void term_afu(struct cxlflash_cfg *cfg) 807 { 808 struct device *dev = &cfg->dev->dev; 809 int k; 810 811 /* 812 * Tear down is carefully orchestrated to ensure 813 * no interrupts can come in when the problem state 814 * area is unmapped. 815 * 816 * 1) Disable all AFU interrupts for each master 817 * 2) Unmap the problem state area 818 * 3) Stop each master context 819 */ 820 for (k = cfg->afu->num_hwqs - 1; k >= 0; k--) 821 term_intr(cfg, UNMAP_THREE, k); 822 823 stop_afu(cfg); 824 825 for (k = cfg->afu->num_hwqs - 1; k >= 0; k--) 826 term_mc(cfg, k); 827 828 dev_dbg(dev, "%s: returning\n", __func__); 829 } 830 831 /** 832 * notify_shutdown() - notifies device of pending shutdown 833 * @cfg: Internal structure associated with the host. 834 * @wait: Whether to wait for shutdown processing to complete. 835 * 836 * This function will notify the AFU that the adapter is being shutdown 837 * and will wait for shutdown processing to complete if wait is true. 838 * This notification should flush pending I/Os to the device and halt 839 * further I/Os until the next AFU reset is issued and device restarted. 840 */ 841 static void notify_shutdown(struct cxlflash_cfg *cfg, bool wait) 842 { 843 struct afu *afu = cfg->afu; 844 struct device *dev = &cfg->dev->dev; 845 struct dev_dependent_vals *ddv; 846 __be64 __iomem *fc_port_regs; 847 u64 reg, status; 848 int i, retry_cnt = 0; 849 850 ddv = (struct dev_dependent_vals *)cfg->dev_id->driver_data; 851 if (!(ddv->flags & CXLFLASH_NOTIFY_SHUTDOWN)) 852 return; 853 854 if (!afu || !afu->afu_map) { 855 dev_dbg(dev, "%s: Problem state area not mapped\n", __func__); 856 return; 857 } 858 859 /* Notify AFU */ 860 for (i = 0; i < cfg->num_fc_ports; i++) { 861 fc_port_regs = get_fc_port_regs(cfg, i); 862 863 reg = readq_be(&fc_port_regs[FC_CONFIG2 / 8]); 864 reg |= SISL_FC_SHUTDOWN_NORMAL; 865 writeq_be(reg, &fc_port_regs[FC_CONFIG2 / 8]); 866 } 867 868 if (!wait) 869 return; 870 871 /* Wait up to 1.5 seconds for shutdown processing to complete */ 872 for (i = 0; i < cfg->num_fc_ports; i++) { 873 fc_port_regs = get_fc_port_regs(cfg, i); 874 retry_cnt = 0; 875 876 while (true) { 877 status = readq_be(&fc_port_regs[FC_STATUS / 8]); 878 if (status & SISL_STATUS_SHUTDOWN_COMPLETE) 879 break; 880 if (++retry_cnt >= MC_RETRY_CNT) { 881 dev_dbg(dev, "%s: port %d shutdown processing " 882 "not yet completed\n", __func__, i); 883 break; 884 } 885 msleep(100 * retry_cnt); 886 } 887 } 888 } 889 890 /** 891 * cxlflash_get_minor() - gets the first available minor number 892 * 893 * Return: Unique minor number that can be used to create the character device. 894 */ 895 static int cxlflash_get_minor(void) 896 { 897 int minor; 898 long bit; 899 900 bit = find_first_zero_bit(cxlflash_minor, CXLFLASH_MAX_ADAPTERS); 901 if (bit >= CXLFLASH_MAX_ADAPTERS) 902 return -1; 903 904 minor = bit & MINORMASK; 905 set_bit(minor, cxlflash_minor); 906 return minor; 907 } 908 909 /** 910 * cxlflash_put_minor() - releases the minor number 911 * @minor: Minor number that is no longer needed. 912 */ 913 static void cxlflash_put_minor(int minor) 914 { 915 clear_bit(minor, cxlflash_minor); 916 } 917 918 /** 919 * cxlflash_release_chrdev() - release the character device for the host 920 * @cfg: Internal structure associated with the host. 921 */ 922 static void cxlflash_release_chrdev(struct cxlflash_cfg *cfg) 923 { 924 device_unregister(cfg->chardev); 925 cfg->chardev = NULL; 926 cdev_del(&cfg->cdev); 927 cxlflash_put_minor(MINOR(cfg->cdev.dev)); 928 } 929 930 /** 931 * cxlflash_remove() - PCI entry point to tear down host 932 * @pdev: PCI device associated with the host. 933 * 934 * Safe to use as a cleanup in partially allocated/initialized state. Note that 935 * the reset_waitq is flushed as part of the stop/termination of user contexts. 936 */ 937 static void cxlflash_remove(struct pci_dev *pdev) 938 { 939 struct cxlflash_cfg *cfg = pci_get_drvdata(pdev); 940 struct device *dev = &pdev->dev; 941 ulong lock_flags; 942 943 if (!pci_is_enabled(pdev)) { 944 dev_dbg(dev, "%s: Device is disabled\n", __func__); 945 return; 946 } 947 948 /* If a Task Management Function is active, wait for it to complete 949 * before continuing with remove. 950 */ 951 spin_lock_irqsave(&cfg->tmf_slock, lock_flags); 952 if (cfg->tmf_active) 953 wait_event_interruptible_lock_irq(cfg->tmf_waitq, 954 !cfg->tmf_active, 955 cfg->tmf_slock); 956 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags); 957 958 /* Notify AFU and wait for shutdown processing to complete */ 959 notify_shutdown(cfg, true); 960 961 cfg->state = STATE_FAILTERM; 962 cxlflash_stop_term_user_contexts(cfg); 963 964 switch (cfg->init_state) { 965 case INIT_STATE_CDEV: 966 cxlflash_release_chrdev(cfg); 967 case INIT_STATE_SCSI: 968 cxlflash_term_local_luns(cfg); 969 scsi_remove_host(cfg->host); 970 case INIT_STATE_AFU: 971 term_afu(cfg); 972 case INIT_STATE_PCI: 973 pci_disable_device(pdev); 974 case INIT_STATE_NONE: 975 free_mem(cfg); 976 scsi_host_put(cfg->host); 977 break; 978 } 979 980 dev_dbg(dev, "%s: returning\n", __func__); 981 } 982 983 /** 984 * alloc_mem() - allocates the AFU and its command pool 985 * @cfg: Internal structure associated with the host. 986 * 987 * A partially allocated state remains on failure. 988 * 989 * Return: 990 * 0 on success 991 * -ENOMEM on failure to allocate memory 992 */ 993 static int alloc_mem(struct cxlflash_cfg *cfg) 994 { 995 int rc = 0; 996 struct device *dev = &cfg->dev->dev; 997 998 /* AFU is ~28k, i.e. only one 64k page or up to seven 4k pages */ 999 cfg->afu = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1000 get_order(sizeof(struct afu))); 1001 if (unlikely(!cfg->afu)) { 1002 dev_err(dev, "%s: cannot get %d free pages\n", 1003 __func__, get_order(sizeof(struct afu))); 1004 rc = -ENOMEM; 1005 goto out; 1006 } 1007 cfg->afu->parent = cfg; 1008 cfg->afu->desired_hwqs = CXLFLASH_DEF_HWQS; 1009 cfg->afu->afu_map = NULL; 1010 out: 1011 return rc; 1012 } 1013 1014 /** 1015 * init_pci() - initializes the host as a PCI device 1016 * @cfg: Internal structure associated with the host. 1017 * 1018 * Return: 0 on success, -errno on failure 1019 */ 1020 static int init_pci(struct cxlflash_cfg *cfg) 1021 { 1022 struct pci_dev *pdev = cfg->dev; 1023 struct device *dev = &cfg->dev->dev; 1024 int rc = 0; 1025 1026 rc = pci_enable_device(pdev); 1027 if (rc || pci_channel_offline(pdev)) { 1028 if (pci_channel_offline(pdev)) { 1029 cxlflash_wait_for_pci_err_recovery(cfg); 1030 rc = pci_enable_device(pdev); 1031 } 1032 1033 if (rc) { 1034 dev_err(dev, "%s: Cannot enable adapter\n", __func__); 1035 cxlflash_wait_for_pci_err_recovery(cfg); 1036 goto out; 1037 } 1038 } 1039 1040 out: 1041 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 1042 return rc; 1043 } 1044 1045 /** 1046 * init_scsi() - adds the host to the SCSI stack and kicks off host scan 1047 * @cfg: Internal structure associated with the host. 1048 * 1049 * Return: 0 on success, -errno on failure 1050 */ 1051 static int init_scsi(struct cxlflash_cfg *cfg) 1052 { 1053 struct pci_dev *pdev = cfg->dev; 1054 struct device *dev = &cfg->dev->dev; 1055 int rc = 0; 1056 1057 rc = scsi_add_host(cfg->host, &pdev->dev); 1058 if (rc) { 1059 dev_err(dev, "%s: scsi_add_host failed rc=%d\n", __func__, rc); 1060 goto out; 1061 } 1062 1063 scsi_scan_host(cfg->host); 1064 1065 out: 1066 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 1067 return rc; 1068 } 1069 1070 /** 1071 * set_port_online() - transitions the specified host FC port to online state 1072 * @fc_regs: Top of MMIO region defined for specified port. 1073 * 1074 * The provided MMIO region must be mapped prior to call. Online state means 1075 * that the FC link layer has synced, completed the handshaking process, and 1076 * is ready for login to start. 1077 */ 1078 static void set_port_online(__be64 __iomem *fc_regs) 1079 { 1080 u64 cmdcfg; 1081 1082 cmdcfg = readq_be(&fc_regs[FC_MTIP_CMDCONFIG / 8]); 1083 cmdcfg &= (~FC_MTIP_CMDCONFIG_OFFLINE); /* clear OFF_LINE */ 1084 cmdcfg |= (FC_MTIP_CMDCONFIG_ONLINE); /* set ON_LINE */ 1085 writeq_be(cmdcfg, &fc_regs[FC_MTIP_CMDCONFIG / 8]); 1086 } 1087 1088 /** 1089 * set_port_offline() - transitions the specified host FC port to offline state 1090 * @fc_regs: Top of MMIO region defined for specified port. 1091 * 1092 * The provided MMIO region must be mapped prior to call. 1093 */ 1094 static void set_port_offline(__be64 __iomem *fc_regs) 1095 { 1096 u64 cmdcfg; 1097 1098 cmdcfg = readq_be(&fc_regs[FC_MTIP_CMDCONFIG / 8]); 1099 cmdcfg &= (~FC_MTIP_CMDCONFIG_ONLINE); /* clear ON_LINE */ 1100 cmdcfg |= (FC_MTIP_CMDCONFIG_OFFLINE); /* set OFF_LINE */ 1101 writeq_be(cmdcfg, &fc_regs[FC_MTIP_CMDCONFIG / 8]); 1102 } 1103 1104 /** 1105 * wait_port_online() - waits for the specified host FC port come online 1106 * @fc_regs: Top of MMIO region defined for specified port. 1107 * @delay_us: Number of microseconds to delay between reading port status. 1108 * @nretry: Number of cycles to retry reading port status. 1109 * 1110 * The provided MMIO region must be mapped prior to call. This will timeout 1111 * when the cable is not plugged in. 1112 * 1113 * Return: 1114 * TRUE (1) when the specified port is online 1115 * FALSE (0) when the specified port fails to come online after timeout 1116 */ 1117 static bool wait_port_online(__be64 __iomem *fc_regs, u32 delay_us, u32 nretry) 1118 { 1119 u64 status; 1120 1121 WARN_ON(delay_us < 1000); 1122 1123 do { 1124 msleep(delay_us / 1000); 1125 status = readq_be(&fc_regs[FC_MTIP_STATUS / 8]); 1126 if (status == U64_MAX) 1127 nretry /= 2; 1128 } while ((status & FC_MTIP_STATUS_MASK) != FC_MTIP_STATUS_ONLINE && 1129 nretry--); 1130 1131 return ((status & FC_MTIP_STATUS_MASK) == FC_MTIP_STATUS_ONLINE); 1132 } 1133 1134 /** 1135 * wait_port_offline() - waits for the specified host FC port go offline 1136 * @fc_regs: Top of MMIO region defined for specified port. 1137 * @delay_us: Number of microseconds to delay between reading port status. 1138 * @nretry: Number of cycles to retry reading port status. 1139 * 1140 * The provided MMIO region must be mapped prior to call. 1141 * 1142 * Return: 1143 * TRUE (1) when the specified port is offline 1144 * FALSE (0) when the specified port fails to go offline after timeout 1145 */ 1146 static bool wait_port_offline(__be64 __iomem *fc_regs, u32 delay_us, u32 nretry) 1147 { 1148 u64 status; 1149 1150 WARN_ON(delay_us < 1000); 1151 1152 do { 1153 msleep(delay_us / 1000); 1154 status = readq_be(&fc_regs[FC_MTIP_STATUS / 8]); 1155 if (status == U64_MAX) 1156 nretry /= 2; 1157 } while ((status & FC_MTIP_STATUS_MASK) != FC_MTIP_STATUS_OFFLINE && 1158 nretry--); 1159 1160 return ((status & FC_MTIP_STATUS_MASK) == FC_MTIP_STATUS_OFFLINE); 1161 } 1162 1163 /** 1164 * afu_set_wwpn() - configures the WWPN for the specified host FC port 1165 * @afu: AFU associated with the host that owns the specified FC port. 1166 * @port: Port number being configured. 1167 * @fc_regs: Top of MMIO region defined for specified port. 1168 * @wwpn: The world-wide-port-number previously discovered for port. 1169 * 1170 * The provided MMIO region must be mapped prior to call. As part of the 1171 * sequence to configure the WWPN, the port is toggled offline and then back 1172 * online. This toggling action can cause this routine to delay up to a few 1173 * seconds. When configured to use the internal LUN feature of the AFU, a 1174 * failure to come online is overridden. 1175 */ 1176 static void afu_set_wwpn(struct afu *afu, int port, __be64 __iomem *fc_regs, 1177 u64 wwpn) 1178 { 1179 struct cxlflash_cfg *cfg = afu->parent; 1180 struct device *dev = &cfg->dev->dev; 1181 1182 set_port_offline(fc_regs); 1183 if (!wait_port_offline(fc_regs, FC_PORT_STATUS_RETRY_INTERVAL_US, 1184 FC_PORT_STATUS_RETRY_CNT)) { 1185 dev_dbg(dev, "%s: wait on port %d to go offline timed out\n", 1186 __func__, port); 1187 } 1188 1189 writeq_be(wwpn, &fc_regs[FC_PNAME / 8]); 1190 1191 set_port_online(fc_regs); 1192 if (!wait_port_online(fc_regs, FC_PORT_STATUS_RETRY_INTERVAL_US, 1193 FC_PORT_STATUS_RETRY_CNT)) { 1194 dev_dbg(dev, "%s: wait on port %d to go online timed out\n", 1195 __func__, port); 1196 } 1197 } 1198 1199 /** 1200 * afu_link_reset() - resets the specified host FC port 1201 * @afu: AFU associated with the host that owns the specified FC port. 1202 * @port: Port number being configured. 1203 * @fc_regs: Top of MMIO region defined for specified port. 1204 * 1205 * The provided MMIO region must be mapped prior to call. The sequence to 1206 * reset the port involves toggling it offline and then back online. This 1207 * action can cause this routine to delay up to a few seconds. An effort 1208 * is made to maintain link with the device by switching to host to use 1209 * the alternate port exclusively while the reset takes place. 1210 * failure to come online is overridden. 1211 */ 1212 static void afu_link_reset(struct afu *afu, int port, __be64 __iomem *fc_regs) 1213 { 1214 struct cxlflash_cfg *cfg = afu->parent; 1215 struct device *dev = &cfg->dev->dev; 1216 u64 port_sel; 1217 1218 /* first switch the AFU to the other links, if any */ 1219 port_sel = readq_be(&afu->afu_map->global.regs.afu_port_sel); 1220 port_sel &= ~(1ULL << port); 1221 writeq_be(port_sel, &afu->afu_map->global.regs.afu_port_sel); 1222 cxlflash_afu_sync(afu, 0, 0, AFU_GSYNC); 1223 1224 set_port_offline(fc_regs); 1225 if (!wait_port_offline(fc_regs, FC_PORT_STATUS_RETRY_INTERVAL_US, 1226 FC_PORT_STATUS_RETRY_CNT)) 1227 dev_err(dev, "%s: wait on port %d to go offline timed out\n", 1228 __func__, port); 1229 1230 set_port_online(fc_regs); 1231 if (!wait_port_online(fc_regs, FC_PORT_STATUS_RETRY_INTERVAL_US, 1232 FC_PORT_STATUS_RETRY_CNT)) 1233 dev_err(dev, "%s: wait on port %d to go online timed out\n", 1234 __func__, port); 1235 1236 /* switch back to include this port */ 1237 port_sel |= (1ULL << port); 1238 writeq_be(port_sel, &afu->afu_map->global.regs.afu_port_sel); 1239 cxlflash_afu_sync(afu, 0, 0, AFU_GSYNC); 1240 1241 dev_dbg(dev, "%s: returning port_sel=%016llx\n", __func__, port_sel); 1242 } 1243 1244 /** 1245 * afu_err_intr_init() - clears and initializes the AFU for error interrupts 1246 * @afu: AFU associated with the host. 1247 */ 1248 static void afu_err_intr_init(struct afu *afu) 1249 { 1250 struct cxlflash_cfg *cfg = afu->parent; 1251 __be64 __iomem *fc_port_regs; 1252 int i; 1253 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ); 1254 u64 reg; 1255 1256 /* global async interrupts: AFU clears afu_ctrl on context exit 1257 * if async interrupts were sent to that context. This prevents 1258 * the AFU form sending further async interrupts when 1259 * there is 1260 * nobody to receive them. 1261 */ 1262 1263 /* mask all */ 1264 writeq_be(-1ULL, &afu->afu_map->global.regs.aintr_mask); 1265 /* set LISN# to send and point to primary master context */ 1266 reg = ((u64) (((hwq->ctx_hndl << 8) | SISL_MSI_ASYNC_ERROR)) << 40); 1267 1268 if (afu->internal_lun) 1269 reg |= 1; /* Bit 63 indicates local lun */ 1270 writeq_be(reg, &afu->afu_map->global.regs.afu_ctrl); 1271 /* clear all */ 1272 writeq_be(-1ULL, &afu->afu_map->global.regs.aintr_clear); 1273 /* unmask bits that are of interest */ 1274 /* note: afu can send an interrupt after this step */ 1275 writeq_be(SISL_ASTATUS_MASK, &afu->afu_map->global.regs.aintr_mask); 1276 /* clear again in case a bit came on after previous clear but before */ 1277 /* unmask */ 1278 writeq_be(-1ULL, &afu->afu_map->global.regs.aintr_clear); 1279 1280 /* Clear/Set internal lun bits */ 1281 fc_port_regs = get_fc_port_regs(cfg, 0); 1282 reg = readq_be(&fc_port_regs[FC_CONFIG2 / 8]); 1283 reg &= SISL_FC_INTERNAL_MASK; 1284 if (afu->internal_lun) 1285 reg |= ((u64)(afu->internal_lun - 1) << SISL_FC_INTERNAL_SHIFT); 1286 writeq_be(reg, &fc_port_regs[FC_CONFIG2 / 8]); 1287 1288 /* now clear FC errors */ 1289 for (i = 0; i < cfg->num_fc_ports; i++) { 1290 fc_port_regs = get_fc_port_regs(cfg, i); 1291 1292 writeq_be(0xFFFFFFFFU, &fc_port_regs[FC_ERROR / 8]); 1293 writeq_be(0, &fc_port_regs[FC_ERRCAP / 8]); 1294 } 1295 1296 /* sync interrupts for master's IOARRIN write */ 1297 /* note that unlike asyncs, there can be no pending sync interrupts */ 1298 /* at this time (this is a fresh context and master has not written */ 1299 /* IOARRIN yet), so there is nothing to clear. */ 1300 1301 /* set LISN#, it is always sent to the context that wrote IOARRIN */ 1302 for (i = 0; i < afu->num_hwqs; i++) { 1303 hwq = get_hwq(afu, i); 1304 1305 writeq_be(SISL_MSI_SYNC_ERROR, &hwq->host_map->ctx_ctrl); 1306 writeq_be(SISL_ISTATUS_MASK, &hwq->host_map->intr_mask); 1307 } 1308 } 1309 1310 /** 1311 * cxlflash_sync_err_irq() - interrupt handler for synchronous errors 1312 * @irq: Interrupt number. 1313 * @data: Private data provided at interrupt registration, the AFU. 1314 * 1315 * Return: Always return IRQ_HANDLED. 1316 */ 1317 static irqreturn_t cxlflash_sync_err_irq(int irq, void *data) 1318 { 1319 struct hwq *hwq = (struct hwq *)data; 1320 struct cxlflash_cfg *cfg = hwq->afu->parent; 1321 struct device *dev = &cfg->dev->dev; 1322 u64 reg; 1323 u64 reg_unmasked; 1324 1325 reg = readq_be(&hwq->host_map->intr_status); 1326 reg_unmasked = (reg & SISL_ISTATUS_UNMASK); 1327 1328 if (reg_unmasked == 0UL) { 1329 dev_err(dev, "%s: spurious interrupt, intr_status=%016llx\n", 1330 __func__, reg); 1331 goto cxlflash_sync_err_irq_exit; 1332 } 1333 1334 dev_err(dev, "%s: unexpected interrupt, intr_status=%016llx\n", 1335 __func__, reg); 1336 1337 writeq_be(reg_unmasked, &hwq->host_map->intr_clear); 1338 1339 cxlflash_sync_err_irq_exit: 1340 return IRQ_HANDLED; 1341 } 1342 1343 /** 1344 * process_hrrq() - process the read-response queue 1345 * @afu: AFU associated with the host. 1346 * @doneq: Queue of commands harvested from the RRQ. 1347 * @budget: Threshold of RRQ entries to process. 1348 * 1349 * This routine must be called holding the disabled RRQ spin lock. 1350 * 1351 * Return: The number of entries processed. 1352 */ 1353 static int process_hrrq(struct hwq *hwq, struct list_head *doneq, int budget) 1354 { 1355 struct afu *afu = hwq->afu; 1356 struct afu_cmd *cmd; 1357 struct sisl_ioasa *ioasa; 1358 struct sisl_ioarcb *ioarcb; 1359 bool toggle = hwq->toggle; 1360 int num_hrrq = 0; 1361 u64 entry, 1362 *hrrq_start = hwq->hrrq_start, 1363 *hrrq_end = hwq->hrrq_end, 1364 *hrrq_curr = hwq->hrrq_curr; 1365 1366 /* Process ready RRQ entries up to the specified budget (if any) */ 1367 while (true) { 1368 entry = *hrrq_curr; 1369 1370 if ((entry & SISL_RESP_HANDLE_T_BIT) != toggle) 1371 break; 1372 1373 entry &= ~SISL_RESP_HANDLE_T_BIT; 1374 1375 if (afu_is_sq_cmd_mode(afu)) { 1376 ioasa = (struct sisl_ioasa *)entry; 1377 cmd = container_of(ioasa, struct afu_cmd, sa); 1378 } else { 1379 ioarcb = (struct sisl_ioarcb *)entry; 1380 cmd = container_of(ioarcb, struct afu_cmd, rcb); 1381 } 1382 1383 list_add_tail(&cmd->queue, doneq); 1384 1385 /* Advance to next entry or wrap and flip the toggle bit */ 1386 if (hrrq_curr < hrrq_end) 1387 hrrq_curr++; 1388 else { 1389 hrrq_curr = hrrq_start; 1390 toggle ^= SISL_RESP_HANDLE_T_BIT; 1391 } 1392 1393 atomic_inc(&hwq->hsq_credits); 1394 num_hrrq++; 1395 1396 if (budget > 0 && num_hrrq >= budget) 1397 break; 1398 } 1399 1400 hwq->hrrq_curr = hrrq_curr; 1401 hwq->toggle = toggle; 1402 1403 return num_hrrq; 1404 } 1405 1406 /** 1407 * process_cmd_doneq() - process a queue of harvested RRQ commands 1408 * @doneq: Queue of completed commands. 1409 * 1410 * Note that upon return the queue can no longer be trusted. 1411 */ 1412 static void process_cmd_doneq(struct list_head *doneq) 1413 { 1414 struct afu_cmd *cmd, *tmp; 1415 1416 WARN_ON(list_empty(doneq)); 1417 1418 list_for_each_entry_safe(cmd, tmp, doneq, queue) 1419 cmd_complete(cmd); 1420 } 1421 1422 /** 1423 * cxlflash_irqpoll() - process a queue of harvested RRQ commands 1424 * @irqpoll: IRQ poll structure associated with queue to poll. 1425 * @budget: Threshold of RRQ entries to process per poll. 1426 * 1427 * Return: The number of entries processed. 1428 */ 1429 static int cxlflash_irqpoll(struct irq_poll *irqpoll, int budget) 1430 { 1431 struct hwq *hwq = container_of(irqpoll, struct hwq, irqpoll); 1432 unsigned long hrrq_flags; 1433 LIST_HEAD(doneq); 1434 int num_entries = 0; 1435 1436 spin_lock_irqsave(&hwq->hrrq_slock, hrrq_flags); 1437 1438 num_entries = process_hrrq(hwq, &doneq, budget); 1439 if (num_entries < budget) 1440 irq_poll_complete(irqpoll); 1441 1442 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags); 1443 1444 process_cmd_doneq(&doneq); 1445 return num_entries; 1446 } 1447 1448 /** 1449 * cxlflash_rrq_irq() - interrupt handler for read-response queue (normal path) 1450 * @irq: Interrupt number. 1451 * @data: Private data provided at interrupt registration, the AFU. 1452 * 1453 * Return: IRQ_HANDLED or IRQ_NONE when no ready entries found. 1454 */ 1455 static irqreturn_t cxlflash_rrq_irq(int irq, void *data) 1456 { 1457 struct hwq *hwq = (struct hwq *)data; 1458 struct afu *afu = hwq->afu; 1459 unsigned long hrrq_flags; 1460 LIST_HEAD(doneq); 1461 int num_entries = 0; 1462 1463 spin_lock_irqsave(&hwq->hrrq_slock, hrrq_flags); 1464 1465 if (afu_is_irqpoll_enabled(afu)) { 1466 irq_poll_sched(&hwq->irqpoll); 1467 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags); 1468 return IRQ_HANDLED; 1469 } 1470 1471 num_entries = process_hrrq(hwq, &doneq, -1); 1472 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags); 1473 1474 if (num_entries == 0) 1475 return IRQ_NONE; 1476 1477 process_cmd_doneq(&doneq); 1478 return IRQ_HANDLED; 1479 } 1480 1481 /* 1482 * Asynchronous interrupt information table 1483 * 1484 * NOTE: 1485 * - Order matters here as this array is indexed by bit position. 1486 * 1487 * - The checkpatch script considers the BUILD_SISL_ASTATUS_FC_PORT macro 1488 * as complex and complains due to a lack of parentheses/braces. 1489 */ 1490 #define ASTATUS_FC(_a, _b, _c, _d) \ 1491 { SISL_ASTATUS_FC##_a##_##_b, _c, _a, (_d) } 1492 1493 #define BUILD_SISL_ASTATUS_FC_PORT(_a) \ 1494 ASTATUS_FC(_a, LINK_UP, "link up", 0), \ 1495 ASTATUS_FC(_a, LINK_DN, "link down", 0), \ 1496 ASTATUS_FC(_a, LOGI_S, "login succeeded", SCAN_HOST), \ 1497 ASTATUS_FC(_a, LOGI_F, "login failed", CLR_FC_ERROR), \ 1498 ASTATUS_FC(_a, LOGI_R, "login timed out, retrying", LINK_RESET), \ 1499 ASTATUS_FC(_a, CRC_T, "CRC threshold exceeded", LINK_RESET), \ 1500 ASTATUS_FC(_a, LOGO, "target initiated LOGO", 0), \ 1501 ASTATUS_FC(_a, OTHER, "other error", CLR_FC_ERROR | LINK_RESET) 1502 1503 static const struct asyc_intr_info ainfo[] = { 1504 BUILD_SISL_ASTATUS_FC_PORT(1), 1505 BUILD_SISL_ASTATUS_FC_PORT(0), 1506 BUILD_SISL_ASTATUS_FC_PORT(3), 1507 BUILD_SISL_ASTATUS_FC_PORT(2) 1508 }; 1509 1510 /** 1511 * cxlflash_async_err_irq() - interrupt handler for asynchronous errors 1512 * @irq: Interrupt number. 1513 * @data: Private data provided at interrupt registration, the AFU. 1514 * 1515 * Return: Always return IRQ_HANDLED. 1516 */ 1517 static irqreturn_t cxlflash_async_err_irq(int irq, void *data) 1518 { 1519 struct hwq *hwq = (struct hwq *)data; 1520 struct afu *afu = hwq->afu; 1521 struct cxlflash_cfg *cfg = afu->parent; 1522 struct device *dev = &cfg->dev->dev; 1523 const struct asyc_intr_info *info; 1524 struct sisl_global_map __iomem *global = &afu->afu_map->global; 1525 __be64 __iomem *fc_port_regs; 1526 u64 reg_unmasked; 1527 u64 reg; 1528 u64 bit; 1529 u8 port; 1530 1531 reg = readq_be(&global->regs.aintr_status); 1532 reg_unmasked = (reg & SISL_ASTATUS_UNMASK); 1533 1534 if (unlikely(reg_unmasked == 0)) { 1535 dev_err(dev, "%s: spurious interrupt, aintr_status=%016llx\n", 1536 __func__, reg); 1537 goto out; 1538 } 1539 1540 /* FYI, it is 'okay' to clear AFU status before FC_ERROR */ 1541 writeq_be(reg_unmasked, &global->regs.aintr_clear); 1542 1543 /* Check each bit that is on */ 1544 for_each_set_bit(bit, (ulong *)®_unmasked, BITS_PER_LONG) { 1545 if (unlikely(bit >= ARRAY_SIZE(ainfo))) { 1546 WARN_ON_ONCE(1); 1547 continue; 1548 } 1549 1550 info = &ainfo[bit]; 1551 if (unlikely(info->status != 1ULL << bit)) { 1552 WARN_ON_ONCE(1); 1553 continue; 1554 } 1555 1556 port = info->port; 1557 fc_port_regs = get_fc_port_regs(cfg, port); 1558 1559 dev_err(dev, "%s: FC Port %d -> %s, fc_status=%016llx\n", 1560 __func__, port, info->desc, 1561 readq_be(&fc_port_regs[FC_STATUS / 8])); 1562 1563 /* 1564 * Do link reset first, some OTHER errors will set FC_ERROR 1565 * again if cleared before or w/o a reset 1566 */ 1567 if (info->action & LINK_RESET) { 1568 dev_err(dev, "%s: FC Port %d: resetting link\n", 1569 __func__, port); 1570 cfg->lr_state = LINK_RESET_REQUIRED; 1571 cfg->lr_port = port; 1572 schedule_work(&cfg->work_q); 1573 } 1574 1575 if (info->action & CLR_FC_ERROR) { 1576 reg = readq_be(&fc_port_regs[FC_ERROR / 8]); 1577 1578 /* 1579 * Since all errors are unmasked, FC_ERROR and FC_ERRCAP 1580 * should be the same and tracing one is sufficient. 1581 */ 1582 1583 dev_err(dev, "%s: fc %d: clearing fc_error=%016llx\n", 1584 __func__, port, reg); 1585 1586 writeq_be(reg, &fc_port_regs[FC_ERROR / 8]); 1587 writeq_be(0, &fc_port_regs[FC_ERRCAP / 8]); 1588 } 1589 1590 if (info->action & SCAN_HOST) { 1591 atomic_inc(&cfg->scan_host_needed); 1592 schedule_work(&cfg->work_q); 1593 } 1594 } 1595 1596 out: 1597 return IRQ_HANDLED; 1598 } 1599 1600 /** 1601 * start_context() - starts the master context 1602 * @cfg: Internal structure associated with the host. 1603 * @index: Index of the hardware queue. 1604 * 1605 * Return: A success or failure value from CXL services. 1606 */ 1607 static int start_context(struct cxlflash_cfg *cfg, u32 index) 1608 { 1609 struct device *dev = &cfg->dev->dev; 1610 struct hwq *hwq = get_hwq(cfg->afu, index); 1611 int rc = 0; 1612 1613 rc = cxl_start_context(hwq->ctx, 1614 hwq->work.work_element_descriptor, 1615 NULL); 1616 1617 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 1618 return rc; 1619 } 1620 1621 /** 1622 * read_vpd() - obtains the WWPNs from VPD 1623 * @cfg: Internal structure associated with the host. 1624 * @wwpn: Array of size MAX_FC_PORTS to pass back WWPNs 1625 * 1626 * Return: 0 on success, -errno on failure 1627 */ 1628 static int read_vpd(struct cxlflash_cfg *cfg, u64 wwpn[]) 1629 { 1630 struct device *dev = &cfg->dev->dev; 1631 struct pci_dev *pdev = cfg->dev; 1632 int rc = 0; 1633 int ro_start, ro_size, i, j, k; 1634 ssize_t vpd_size; 1635 char vpd_data[CXLFLASH_VPD_LEN]; 1636 char tmp_buf[WWPN_BUF_LEN] = { 0 }; 1637 char *wwpn_vpd_tags[MAX_FC_PORTS] = { "V5", "V6", "V7", "V8" }; 1638 1639 /* Get the VPD data from the device */ 1640 vpd_size = cxl_read_adapter_vpd(pdev, vpd_data, sizeof(vpd_data)); 1641 if (unlikely(vpd_size <= 0)) { 1642 dev_err(dev, "%s: Unable to read VPD (size = %ld)\n", 1643 __func__, vpd_size); 1644 rc = -ENODEV; 1645 goto out; 1646 } 1647 1648 /* Get the read only section offset */ 1649 ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, 1650 PCI_VPD_LRDT_RO_DATA); 1651 if (unlikely(ro_start < 0)) { 1652 dev_err(dev, "%s: VPD Read-only data not found\n", __func__); 1653 rc = -ENODEV; 1654 goto out; 1655 } 1656 1657 /* Get the read only section size, cap when extends beyond read VPD */ 1658 ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]); 1659 j = ro_size; 1660 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 1661 if (unlikely((i + j) > vpd_size)) { 1662 dev_dbg(dev, "%s: Might need to read more VPD (%d > %ld)\n", 1663 __func__, (i + j), vpd_size); 1664 ro_size = vpd_size - i; 1665 } 1666 1667 /* 1668 * Find the offset of the WWPN tag within the read only 1669 * VPD data and validate the found field (partials are 1670 * no good to us). Convert the ASCII data to an integer 1671 * value. Note that we must copy to a temporary buffer 1672 * because the conversion service requires that the ASCII 1673 * string be terminated. 1674 */ 1675 for (k = 0; k < cfg->num_fc_ports; k++) { 1676 j = ro_size; 1677 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 1678 1679 i = pci_vpd_find_info_keyword(vpd_data, i, j, wwpn_vpd_tags[k]); 1680 if (unlikely(i < 0)) { 1681 dev_err(dev, "%s: Port %d WWPN not found in VPD\n", 1682 __func__, k); 1683 rc = -ENODEV; 1684 goto out; 1685 } 1686 1687 j = pci_vpd_info_field_size(&vpd_data[i]); 1688 i += PCI_VPD_INFO_FLD_HDR_SIZE; 1689 if (unlikely((i + j > vpd_size) || (j != WWPN_LEN))) { 1690 dev_err(dev, "%s: Port %d WWPN incomplete or bad VPD\n", 1691 __func__, k); 1692 rc = -ENODEV; 1693 goto out; 1694 } 1695 1696 memcpy(tmp_buf, &vpd_data[i], WWPN_LEN); 1697 rc = kstrtoul(tmp_buf, WWPN_LEN, (ulong *)&wwpn[k]); 1698 if (unlikely(rc)) { 1699 dev_err(dev, "%s: WWPN conversion failed for port %d\n", 1700 __func__, k); 1701 rc = -ENODEV; 1702 goto out; 1703 } 1704 1705 dev_dbg(dev, "%s: wwpn%d=%016llx\n", __func__, k, wwpn[k]); 1706 } 1707 1708 out: 1709 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 1710 return rc; 1711 } 1712 1713 /** 1714 * init_pcr() - initialize the provisioning and control registers 1715 * @cfg: Internal structure associated with the host. 1716 * 1717 * Also sets up fast access to the mapped registers and initializes AFU 1718 * command fields that never change. 1719 */ 1720 static void init_pcr(struct cxlflash_cfg *cfg) 1721 { 1722 struct afu *afu = cfg->afu; 1723 struct sisl_ctrl_map __iomem *ctrl_map; 1724 struct hwq *hwq; 1725 int i; 1726 1727 for (i = 0; i < MAX_CONTEXT; i++) { 1728 ctrl_map = &afu->afu_map->ctrls[i].ctrl; 1729 /* Disrupt any clients that could be running */ 1730 /* e.g. clients that survived a master restart */ 1731 writeq_be(0, &ctrl_map->rht_start); 1732 writeq_be(0, &ctrl_map->rht_cnt_id); 1733 writeq_be(0, &ctrl_map->ctx_cap); 1734 } 1735 1736 /* Copy frequently used fields into hwq */ 1737 for (i = 0; i < afu->num_hwqs; i++) { 1738 hwq = get_hwq(afu, i); 1739 1740 hwq->ctx_hndl = (u16) cxl_process_element(hwq->ctx); 1741 hwq->host_map = &afu->afu_map->hosts[hwq->ctx_hndl].host; 1742 hwq->ctrl_map = &afu->afu_map->ctrls[hwq->ctx_hndl].ctrl; 1743 1744 /* Program the Endian Control for the master context */ 1745 writeq_be(SISL_ENDIAN_CTRL, &hwq->host_map->endian_ctrl); 1746 } 1747 } 1748 1749 /** 1750 * init_global() - initialize AFU global registers 1751 * @cfg: Internal structure associated with the host. 1752 */ 1753 static int init_global(struct cxlflash_cfg *cfg) 1754 { 1755 struct afu *afu = cfg->afu; 1756 struct device *dev = &cfg->dev->dev; 1757 struct hwq *hwq; 1758 struct sisl_host_map __iomem *hmap; 1759 __be64 __iomem *fc_port_regs; 1760 u64 wwpn[MAX_FC_PORTS]; /* wwpn of AFU ports */ 1761 int i = 0, num_ports = 0; 1762 int rc = 0; 1763 u64 reg; 1764 1765 rc = read_vpd(cfg, &wwpn[0]); 1766 if (rc) { 1767 dev_err(dev, "%s: could not read vpd rc=%d\n", __func__, rc); 1768 goto out; 1769 } 1770 1771 /* Set up RRQ and SQ in HWQ for master issued cmds */ 1772 for (i = 0; i < afu->num_hwqs; i++) { 1773 hwq = get_hwq(afu, i); 1774 hmap = hwq->host_map; 1775 1776 writeq_be((u64) hwq->hrrq_start, &hmap->rrq_start); 1777 writeq_be((u64) hwq->hrrq_end, &hmap->rrq_end); 1778 1779 if (afu_is_sq_cmd_mode(afu)) { 1780 writeq_be((u64)hwq->hsq_start, &hmap->sq_start); 1781 writeq_be((u64)hwq->hsq_end, &hmap->sq_end); 1782 } 1783 } 1784 1785 /* AFU configuration */ 1786 reg = readq_be(&afu->afu_map->global.regs.afu_config); 1787 reg |= SISL_AFUCONF_AR_ALL|SISL_AFUCONF_ENDIAN; 1788 /* enable all auto retry options and control endianness */ 1789 /* leave others at default: */ 1790 /* CTX_CAP write protected, mbox_r does not clear on read and */ 1791 /* checker on if dual afu */ 1792 writeq_be(reg, &afu->afu_map->global.regs.afu_config); 1793 1794 /* Global port select: select either port */ 1795 if (afu->internal_lun) { 1796 /* Only use port 0 */ 1797 writeq_be(PORT0, &afu->afu_map->global.regs.afu_port_sel); 1798 num_ports = 0; 1799 } else { 1800 writeq_be(PORT_MASK(cfg->num_fc_ports), 1801 &afu->afu_map->global.regs.afu_port_sel); 1802 num_ports = cfg->num_fc_ports; 1803 } 1804 1805 for (i = 0; i < num_ports; i++) { 1806 fc_port_regs = get_fc_port_regs(cfg, i); 1807 1808 /* Unmask all errors (but they are still masked at AFU) */ 1809 writeq_be(0, &fc_port_regs[FC_ERRMSK / 8]); 1810 /* Clear CRC error cnt & set a threshold */ 1811 (void)readq_be(&fc_port_regs[FC_CNT_CRCERR / 8]); 1812 writeq_be(MC_CRC_THRESH, &fc_port_regs[FC_CRC_THRESH / 8]); 1813 1814 /* Set WWPNs. If already programmed, wwpn[i] is 0 */ 1815 if (wwpn[i] != 0) 1816 afu_set_wwpn(afu, i, &fc_port_regs[0], wwpn[i]); 1817 /* Programming WWPN back to back causes additional 1818 * offline/online transitions and a PLOGI 1819 */ 1820 msleep(100); 1821 } 1822 1823 /* Set up master's own CTX_CAP to allow real mode, host translation */ 1824 /* tables, afu cmds and read/write GSCSI cmds. */ 1825 /* First, unlock ctx_cap write by reading mbox */ 1826 for (i = 0; i < afu->num_hwqs; i++) { 1827 hwq = get_hwq(afu, i); 1828 1829 (void)readq_be(&hwq->ctrl_map->mbox_r); /* unlock ctx_cap */ 1830 writeq_be((SISL_CTX_CAP_REAL_MODE | SISL_CTX_CAP_HOST_XLATE | 1831 SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD | 1832 SISL_CTX_CAP_AFU_CMD | SISL_CTX_CAP_GSCSI_CMD), 1833 &hwq->ctrl_map->ctx_cap); 1834 } 1835 1836 /* 1837 * Determine write-same unmap support for host by evaluating the unmap 1838 * sector support bit of the context control register associated with 1839 * the primary hardware queue. Note that while this status is reflected 1840 * in a context register, the outcome can be assumed to be host-wide. 1841 */ 1842 hwq = get_hwq(afu, PRIMARY_HWQ); 1843 reg = readq_be(&hwq->host_map->ctx_ctrl); 1844 if (reg & SISL_CTX_CTRL_UNMAP_SECTOR) 1845 cfg->ws_unmap = true; 1846 1847 /* Initialize heartbeat */ 1848 afu->hb = readq_be(&afu->afu_map->global.regs.afu_hb); 1849 out: 1850 return rc; 1851 } 1852 1853 /** 1854 * start_afu() - initializes and starts the AFU 1855 * @cfg: Internal structure associated with the host. 1856 */ 1857 static int start_afu(struct cxlflash_cfg *cfg) 1858 { 1859 struct afu *afu = cfg->afu; 1860 struct device *dev = &cfg->dev->dev; 1861 struct hwq *hwq; 1862 int rc = 0; 1863 int i; 1864 1865 init_pcr(cfg); 1866 1867 /* Initialize each HWQ */ 1868 for (i = 0; i < afu->num_hwqs; i++) { 1869 hwq = get_hwq(afu, i); 1870 1871 /* After an AFU reset, RRQ entries are stale, clear them */ 1872 memset(&hwq->rrq_entry, 0, sizeof(hwq->rrq_entry)); 1873 1874 /* Initialize RRQ pointers */ 1875 hwq->hrrq_start = &hwq->rrq_entry[0]; 1876 hwq->hrrq_end = &hwq->rrq_entry[NUM_RRQ_ENTRY - 1]; 1877 hwq->hrrq_curr = hwq->hrrq_start; 1878 hwq->toggle = 1; 1879 1880 /* Initialize spin locks */ 1881 spin_lock_init(&hwq->hrrq_slock); 1882 spin_lock_init(&hwq->hsq_slock); 1883 1884 /* Initialize SQ */ 1885 if (afu_is_sq_cmd_mode(afu)) { 1886 memset(&hwq->sq, 0, sizeof(hwq->sq)); 1887 hwq->hsq_start = &hwq->sq[0]; 1888 hwq->hsq_end = &hwq->sq[NUM_SQ_ENTRY - 1]; 1889 hwq->hsq_curr = hwq->hsq_start; 1890 1891 atomic_set(&hwq->hsq_credits, NUM_SQ_ENTRY - 1); 1892 } 1893 1894 /* Initialize IRQ poll */ 1895 if (afu_is_irqpoll_enabled(afu)) 1896 irq_poll_init(&hwq->irqpoll, afu->irqpoll_weight, 1897 cxlflash_irqpoll); 1898 1899 } 1900 1901 rc = init_global(cfg); 1902 1903 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 1904 return rc; 1905 } 1906 1907 /** 1908 * init_intr() - setup interrupt handlers for the master context 1909 * @cfg: Internal structure associated with the host. 1910 * @hwq: Hardware queue to initialize. 1911 * 1912 * Return: 0 on success, -errno on failure 1913 */ 1914 static enum undo_level init_intr(struct cxlflash_cfg *cfg, 1915 struct hwq *hwq) 1916 { 1917 struct device *dev = &cfg->dev->dev; 1918 struct cxl_context *ctx = hwq->ctx; 1919 int rc = 0; 1920 enum undo_level level = UNDO_NOOP; 1921 bool is_primary_hwq = (hwq->index == PRIMARY_HWQ); 1922 int num_irqs = is_primary_hwq ? 3 : 2; 1923 1924 rc = cxl_allocate_afu_irqs(ctx, num_irqs); 1925 if (unlikely(rc)) { 1926 dev_err(dev, "%s: allocate_afu_irqs failed rc=%d\n", 1927 __func__, rc); 1928 level = UNDO_NOOP; 1929 goto out; 1930 } 1931 1932 rc = cxl_map_afu_irq(ctx, 1, cxlflash_sync_err_irq, hwq, 1933 "SISL_MSI_SYNC_ERROR"); 1934 if (unlikely(rc <= 0)) { 1935 dev_err(dev, "%s: SISL_MSI_SYNC_ERROR map failed\n", __func__); 1936 level = FREE_IRQ; 1937 goto out; 1938 } 1939 1940 rc = cxl_map_afu_irq(ctx, 2, cxlflash_rrq_irq, hwq, 1941 "SISL_MSI_RRQ_UPDATED"); 1942 if (unlikely(rc <= 0)) { 1943 dev_err(dev, "%s: SISL_MSI_RRQ_UPDATED map failed\n", __func__); 1944 level = UNMAP_ONE; 1945 goto out; 1946 } 1947 1948 /* SISL_MSI_ASYNC_ERROR is setup only for the primary HWQ */ 1949 if (!is_primary_hwq) 1950 goto out; 1951 1952 rc = cxl_map_afu_irq(ctx, 3, cxlflash_async_err_irq, hwq, 1953 "SISL_MSI_ASYNC_ERROR"); 1954 if (unlikely(rc <= 0)) { 1955 dev_err(dev, "%s: SISL_MSI_ASYNC_ERROR map failed\n", __func__); 1956 level = UNMAP_TWO; 1957 goto out; 1958 } 1959 out: 1960 return level; 1961 } 1962 1963 /** 1964 * init_mc() - create and register as the master context 1965 * @cfg: Internal structure associated with the host. 1966 * index: HWQ Index of the master context. 1967 * 1968 * Return: 0 on success, -errno on failure 1969 */ 1970 static int init_mc(struct cxlflash_cfg *cfg, u32 index) 1971 { 1972 struct cxl_context *ctx; 1973 struct device *dev = &cfg->dev->dev; 1974 struct hwq *hwq = get_hwq(cfg->afu, index); 1975 int rc = 0; 1976 enum undo_level level; 1977 1978 hwq->afu = cfg->afu; 1979 hwq->index = index; 1980 INIT_LIST_HEAD(&hwq->pending_cmds); 1981 1982 if (index == PRIMARY_HWQ) 1983 ctx = cxl_get_context(cfg->dev); 1984 else 1985 ctx = cxl_dev_context_init(cfg->dev); 1986 if (unlikely(!ctx)) { 1987 rc = -ENOMEM; 1988 goto err1; 1989 } 1990 1991 WARN_ON(hwq->ctx); 1992 hwq->ctx = ctx; 1993 1994 /* Set it up as a master with the CXL */ 1995 cxl_set_master(ctx); 1996 1997 /* Reset AFU when initializing primary context */ 1998 if (index == PRIMARY_HWQ) { 1999 rc = cxl_afu_reset(ctx); 2000 if (unlikely(rc)) { 2001 dev_err(dev, "%s: AFU reset failed rc=%d\n", 2002 __func__, rc); 2003 goto err1; 2004 } 2005 } 2006 2007 level = init_intr(cfg, hwq); 2008 if (unlikely(level)) { 2009 dev_err(dev, "%s: interrupt init failed rc=%d\n", __func__, rc); 2010 goto err2; 2011 } 2012 2013 /* This performs the equivalent of the CXL_IOCTL_START_WORK. 2014 * The CXL_IOCTL_GET_PROCESS_ELEMENT is implicit in the process 2015 * element (pe) that is embedded in the context (ctx) 2016 */ 2017 rc = start_context(cfg, index); 2018 if (unlikely(rc)) { 2019 dev_err(dev, "%s: start context failed rc=%d\n", __func__, rc); 2020 level = UNMAP_THREE; 2021 goto err2; 2022 } 2023 2024 out: 2025 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 2026 return rc; 2027 err2: 2028 term_intr(cfg, level, index); 2029 if (index != PRIMARY_HWQ) 2030 cxl_release_context(ctx); 2031 err1: 2032 hwq->ctx = NULL; 2033 goto out; 2034 } 2035 2036 /** 2037 * get_num_afu_ports() - determines and configures the number of AFU ports 2038 * @cfg: Internal structure associated with the host. 2039 * 2040 * This routine determines the number of AFU ports by converting the global 2041 * port selection mask. The converted value is only valid following an AFU 2042 * reset (explicit or power-on). This routine must be invoked shortly after 2043 * mapping as other routines are dependent on the number of ports during the 2044 * initialization sequence. 2045 * 2046 * To support legacy AFUs that might not have reflected an initial global 2047 * port mask (value read is 0), default to the number of ports originally 2048 * supported by the cxlflash driver (2) before hardware with other port 2049 * offerings was introduced. 2050 */ 2051 static void get_num_afu_ports(struct cxlflash_cfg *cfg) 2052 { 2053 struct afu *afu = cfg->afu; 2054 struct device *dev = &cfg->dev->dev; 2055 u64 port_mask; 2056 int num_fc_ports = LEGACY_FC_PORTS; 2057 2058 port_mask = readq_be(&afu->afu_map->global.regs.afu_port_sel); 2059 if (port_mask != 0ULL) 2060 num_fc_ports = min(ilog2(port_mask) + 1, MAX_FC_PORTS); 2061 2062 dev_dbg(dev, "%s: port_mask=%016llx num_fc_ports=%d\n", 2063 __func__, port_mask, num_fc_ports); 2064 2065 cfg->num_fc_ports = num_fc_ports; 2066 cfg->host->max_channel = PORTNUM2CHAN(num_fc_ports); 2067 } 2068 2069 /** 2070 * init_afu() - setup as master context and start AFU 2071 * @cfg: Internal structure associated with the host. 2072 * 2073 * This routine is a higher level of control for configuring the 2074 * AFU on probe and reset paths. 2075 * 2076 * Return: 0 on success, -errno on failure 2077 */ 2078 static int init_afu(struct cxlflash_cfg *cfg) 2079 { 2080 u64 reg; 2081 int rc = 0; 2082 struct afu *afu = cfg->afu; 2083 struct device *dev = &cfg->dev->dev; 2084 struct hwq *hwq; 2085 int i; 2086 2087 cxl_perst_reloads_same_image(cfg->cxl_afu, true); 2088 2089 afu->num_hwqs = afu->desired_hwqs; 2090 for (i = 0; i < afu->num_hwqs; i++) { 2091 rc = init_mc(cfg, i); 2092 if (rc) { 2093 dev_err(dev, "%s: init_mc failed rc=%d index=%d\n", 2094 __func__, rc, i); 2095 goto err1; 2096 } 2097 } 2098 2099 /* Map the entire MMIO space of the AFU using the first context */ 2100 hwq = get_hwq(afu, PRIMARY_HWQ); 2101 afu->afu_map = cxl_psa_map(hwq->ctx); 2102 if (!afu->afu_map) { 2103 dev_err(dev, "%s: cxl_psa_map failed\n", __func__); 2104 rc = -ENOMEM; 2105 goto err1; 2106 } 2107 2108 /* No byte reverse on reading afu_version or string will be backwards */ 2109 reg = readq(&afu->afu_map->global.regs.afu_version); 2110 memcpy(afu->version, ®, sizeof(reg)); 2111 afu->interface_version = 2112 readq_be(&afu->afu_map->global.regs.interface_version); 2113 if ((afu->interface_version + 1) == 0) { 2114 dev_err(dev, "Back level AFU, please upgrade. AFU version %s " 2115 "interface version %016llx\n", afu->version, 2116 afu->interface_version); 2117 rc = -EINVAL; 2118 goto err1; 2119 } 2120 2121 if (afu_is_sq_cmd_mode(afu)) { 2122 afu->send_cmd = send_cmd_sq; 2123 afu->context_reset = context_reset_sq; 2124 } else { 2125 afu->send_cmd = send_cmd_ioarrin; 2126 afu->context_reset = context_reset_ioarrin; 2127 } 2128 2129 dev_dbg(dev, "%s: afu_ver=%s interface_ver=%016llx\n", __func__, 2130 afu->version, afu->interface_version); 2131 2132 get_num_afu_ports(cfg); 2133 2134 rc = start_afu(cfg); 2135 if (rc) { 2136 dev_err(dev, "%s: start_afu failed, rc=%d\n", __func__, rc); 2137 goto err1; 2138 } 2139 2140 afu_err_intr_init(cfg->afu); 2141 for (i = 0; i < afu->num_hwqs; i++) { 2142 hwq = get_hwq(afu, i); 2143 2144 hwq->room = readq_be(&hwq->host_map->cmd_room); 2145 } 2146 2147 /* Restore the LUN mappings */ 2148 cxlflash_restore_luntable(cfg); 2149 out: 2150 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 2151 return rc; 2152 2153 err1: 2154 for (i = afu->num_hwqs - 1; i >= 0; i--) { 2155 term_intr(cfg, UNMAP_THREE, i); 2156 term_mc(cfg, i); 2157 } 2158 goto out; 2159 } 2160 2161 /** 2162 * afu_reset() - resets the AFU 2163 * @cfg: Internal structure associated with the host. 2164 * 2165 * Return: 0 on success, -errno on failure 2166 */ 2167 static int afu_reset(struct cxlflash_cfg *cfg) 2168 { 2169 struct device *dev = &cfg->dev->dev; 2170 int rc = 0; 2171 2172 /* Stop the context before the reset. Since the context is 2173 * no longer available restart it after the reset is complete 2174 */ 2175 term_afu(cfg); 2176 2177 rc = init_afu(cfg); 2178 2179 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 2180 return rc; 2181 } 2182 2183 /** 2184 * drain_ioctls() - wait until all currently executing ioctls have completed 2185 * @cfg: Internal structure associated with the host. 2186 * 2187 * Obtain write access to read/write semaphore that wraps ioctl 2188 * handling to 'drain' ioctls currently executing. 2189 */ 2190 static void drain_ioctls(struct cxlflash_cfg *cfg) 2191 { 2192 down_write(&cfg->ioctl_rwsem); 2193 up_write(&cfg->ioctl_rwsem); 2194 } 2195 2196 /** 2197 * cxlflash_async_reset_host() - asynchronous host reset handler 2198 * @data: Private data provided while scheduling reset. 2199 * @cookie: Cookie that can be used for checkpointing. 2200 */ 2201 static void cxlflash_async_reset_host(void *data, async_cookie_t cookie) 2202 { 2203 struct cxlflash_cfg *cfg = data; 2204 struct device *dev = &cfg->dev->dev; 2205 int rc = 0; 2206 2207 if (cfg->state != STATE_RESET) { 2208 dev_dbg(dev, "%s: Not performing a reset, state=%d\n", 2209 __func__, cfg->state); 2210 goto out; 2211 } 2212 2213 drain_ioctls(cfg); 2214 cxlflash_mark_contexts_error(cfg); 2215 rc = afu_reset(cfg); 2216 if (rc) 2217 cfg->state = STATE_FAILTERM; 2218 else 2219 cfg->state = STATE_NORMAL; 2220 wake_up_all(&cfg->reset_waitq); 2221 2222 out: 2223 scsi_unblock_requests(cfg->host); 2224 } 2225 2226 /** 2227 * cxlflash_schedule_async_reset() - schedule an asynchronous host reset 2228 * @cfg: Internal structure associated with the host. 2229 */ 2230 static void cxlflash_schedule_async_reset(struct cxlflash_cfg *cfg) 2231 { 2232 struct device *dev = &cfg->dev->dev; 2233 2234 if (cfg->state != STATE_NORMAL) { 2235 dev_dbg(dev, "%s: Not performing reset state=%d\n", 2236 __func__, cfg->state); 2237 return; 2238 } 2239 2240 cfg->state = STATE_RESET; 2241 scsi_block_requests(cfg->host); 2242 cfg->async_reset_cookie = async_schedule(cxlflash_async_reset_host, 2243 cfg); 2244 } 2245 2246 /** 2247 * send_afu_cmd() - builds and sends an internal AFU command 2248 * @afu: AFU associated with the host. 2249 * @rcb: Pre-populated IOARCB describing command to send. 2250 * 2251 * The AFU can only take one internal AFU command at a time. This limitation is 2252 * enforced by using a mutex to provide exclusive access to the AFU during the 2253 * operation. This design point requires calling threads to not be on interrupt 2254 * context due to the possibility of sleeping during concurrent AFU operations. 2255 * 2256 * The command status is optionally passed back to the caller when the caller 2257 * populates the IOASA field of the IOARCB with a pointer to an IOASA structure. 2258 * 2259 * Return: 2260 * 0 on success, -errno on failure 2261 */ 2262 static int send_afu_cmd(struct afu *afu, struct sisl_ioarcb *rcb) 2263 { 2264 struct cxlflash_cfg *cfg = afu->parent; 2265 struct device *dev = &cfg->dev->dev; 2266 struct afu_cmd *cmd = NULL; 2267 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ); 2268 char *buf = NULL; 2269 int rc = 0; 2270 int nretry = 0; 2271 static DEFINE_MUTEX(sync_active); 2272 2273 if (cfg->state != STATE_NORMAL) { 2274 dev_dbg(dev, "%s: Sync not required state=%u\n", 2275 __func__, cfg->state); 2276 return 0; 2277 } 2278 2279 mutex_lock(&sync_active); 2280 atomic_inc(&afu->cmds_active); 2281 buf = kmalloc(sizeof(*cmd) + __alignof__(*cmd) - 1, GFP_KERNEL); 2282 if (unlikely(!buf)) { 2283 dev_err(dev, "%s: no memory for command\n", __func__); 2284 rc = -ENOMEM; 2285 goto out; 2286 } 2287 2288 cmd = (struct afu_cmd *)PTR_ALIGN(buf, __alignof__(*cmd)); 2289 2290 retry: 2291 memset(cmd, 0, sizeof(*cmd)); 2292 memcpy(&cmd->rcb, rcb, sizeof(*rcb)); 2293 INIT_LIST_HEAD(&cmd->queue); 2294 init_completion(&cmd->cevent); 2295 cmd->parent = afu; 2296 cmd->hwq_index = hwq->index; 2297 cmd->rcb.ctx_id = hwq->ctx_hndl; 2298 2299 dev_dbg(dev, "%s: afu=%p cmd=%p type=%02x nretry=%d\n", 2300 __func__, afu, cmd, cmd->rcb.cdb[0], nretry); 2301 2302 rc = afu->send_cmd(afu, cmd); 2303 if (unlikely(rc)) { 2304 rc = -ENOBUFS; 2305 goto out; 2306 } 2307 2308 rc = wait_resp(afu, cmd); 2309 switch (rc) { 2310 case -ETIMEDOUT: 2311 rc = afu->context_reset(hwq); 2312 if (rc) { 2313 cxlflash_schedule_async_reset(cfg); 2314 break; 2315 } 2316 /* fall through to retry */ 2317 case -EAGAIN: 2318 if (++nretry < 2) 2319 goto retry; 2320 /* fall through to exit */ 2321 default: 2322 break; 2323 } 2324 2325 if (rcb->ioasa) 2326 *rcb->ioasa = cmd->sa; 2327 out: 2328 atomic_dec(&afu->cmds_active); 2329 mutex_unlock(&sync_active); 2330 kfree(buf); 2331 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 2332 return rc; 2333 } 2334 2335 /** 2336 * cxlflash_afu_sync() - builds and sends an AFU sync command 2337 * @afu: AFU associated with the host. 2338 * @ctx: Identifies context requesting sync. 2339 * @res: Identifies resource requesting sync. 2340 * @mode: Type of sync to issue (lightweight, heavyweight, global). 2341 * 2342 * AFU sync operations are only necessary and allowed when the device is 2343 * operating normally. When not operating normally, sync requests can occur as 2344 * part of cleaning up resources associated with an adapter prior to removal. 2345 * In this scenario, these requests are simply ignored (safe due to the AFU 2346 * going away). 2347 * 2348 * Return: 2349 * 0 on success, -errno on failure 2350 */ 2351 int cxlflash_afu_sync(struct afu *afu, ctx_hndl_t ctx, res_hndl_t res, u8 mode) 2352 { 2353 struct cxlflash_cfg *cfg = afu->parent; 2354 struct device *dev = &cfg->dev->dev; 2355 struct sisl_ioarcb rcb = { 0 }; 2356 2357 dev_dbg(dev, "%s: afu=%p ctx=%u res=%u mode=%u\n", 2358 __func__, afu, ctx, res, mode); 2359 2360 rcb.req_flags = SISL_REQ_FLAGS_AFU_CMD; 2361 rcb.msi = SISL_MSI_RRQ_UPDATED; 2362 rcb.timeout = MC_AFU_SYNC_TIMEOUT; 2363 2364 rcb.cdb[0] = SISL_AFU_CMD_SYNC; 2365 rcb.cdb[1] = mode; 2366 put_unaligned_be16(ctx, &rcb.cdb[2]); 2367 put_unaligned_be32(res, &rcb.cdb[4]); 2368 2369 return send_afu_cmd(afu, &rcb); 2370 } 2371 2372 /** 2373 * cxlflash_eh_abort_handler() - abort a SCSI command 2374 * @scp: SCSI command to abort. 2375 * 2376 * CXL Flash devices do not support a single command abort. Reset the context 2377 * as per SISLite specification. Flush any pending commands in the hardware 2378 * queue before the reset. 2379 * 2380 * Return: SUCCESS/FAILED as defined in scsi/scsi.h 2381 */ 2382 static int cxlflash_eh_abort_handler(struct scsi_cmnd *scp) 2383 { 2384 int rc = FAILED; 2385 struct Scsi_Host *host = scp->device->host; 2386 struct cxlflash_cfg *cfg = shost_priv(host); 2387 struct afu_cmd *cmd = sc_to_afuc(scp); 2388 struct device *dev = &cfg->dev->dev; 2389 struct afu *afu = cfg->afu; 2390 struct hwq *hwq = get_hwq(afu, cmd->hwq_index); 2391 2392 dev_dbg(dev, "%s: (scp=%p) %d/%d/%d/%llu " 2393 "cdb=(%08x-%08x-%08x-%08x)\n", __func__, scp, host->host_no, 2394 scp->device->channel, scp->device->id, scp->device->lun, 2395 get_unaligned_be32(&((u32 *)scp->cmnd)[0]), 2396 get_unaligned_be32(&((u32 *)scp->cmnd)[1]), 2397 get_unaligned_be32(&((u32 *)scp->cmnd)[2]), 2398 get_unaligned_be32(&((u32 *)scp->cmnd)[3])); 2399 2400 /* When the state is not normal, another reset/reload is in progress. 2401 * Return failed and the mid-layer will invoke host reset handler. 2402 */ 2403 if (cfg->state != STATE_NORMAL) { 2404 dev_dbg(dev, "%s: Invalid state for abort, state=%d\n", 2405 __func__, cfg->state); 2406 goto out; 2407 } 2408 2409 rc = afu->context_reset(hwq); 2410 if (unlikely(rc)) 2411 goto out; 2412 2413 rc = SUCCESS; 2414 2415 out: 2416 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 2417 return rc; 2418 } 2419 2420 /** 2421 * cxlflash_eh_device_reset_handler() - reset a single LUN 2422 * @scp: SCSI command to send. 2423 * 2424 * Return: 2425 * SUCCESS as defined in scsi/scsi.h 2426 * FAILED as defined in scsi/scsi.h 2427 */ 2428 static int cxlflash_eh_device_reset_handler(struct scsi_cmnd *scp) 2429 { 2430 int rc = SUCCESS; 2431 struct scsi_device *sdev = scp->device; 2432 struct Scsi_Host *host = sdev->host; 2433 struct cxlflash_cfg *cfg = shost_priv(host); 2434 struct device *dev = &cfg->dev->dev; 2435 int rcr = 0; 2436 2437 dev_dbg(dev, "%s: %d/%d/%d/%llu\n", __func__, 2438 host->host_no, sdev->channel, sdev->id, sdev->lun); 2439 retry: 2440 switch (cfg->state) { 2441 case STATE_NORMAL: 2442 rcr = send_tmf(cfg, sdev, TMF_LUN_RESET); 2443 if (unlikely(rcr)) 2444 rc = FAILED; 2445 break; 2446 case STATE_RESET: 2447 wait_event(cfg->reset_waitq, cfg->state != STATE_RESET); 2448 goto retry; 2449 default: 2450 rc = FAILED; 2451 break; 2452 } 2453 2454 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 2455 return rc; 2456 } 2457 2458 /** 2459 * cxlflash_eh_host_reset_handler() - reset the host adapter 2460 * @scp: SCSI command from stack identifying host. 2461 * 2462 * Following a reset, the state is evaluated again in case an EEH occurred 2463 * during the reset. In such a scenario, the host reset will either yield 2464 * until the EEH recovery is complete or return success or failure based 2465 * upon the current device state. 2466 * 2467 * Return: 2468 * SUCCESS as defined in scsi/scsi.h 2469 * FAILED as defined in scsi/scsi.h 2470 */ 2471 static int cxlflash_eh_host_reset_handler(struct scsi_cmnd *scp) 2472 { 2473 int rc = SUCCESS; 2474 int rcr = 0; 2475 struct Scsi_Host *host = scp->device->host; 2476 struct cxlflash_cfg *cfg = shost_priv(host); 2477 struct device *dev = &cfg->dev->dev; 2478 2479 dev_dbg(dev, "%s: %d\n", __func__, host->host_no); 2480 2481 switch (cfg->state) { 2482 case STATE_NORMAL: 2483 cfg->state = STATE_RESET; 2484 drain_ioctls(cfg); 2485 cxlflash_mark_contexts_error(cfg); 2486 rcr = afu_reset(cfg); 2487 if (rcr) { 2488 rc = FAILED; 2489 cfg->state = STATE_FAILTERM; 2490 } else 2491 cfg->state = STATE_NORMAL; 2492 wake_up_all(&cfg->reset_waitq); 2493 ssleep(1); 2494 /* fall through */ 2495 case STATE_RESET: 2496 wait_event(cfg->reset_waitq, cfg->state != STATE_RESET); 2497 if (cfg->state == STATE_NORMAL) 2498 break; 2499 /* fall through */ 2500 default: 2501 rc = FAILED; 2502 break; 2503 } 2504 2505 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 2506 return rc; 2507 } 2508 2509 /** 2510 * cxlflash_change_queue_depth() - change the queue depth for the device 2511 * @sdev: SCSI device destined for queue depth change. 2512 * @qdepth: Requested queue depth value to set. 2513 * 2514 * The requested queue depth is capped to the maximum supported value. 2515 * 2516 * Return: The actual queue depth set. 2517 */ 2518 static int cxlflash_change_queue_depth(struct scsi_device *sdev, int qdepth) 2519 { 2520 2521 if (qdepth > CXLFLASH_MAX_CMDS_PER_LUN) 2522 qdepth = CXLFLASH_MAX_CMDS_PER_LUN; 2523 2524 scsi_change_queue_depth(sdev, qdepth); 2525 return sdev->queue_depth; 2526 } 2527 2528 /** 2529 * cxlflash_show_port_status() - queries and presents the current port status 2530 * @port: Desired port for status reporting. 2531 * @cfg: Internal structure associated with the host. 2532 * @buf: Buffer of length PAGE_SIZE to report back port status in ASCII. 2533 * 2534 * Return: The size of the ASCII string returned in @buf or -EINVAL. 2535 */ 2536 static ssize_t cxlflash_show_port_status(u32 port, 2537 struct cxlflash_cfg *cfg, 2538 char *buf) 2539 { 2540 struct device *dev = &cfg->dev->dev; 2541 char *disp_status; 2542 u64 status; 2543 __be64 __iomem *fc_port_regs; 2544 2545 WARN_ON(port >= MAX_FC_PORTS); 2546 2547 if (port >= cfg->num_fc_ports) { 2548 dev_info(dev, "%s: Port %d not supported on this card.\n", 2549 __func__, port); 2550 return -EINVAL; 2551 } 2552 2553 fc_port_regs = get_fc_port_regs(cfg, port); 2554 status = readq_be(&fc_port_regs[FC_MTIP_STATUS / 8]); 2555 status &= FC_MTIP_STATUS_MASK; 2556 2557 if (status == FC_MTIP_STATUS_ONLINE) 2558 disp_status = "online"; 2559 else if (status == FC_MTIP_STATUS_OFFLINE) 2560 disp_status = "offline"; 2561 else 2562 disp_status = "unknown"; 2563 2564 return scnprintf(buf, PAGE_SIZE, "%s\n", disp_status); 2565 } 2566 2567 /** 2568 * port0_show() - queries and presents the current status of port 0 2569 * @dev: Generic device associated with the host owning the port. 2570 * @attr: Device attribute representing the port. 2571 * @buf: Buffer of length PAGE_SIZE to report back port status in ASCII. 2572 * 2573 * Return: The size of the ASCII string returned in @buf. 2574 */ 2575 static ssize_t port0_show(struct device *dev, 2576 struct device_attribute *attr, 2577 char *buf) 2578 { 2579 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2580 2581 return cxlflash_show_port_status(0, cfg, buf); 2582 } 2583 2584 /** 2585 * port1_show() - queries and presents the current status of port 1 2586 * @dev: Generic device associated with the host owning the port. 2587 * @attr: Device attribute representing the port. 2588 * @buf: Buffer of length PAGE_SIZE to report back port status in ASCII. 2589 * 2590 * Return: The size of the ASCII string returned in @buf. 2591 */ 2592 static ssize_t port1_show(struct device *dev, 2593 struct device_attribute *attr, 2594 char *buf) 2595 { 2596 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2597 2598 return cxlflash_show_port_status(1, cfg, buf); 2599 } 2600 2601 /** 2602 * port2_show() - queries and presents the current status of port 2 2603 * @dev: Generic device associated with the host owning the port. 2604 * @attr: Device attribute representing the port. 2605 * @buf: Buffer of length PAGE_SIZE to report back port status in ASCII. 2606 * 2607 * Return: The size of the ASCII string returned in @buf. 2608 */ 2609 static ssize_t port2_show(struct device *dev, 2610 struct device_attribute *attr, 2611 char *buf) 2612 { 2613 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2614 2615 return cxlflash_show_port_status(2, cfg, buf); 2616 } 2617 2618 /** 2619 * port3_show() - queries and presents the current status of port 3 2620 * @dev: Generic device associated with the host owning the port. 2621 * @attr: Device attribute representing the port. 2622 * @buf: Buffer of length PAGE_SIZE to report back port status in ASCII. 2623 * 2624 * Return: The size of the ASCII string returned in @buf. 2625 */ 2626 static ssize_t port3_show(struct device *dev, 2627 struct device_attribute *attr, 2628 char *buf) 2629 { 2630 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2631 2632 return cxlflash_show_port_status(3, cfg, buf); 2633 } 2634 2635 /** 2636 * lun_mode_show() - presents the current LUN mode of the host 2637 * @dev: Generic device associated with the host. 2638 * @attr: Device attribute representing the LUN mode. 2639 * @buf: Buffer of length PAGE_SIZE to report back the LUN mode in ASCII. 2640 * 2641 * Return: The size of the ASCII string returned in @buf. 2642 */ 2643 static ssize_t lun_mode_show(struct device *dev, 2644 struct device_attribute *attr, char *buf) 2645 { 2646 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2647 struct afu *afu = cfg->afu; 2648 2649 return scnprintf(buf, PAGE_SIZE, "%u\n", afu->internal_lun); 2650 } 2651 2652 /** 2653 * lun_mode_store() - sets the LUN mode of the host 2654 * @dev: Generic device associated with the host. 2655 * @attr: Device attribute representing the LUN mode. 2656 * @buf: Buffer of length PAGE_SIZE containing the LUN mode in ASCII. 2657 * @count: Length of data resizing in @buf. 2658 * 2659 * The CXL Flash AFU supports a dummy LUN mode where the external 2660 * links and storage are not required. Space on the FPGA is used 2661 * to create 1 or 2 small LUNs which are presented to the system 2662 * as if they were a normal storage device. This feature is useful 2663 * during development and also provides manufacturing with a way 2664 * to test the AFU without an actual device. 2665 * 2666 * 0 = external LUN[s] (default) 2667 * 1 = internal LUN (1 x 64K, 512B blocks, id 0) 2668 * 2 = internal LUN (1 x 64K, 4K blocks, id 0) 2669 * 3 = internal LUN (2 x 32K, 512B blocks, ids 0,1) 2670 * 4 = internal LUN (2 x 32K, 4K blocks, ids 0,1) 2671 * 2672 * Return: The size of the ASCII string returned in @buf. 2673 */ 2674 static ssize_t lun_mode_store(struct device *dev, 2675 struct device_attribute *attr, 2676 const char *buf, size_t count) 2677 { 2678 struct Scsi_Host *shost = class_to_shost(dev); 2679 struct cxlflash_cfg *cfg = shost_priv(shost); 2680 struct afu *afu = cfg->afu; 2681 int rc; 2682 u32 lun_mode; 2683 2684 rc = kstrtouint(buf, 10, &lun_mode); 2685 if (!rc && (lun_mode < 5) && (lun_mode != afu->internal_lun)) { 2686 afu->internal_lun = lun_mode; 2687 2688 /* 2689 * When configured for internal LUN, there is only one channel, 2690 * channel number 0, else there will be one less than the number 2691 * of fc ports for this card. 2692 */ 2693 if (afu->internal_lun) 2694 shost->max_channel = 0; 2695 else 2696 shost->max_channel = PORTNUM2CHAN(cfg->num_fc_ports); 2697 2698 afu_reset(cfg); 2699 scsi_scan_host(cfg->host); 2700 } 2701 2702 return count; 2703 } 2704 2705 /** 2706 * ioctl_version_show() - presents the current ioctl version of the host 2707 * @dev: Generic device associated with the host. 2708 * @attr: Device attribute representing the ioctl version. 2709 * @buf: Buffer of length PAGE_SIZE to report back the ioctl version. 2710 * 2711 * Return: The size of the ASCII string returned in @buf. 2712 */ 2713 static ssize_t ioctl_version_show(struct device *dev, 2714 struct device_attribute *attr, char *buf) 2715 { 2716 ssize_t bytes = 0; 2717 2718 bytes = scnprintf(buf, PAGE_SIZE, 2719 "disk: %u\n", DK_CXLFLASH_VERSION_0); 2720 bytes += scnprintf(buf + bytes, PAGE_SIZE - bytes, 2721 "host: %u\n", HT_CXLFLASH_VERSION_0); 2722 2723 return bytes; 2724 } 2725 2726 /** 2727 * cxlflash_show_port_lun_table() - queries and presents the port LUN table 2728 * @port: Desired port for status reporting. 2729 * @cfg: Internal structure associated with the host. 2730 * @buf: Buffer of length PAGE_SIZE to report back port status in ASCII. 2731 * 2732 * Return: The size of the ASCII string returned in @buf or -EINVAL. 2733 */ 2734 static ssize_t cxlflash_show_port_lun_table(u32 port, 2735 struct cxlflash_cfg *cfg, 2736 char *buf) 2737 { 2738 struct device *dev = &cfg->dev->dev; 2739 __be64 __iomem *fc_port_luns; 2740 int i; 2741 ssize_t bytes = 0; 2742 2743 WARN_ON(port >= MAX_FC_PORTS); 2744 2745 if (port >= cfg->num_fc_ports) { 2746 dev_info(dev, "%s: Port %d not supported on this card.\n", 2747 __func__, port); 2748 return -EINVAL; 2749 } 2750 2751 fc_port_luns = get_fc_port_luns(cfg, port); 2752 2753 for (i = 0; i < CXLFLASH_NUM_VLUNS; i++) 2754 bytes += scnprintf(buf + bytes, PAGE_SIZE - bytes, 2755 "%03d: %016llx\n", 2756 i, readq_be(&fc_port_luns[i])); 2757 return bytes; 2758 } 2759 2760 /** 2761 * port0_lun_table_show() - presents the current LUN table of port 0 2762 * @dev: Generic device associated with the host owning the port. 2763 * @attr: Device attribute representing the port. 2764 * @buf: Buffer of length PAGE_SIZE to report back port status in ASCII. 2765 * 2766 * Return: The size of the ASCII string returned in @buf. 2767 */ 2768 static ssize_t port0_lun_table_show(struct device *dev, 2769 struct device_attribute *attr, 2770 char *buf) 2771 { 2772 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2773 2774 return cxlflash_show_port_lun_table(0, cfg, buf); 2775 } 2776 2777 /** 2778 * port1_lun_table_show() - presents the current LUN table of port 1 2779 * @dev: Generic device associated with the host owning the port. 2780 * @attr: Device attribute representing the port. 2781 * @buf: Buffer of length PAGE_SIZE to report back port status in ASCII. 2782 * 2783 * Return: The size of the ASCII string returned in @buf. 2784 */ 2785 static ssize_t port1_lun_table_show(struct device *dev, 2786 struct device_attribute *attr, 2787 char *buf) 2788 { 2789 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2790 2791 return cxlflash_show_port_lun_table(1, cfg, buf); 2792 } 2793 2794 /** 2795 * port2_lun_table_show() - presents the current LUN table of port 2 2796 * @dev: Generic device associated with the host owning the port. 2797 * @attr: Device attribute representing the port. 2798 * @buf: Buffer of length PAGE_SIZE to report back port status in ASCII. 2799 * 2800 * Return: The size of the ASCII string returned in @buf. 2801 */ 2802 static ssize_t port2_lun_table_show(struct device *dev, 2803 struct device_attribute *attr, 2804 char *buf) 2805 { 2806 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2807 2808 return cxlflash_show_port_lun_table(2, cfg, buf); 2809 } 2810 2811 /** 2812 * port3_lun_table_show() - presents the current LUN table of port 3 2813 * @dev: Generic device associated with the host owning the port. 2814 * @attr: Device attribute representing the port. 2815 * @buf: Buffer of length PAGE_SIZE to report back port status in ASCII. 2816 * 2817 * Return: The size of the ASCII string returned in @buf. 2818 */ 2819 static ssize_t port3_lun_table_show(struct device *dev, 2820 struct device_attribute *attr, 2821 char *buf) 2822 { 2823 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2824 2825 return cxlflash_show_port_lun_table(3, cfg, buf); 2826 } 2827 2828 /** 2829 * irqpoll_weight_show() - presents the current IRQ poll weight for the host 2830 * @dev: Generic device associated with the host. 2831 * @attr: Device attribute representing the IRQ poll weight. 2832 * @buf: Buffer of length PAGE_SIZE to report back the current IRQ poll 2833 * weight in ASCII. 2834 * 2835 * An IRQ poll weight of 0 indicates polling is disabled. 2836 * 2837 * Return: The size of the ASCII string returned in @buf. 2838 */ 2839 static ssize_t irqpoll_weight_show(struct device *dev, 2840 struct device_attribute *attr, char *buf) 2841 { 2842 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2843 struct afu *afu = cfg->afu; 2844 2845 return scnprintf(buf, PAGE_SIZE, "%u\n", afu->irqpoll_weight); 2846 } 2847 2848 /** 2849 * irqpoll_weight_store() - sets the current IRQ poll weight for the host 2850 * @dev: Generic device associated with the host. 2851 * @attr: Device attribute representing the IRQ poll weight. 2852 * @buf: Buffer of length PAGE_SIZE containing the desired IRQ poll 2853 * weight in ASCII. 2854 * @count: Length of data resizing in @buf. 2855 * 2856 * An IRQ poll weight of 0 indicates polling is disabled. 2857 * 2858 * Return: The size of the ASCII string returned in @buf. 2859 */ 2860 static ssize_t irqpoll_weight_store(struct device *dev, 2861 struct device_attribute *attr, 2862 const char *buf, size_t count) 2863 { 2864 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2865 struct device *cfgdev = &cfg->dev->dev; 2866 struct afu *afu = cfg->afu; 2867 struct hwq *hwq; 2868 u32 weight; 2869 int rc, i; 2870 2871 rc = kstrtouint(buf, 10, &weight); 2872 if (rc) 2873 return -EINVAL; 2874 2875 if (weight > 256) { 2876 dev_info(cfgdev, 2877 "Invalid IRQ poll weight. It must be 256 or less.\n"); 2878 return -EINVAL; 2879 } 2880 2881 if (weight == afu->irqpoll_weight) { 2882 dev_info(cfgdev, 2883 "Current IRQ poll weight has the same weight.\n"); 2884 return -EINVAL; 2885 } 2886 2887 if (afu_is_irqpoll_enabled(afu)) { 2888 for (i = 0; i < afu->num_hwqs; i++) { 2889 hwq = get_hwq(afu, i); 2890 2891 irq_poll_disable(&hwq->irqpoll); 2892 } 2893 } 2894 2895 afu->irqpoll_weight = weight; 2896 2897 if (weight > 0) { 2898 for (i = 0; i < afu->num_hwqs; i++) { 2899 hwq = get_hwq(afu, i); 2900 2901 irq_poll_init(&hwq->irqpoll, weight, cxlflash_irqpoll); 2902 } 2903 } 2904 2905 return count; 2906 } 2907 2908 /** 2909 * num_hwqs_show() - presents the number of hardware queues for the host 2910 * @dev: Generic device associated with the host. 2911 * @attr: Device attribute representing the number of hardware queues. 2912 * @buf: Buffer of length PAGE_SIZE to report back the number of hardware 2913 * queues in ASCII. 2914 * 2915 * Return: The size of the ASCII string returned in @buf. 2916 */ 2917 static ssize_t num_hwqs_show(struct device *dev, 2918 struct device_attribute *attr, char *buf) 2919 { 2920 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2921 struct afu *afu = cfg->afu; 2922 2923 return scnprintf(buf, PAGE_SIZE, "%u\n", afu->num_hwqs); 2924 } 2925 2926 /** 2927 * num_hwqs_store() - sets the number of hardware queues for the host 2928 * @dev: Generic device associated with the host. 2929 * @attr: Device attribute representing the number of hardware queues. 2930 * @buf: Buffer of length PAGE_SIZE containing the number of hardware 2931 * queues in ASCII. 2932 * @count: Length of data resizing in @buf. 2933 * 2934 * n > 0: num_hwqs = n 2935 * n = 0: num_hwqs = num_online_cpus() 2936 * n < 0: num_online_cpus() / abs(n) 2937 * 2938 * Return: The size of the ASCII string returned in @buf. 2939 */ 2940 static ssize_t num_hwqs_store(struct device *dev, 2941 struct device_attribute *attr, 2942 const char *buf, size_t count) 2943 { 2944 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 2945 struct afu *afu = cfg->afu; 2946 int rc; 2947 int nhwqs, num_hwqs; 2948 2949 rc = kstrtoint(buf, 10, &nhwqs); 2950 if (rc) 2951 return -EINVAL; 2952 2953 if (nhwqs >= 1) 2954 num_hwqs = nhwqs; 2955 else if (nhwqs == 0) 2956 num_hwqs = num_online_cpus(); 2957 else 2958 num_hwqs = num_online_cpus() / abs(nhwqs); 2959 2960 afu->desired_hwqs = min(num_hwqs, CXLFLASH_MAX_HWQS); 2961 WARN_ON_ONCE(afu->desired_hwqs == 0); 2962 2963 retry: 2964 switch (cfg->state) { 2965 case STATE_NORMAL: 2966 cfg->state = STATE_RESET; 2967 drain_ioctls(cfg); 2968 cxlflash_mark_contexts_error(cfg); 2969 rc = afu_reset(cfg); 2970 if (rc) 2971 cfg->state = STATE_FAILTERM; 2972 else 2973 cfg->state = STATE_NORMAL; 2974 wake_up_all(&cfg->reset_waitq); 2975 break; 2976 case STATE_RESET: 2977 wait_event(cfg->reset_waitq, cfg->state != STATE_RESET); 2978 if (cfg->state == STATE_NORMAL) 2979 goto retry; 2980 default: 2981 /* Ideally should not happen */ 2982 dev_err(dev, "%s: Device is not ready, state=%d\n", 2983 __func__, cfg->state); 2984 break; 2985 } 2986 2987 return count; 2988 } 2989 2990 static const char *hwq_mode_name[MAX_HWQ_MODE] = { "rr", "tag", "cpu" }; 2991 2992 /** 2993 * hwq_mode_show() - presents the HWQ steering mode for the host 2994 * @dev: Generic device associated with the host. 2995 * @attr: Device attribute representing the HWQ steering mode. 2996 * @buf: Buffer of length PAGE_SIZE to report back the HWQ steering mode 2997 * as a character string. 2998 * 2999 * Return: The size of the ASCII string returned in @buf. 3000 */ 3001 static ssize_t hwq_mode_show(struct device *dev, 3002 struct device_attribute *attr, char *buf) 3003 { 3004 struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev)); 3005 struct afu *afu = cfg->afu; 3006 3007 return scnprintf(buf, PAGE_SIZE, "%s\n", hwq_mode_name[afu->hwq_mode]); 3008 } 3009 3010 /** 3011 * hwq_mode_store() - sets the HWQ steering mode for the host 3012 * @dev: Generic device associated with the host. 3013 * @attr: Device attribute representing the HWQ steering mode. 3014 * @buf: Buffer of length PAGE_SIZE containing the HWQ steering mode 3015 * as a character string. 3016 * @count: Length of data resizing in @buf. 3017 * 3018 * rr = Round-Robin 3019 * tag = Block MQ Tagging 3020 * cpu = CPU Affinity 3021 * 3022 * Return: The size of the ASCII string returned in @buf. 3023 */ 3024 static ssize_t hwq_mode_store(struct device *dev, 3025 struct device_attribute *attr, 3026 const char *buf, size_t count) 3027 { 3028 struct Scsi_Host *shost = class_to_shost(dev); 3029 struct cxlflash_cfg *cfg = shost_priv(shost); 3030 struct device *cfgdev = &cfg->dev->dev; 3031 struct afu *afu = cfg->afu; 3032 int i; 3033 u32 mode = MAX_HWQ_MODE; 3034 3035 for (i = 0; i < MAX_HWQ_MODE; i++) { 3036 if (!strncmp(hwq_mode_name[i], buf, strlen(hwq_mode_name[i]))) { 3037 mode = i; 3038 break; 3039 } 3040 } 3041 3042 if (mode >= MAX_HWQ_MODE) { 3043 dev_info(cfgdev, "Invalid HWQ steering mode.\n"); 3044 return -EINVAL; 3045 } 3046 3047 if ((mode == HWQ_MODE_TAG) && !shost_use_blk_mq(shost)) { 3048 dev_info(cfgdev, "SCSI-MQ is not enabled, use a different " 3049 "HWQ steering mode.\n"); 3050 return -EINVAL; 3051 } 3052 3053 afu->hwq_mode = mode; 3054 3055 return count; 3056 } 3057 3058 /** 3059 * mode_show() - presents the current mode of the device 3060 * @dev: Generic device associated with the device. 3061 * @attr: Device attribute representing the device mode. 3062 * @buf: Buffer of length PAGE_SIZE to report back the dev mode in ASCII. 3063 * 3064 * Return: The size of the ASCII string returned in @buf. 3065 */ 3066 static ssize_t mode_show(struct device *dev, 3067 struct device_attribute *attr, char *buf) 3068 { 3069 struct scsi_device *sdev = to_scsi_device(dev); 3070 3071 return scnprintf(buf, PAGE_SIZE, "%s\n", 3072 sdev->hostdata ? "superpipe" : "legacy"); 3073 } 3074 3075 /* 3076 * Host attributes 3077 */ 3078 static DEVICE_ATTR_RO(port0); 3079 static DEVICE_ATTR_RO(port1); 3080 static DEVICE_ATTR_RO(port2); 3081 static DEVICE_ATTR_RO(port3); 3082 static DEVICE_ATTR_RW(lun_mode); 3083 static DEVICE_ATTR_RO(ioctl_version); 3084 static DEVICE_ATTR_RO(port0_lun_table); 3085 static DEVICE_ATTR_RO(port1_lun_table); 3086 static DEVICE_ATTR_RO(port2_lun_table); 3087 static DEVICE_ATTR_RO(port3_lun_table); 3088 static DEVICE_ATTR_RW(irqpoll_weight); 3089 static DEVICE_ATTR_RW(num_hwqs); 3090 static DEVICE_ATTR_RW(hwq_mode); 3091 3092 static struct device_attribute *cxlflash_host_attrs[] = { 3093 &dev_attr_port0, 3094 &dev_attr_port1, 3095 &dev_attr_port2, 3096 &dev_attr_port3, 3097 &dev_attr_lun_mode, 3098 &dev_attr_ioctl_version, 3099 &dev_attr_port0_lun_table, 3100 &dev_attr_port1_lun_table, 3101 &dev_attr_port2_lun_table, 3102 &dev_attr_port3_lun_table, 3103 &dev_attr_irqpoll_weight, 3104 &dev_attr_num_hwqs, 3105 &dev_attr_hwq_mode, 3106 NULL 3107 }; 3108 3109 /* 3110 * Device attributes 3111 */ 3112 static DEVICE_ATTR_RO(mode); 3113 3114 static struct device_attribute *cxlflash_dev_attrs[] = { 3115 &dev_attr_mode, 3116 NULL 3117 }; 3118 3119 /* 3120 * Host template 3121 */ 3122 static struct scsi_host_template driver_template = { 3123 .module = THIS_MODULE, 3124 .name = CXLFLASH_ADAPTER_NAME, 3125 .info = cxlflash_driver_info, 3126 .ioctl = cxlflash_ioctl, 3127 .proc_name = CXLFLASH_NAME, 3128 .queuecommand = cxlflash_queuecommand, 3129 .eh_abort_handler = cxlflash_eh_abort_handler, 3130 .eh_device_reset_handler = cxlflash_eh_device_reset_handler, 3131 .eh_host_reset_handler = cxlflash_eh_host_reset_handler, 3132 .change_queue_depth = cxlflash_change_queue_depth, 3133 .cmd_per_lun = CXLFLASH_MAX_CMDS_PER_LUN, 3134 .can_queue = CXLFLASH_MAX_CMDS, 3135 .cmd_size = sizeof(struct afu_cmd) + __alignof__(struct afu_cmd) - 1, 3136 .this_id = -1, 3137 .sg_tablesize = 1, /* No scatter gather support */ 3138 .max_sectors = CXLFLASH_MAX_SECTORS, 3139 .use_clustering = ENABLE_CLUSTERING, 3140 .shost_attrs = cxlflash_host_attrs, 3141 .sdev_attrs = cxlflash_dev_attrs, 3142 }; 3143 3144 /* 3145 * Device dependent values 3146 */ 3147 static struct dev_dependent_vals dev_corsa_vals = { CXLFLASH_MAX_SECTORS, 3148 0ULL }; 3149 static struct dev_dependent_vals dev_flash_gt_vals = { CXLFLASH_MAX_SECTORS, 3150 CXLFLASH_NOTIFY_SHUTDOWN }; 3151 static struct dev_dependent_vals dev_briard_vals = { CXLFLASH_MAX_SECTORS, 3152 CXLFLASH_NOTIFY_SHUTDOWN }; 3153 3154 /* 3155 * PCI device binding table 3156 */ 3157 static struct pci_device_id cxlflash_pci_table[] = { 3158 {PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CORSA, 3159 PCI_ANY_ID, PCI_ANY_ID, 0, 0, (kernel_ulong_t)&dev_corsa_vals}, 3160 {PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_FLASH_GT, 3161 PCI_ANY_ID, PCI_ANY_ID, 0, 0, (kernel_ulong_t)&dev_flash_gt_vals}, 3162 {PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_BRIARD, 3163 PCI_ANY_ID, PCI_ANY_ID, 0, 0, (kernel_ulong_t)&dev_briard_vals}, 3164 {} 3165 }; 3166 3167 MODULE_DEVICE_TABLE(pci, cxlflash_pci_table); 3168 3169 /** 3170 * cxlflash_worker_thread() - work thread handler for the AFU 3171 * @work: Work structure contained within cxlflash associated with host. 3172 * 3173 * Handles the following events: 3174 * - Link reset which cannot be performed on interrupt context due to 3175 * blocking up to a few seconds 3176 * - Rescan the host 3177 */ 3178 static void cxlflash_worker_thread(struct work_struct *work) 3179 { 3180 struct cxlflash_cfg *cfg = container_of(work, struct cxlflash_cfg, 3181 work_q); 3182 struct afu *afu = cfg->afu; 3183 struct device *dev = &cfg->dev->dev; 3184 __be64 __iomem *fc_port_regs; 3185 int port; 3186 ulong lock_flags; 3187 3188 /* Avoid MMIO if the device has failed */ 3189 3190 if (cfg->state != STATE_NORMAL) 3191 return; 3192 3193 spin_lock_irqsave(cfg->host->host_lock, lock_flags); 3194 3195 if (cfg->lr_state == LINK_RESET_REQUIRED) { 3196 port = cfg->lr_port; 3197 if (port < 0) 3198 dev_err(dev, "%s: invalid port index %d\n", 3199 __func__, port); 3200 else { 3201 spin_unlock_irqrestore(cfg->host->host_lock, 3202 lock_flags); 3203 3204 /* The reset can block... */ 3205 fc_port_regs = get_fc_port_regs(cfg, port); 3206 afu_link_reset(afu, port, fc_port_regs); 3207 spin_lock_irqsave(cfg->host->host_lock, lock_flags); 3208 } 3209 3210 cfg->lr_state = LINK_RESET_COMPLETE; 3211 } 3212 3213 spin_unlock_irqrestore(cfg->host->host_lock, lock_flags); 3214 3215 if (atomic_dec_if_positive(&cfg->scan_host_needed) >= 0) 3216 scsi_scan_host(cfg->host); 3217 } 3218 3219 /** 3220 * cxlflash_chr_open() - character device open handler 3221 * @inode: Device inode associated with this character device. 3222 * @file: File pointer for this device. 3223 * 3224 * Only users with admin privileges are allowed to open the character device. 3225 * 3226 * Return: 0 on success, -errno on failure 3227 */ 3228 static int cxlflash_chr_open(struct inode *inode, struct file *file) 3229 { 3230 struct cxlflash_cfg *cfg; 3231 3232 if (!capable(CAP_SYS_ADMIN)) 3233 return -EACCES; 3234 3235 cfg = container_of(inode->i_cdev, struct cxlflash_cfg, cdev); 3236 file->private_data = cfg; 3237 3238 return 0; 3239 } 3240 3241 /** 3242 * decode_hioctl() - translates encoded host ioctl to easily identifiable string 3243 * @cmd: The host ioctl command to decode. 3244 * 3245 * Return: A string identifying the decoded host ioctl. 3246 */ 3247 static char *decode_hioctl(int cmd) 3248 { 3249 switch (cmd) { 3250 case HT_CXLFLASH_LUN_PROVISION: 3251 return __stringify_1(HT_CXLFLASH_LUN_PROVISION); 3252 } 3253 3254 return "UNKNOWN"; 3255 } 3256 3257 /** 3258 * cxlflash_lun_provision() - host LUN provisioning handler 3259 * @cfg: Internal structure associated with the host. 3260 * @arg: Kernel copy of userspace ioctl data structure. 3261 * 3262 * Return: 0 on success, -errno on failure 3263 */ 3264 static int cxlflash_lun_provision(struct cxlflash_cfg *cfg, 3265 struct ht_cxlflash_lun_provision *lunprov) 3266 { 3267 struct afu *afu = cfg->afu; 3268 struct device *dev = &cfg->dev->dev; 3269 struct sisl_ioarcb rcb; 3270 struct sisl_ioasa asa; 3271 __be64 __iomem *fc_port_regs; 3272 u16 port = lunprov->port; 3273 u16 scmd = lunprov->hdr.subcmd; 3274 u16 type; 3275 u64 reg; 3276 u64 size; 3277 u64 lun_id; 3278 int rc = 0; 3279 3280 if (!afu_is_lun_provision(afu)) { 3281 rc = -ENOTSUPP; 3282 goto out; 3283 } 3284 3285 if (port >= cfg->num_fc_ports) { 3286 rc = -EINVAL; 3287 goto out; 3288 } 3289 3290 switch (scmd) { 3291 case HT_CXLFLASH_LUN_PROVISION_SUBCMD_CREATE_LUN: 3292 type = SISL_AFU_LUN_PROVISION_CREATE; 3293 size = lunprov->size; 3294 lun_id = 0; 3295 break; 3296 case HT_CXLFLASH_LUN_PROVISION_SUBCMD_DELETE_LUN: 3297 type = SISL_AFU_LUN_PROVISION_DELETE; 3298 size = 0; 3299 lun_id = lunprov->lun_id; 3300 break; 3301 case HT_CXLFLASH_LUN_PROVISION_SUBCMD_QUERY_PORT: 3302 fc_port_regs = get_fc_port_regs(cfg, port); 3303 3304 reg = readq_be(&fc_port_regs[FC_MAX_NUM_LUNS / 8]); 3305 lunprov->max_num_luns = reg; 3306 reg = readq_be(&fc_port_regs[FC_CUR_NUM_LUNS / 8]); 3307 lunprov->cur_num_luns = reg; 3308 reg = readq_be(&fc_port_regs[FC_MAX_CAP_PORT / 8]); 3309 lunprov->max_cap_port = reg; 3310 reg = readq_be(&fc_port_regs[FC_CUR_CAP_PORT / 8]); 3311 lunprov->cur_cap_port = reg; 3312 3313 goto out; 3314 default: 3315 rc = -EINVAL; 3316 goto out; 3317 } 3318 3319 memset(&rcb, 0, sizeof(rcb)); 3320 memset(&asa, 0, sizeof(asa)); 3321 rcb.req_flags = SISL_REQ_FLAGS_AFU_CMD; 3322 rcb.lun_id = lun_id; 3323 rcb.msi = SISL_MSI_RRQ_UPDATED; 3324 rcb.timeout = MC_LUN_PROV_TIMEOUT; 3325 rcb.ioasa = &asa; 3326 3327 rcb.cdb[0] = SISL_AFU_CMD_LUN_PROVISION; 3328 rcb.cdb[1] = type; 3329 rcb.cdb[2] = port; 3330 put_unaligned_be64(size, &rcb.cdb[8]); 3331 3332 rc = send_afu_cmd(afu, &rcb); 3333 if (rc) { 3334 dev_err(dev, "%s: send_afu_cmd failed rc=%d asc=%08x afux=%x\n", 3335 __func__, rc, asa.ioasc, asa.afu_extra); 3336 goto out; 3337 } 3338 3339 if (scmd == HT_CXLFLASH_LUN_PROVISION_SUBCMD_CREATE_LUN) { 3340 lunprov->lun_id = (u64)asa.lunid_hi << 32 | asa.lunid_lo; 3341 memcpy(lunprov->wwid, asa.wwid, sizeof(lunprov->wwid)); 3342 } 3343 out: 3344 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 3345 return rc; 3346 } 3347 3348 /** 3349 * cxlflash_afu_debug() - host AFU debug handler 3350 * @cfg: Internal structure associated with the host. 3351 * @arg: Kernel copy of userspace ioctl data structure. 3352 * 3353 * For debug requests requiring a data buffer, always provide an aligned 3354 * (cache line) buffer to the AFU to appease any alignment requirements. 3355 * 3356 * Return: 0 on success, -errno on failure 3357 */ 3358 static int cxlflash_afu_debug(struct cxlflash_cfg *cfg, 3359 struct ht_cxlflash_afu_debug *afu_dbg) 3360 { 3361 struct afu *afu = cfg->afu; 3362 struct device *dev = &cfg->dev->dev; 3363 struct sisl_ioarcb rcb; 3364 struct sisl_ioasa asa; 3365 char *buf = NULL; 3366 char *kbuf = NULL; 3367 void __user *ubuf = (__force void __user *)afu_dbg->data_ea; 3368 u16 req_flags = SISL_REQ_FLAGS_AFU_CMD; 3369 u32 ulen = afu_dbg->data_len; 3370 bool is_write = afu_dbg->hdr.flags & HT_CXLFLASH_HOST_WRITE; 3371 int rc = 0; 3372 3373 if (!afu_is_afu_debug(afu)) { 3374 rc = -ENOTSUPP; 3375 goto out; 3376 } 3377 3378 if (ulen) { 3379 req_flags |= SISL_REQ_FLAGS_SUP_UNDERRUN; 3380 3381 if (ulen > HT_CXLFLASH_AFU_DEBUG_MAX_DATA_LEN) { 3382 rc = -EINVAL; 3383 goto out; 3384 } 3385 3386 if (unlikely(!access_ok(is_write ? VERIFY_READ : VERIFY_WRITE, 3387 ubuf, ulen))) { 3388 rc = -EFAULT; 3389 goto out; 3390 } 3391 3392 buf = kmalloc(ulen + cache_line_size() - 1, GFP_KERNEL); 3393 if (unlikely(!buf)) { 3394 rc = -ENOMEM; 3395 goto out; 3396 } 3397 3398 kbuf = PTR_ALIGN(buf, cache_line_size()); 3399 3400 if (is_write) { 3401 req_flags |= SISL_REQ_FLAGS_HOST_WRITE; 3402 3403 if (copy_from_user(kbuf, ubuf, ulen)) { 3404 rc = -EFAULT; 3405 goto out; 3406 } 3407 } 3408 } 3409 3410 memset(&rcb, 0, sizeof(rcb)); 3411 memset(&asa, 0, sizeof(asa)); 3412 3413 rcb.req_flags = req_flags; 3414 rcb.msi = SISL_MSI_RRQ_UPDATED; 3415 rcb.timeout = MC_AFU_DEBUG_TIMEOUT; 3416 rcb.ioasa = &asa; 3417 3418 if (ulen) { 3419 rcb.data_len = ulen; 3420 rcb.data_ea = (uintptr_t)kbuf; 3421 } 3422 3423 rcb.cdb[0] = SISL_AFU_CMD_DEBUG; 3424 memcpy(&rcb.cdb[4], afu_dbg->afu_subcmd, 3425 HT_CXLFLASH_AFU_DEBUG_SUBCMD_LEN); 3426 3427 rc = send_afu_cmd(afu, &rcb); 3428 if (rc) { 3429 dev_err(dev, "%s: send_afu_cmd failed rc=%d asc=%08x afux=%x\n", 3430 __func__, rc, asa.ioasc, asa.afu_extra); 3431 goto out; 3432 } 3433 3434 if (ulen && !is_write) { 3435 if (copy_to_user(ubuf, kbuf, ulen)) 3436 rc = -EFAULT; 3437 } 3438 out: 3439 kfree(buf); 3440 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 3441 return rc; 3442 } 3443 3444 /** 3445 * cxlflash_chr_ioctl() - character device IOCTL handler 3446 * @file: File pointer for this device. 3447 * @cmd: IOCTL command. 3448 * @arg: Userspace ioctl data structure. 3449 * 3450 * A read/write semaphore is used to implement a 'drain' of currently 3451 * running ioctls. The read semaphore is taken at the beginning of each 3452 * ioctl thread and released upon concluding execution. Additionally the 3453 * semaphore should be released and then reacquired in any ioctl execution 3454 * path which will wait for an event to occur that is outside the scope of 3455 * the ioctl (i.e. an adapter reset). To drain the ioctls currently running, 3456 * a thread simply needs to acquire the write semaphore. 3457 * 3458 * Return: 0 on success, -errno on failure 3459 */ 3460 static long cxlflash_chr_ioctl(struct file *file, unsigned int cmd, 3461 unsigned long arg) 3462 { 3463 typedef int (*hioctl) (struct cxlflash_cfg *, void *); 3464 3465 struct cxlflash_cfg *cfg = file->private_data; 3466 struct device *dev = &cfg->dev->dev; 3467 char buf[sizeof(union cxlflash_ht_ioctls)]; 3468 void __user *uarg = (void __user *)arg; 3469 struct ht_cxlflash_hdr *hdr; 3470 size_t size = 0; 3471 bool known_ioctl = false; 3472 int idx = 0; 3473 int rc = 0; 3474 hioctl do_ioctl = NULL; 3475 3476 static const struct { 3477 size_t size; 3478 hioctl ioctl; 3479 } ioctl_tbl[] = { /* NOTE: order matters here */ 3480 { sizeof(struct ht_cxlflash_lun_provision), 3481 (hioctl)cxlflash_lun_provision }, 3482 { sizeof(struct ht_cxlflash_afu_debug), 3483 (hioctl)cxlflash_afu_debug }, 3484 }; 3485 3486 /* Hold read semaphore so we can drain if needed */ 3487 down_read(&cfg->ioctl_rwsem); 3488 3489 dev_dbg(dev, "%s: cmd=%u idx=%d tbl_size=%lu\n", 3490 __func__, cmd, idx, sizeof(ioctl_tbl)); 3491 3492 switch (cmd) { 3493 case HT_CXLFLASH_LUN_PROVISION: 3494 case HT_CXLFLASH_AFU_DEBUG: 3495 known_ioctl = true; 3496 idx = _IOC_NR(HT_CXLFLASH_LUN_PROVISION) - _IOC_NR(cmd); 3497 size = ioctl_tbl[idx].size; 3498 do_ioctl = ioctl_tbl[idx].ioctl; 3499 3500 if (likely(do_ioctl)) 3501 break; 3502 3503 /* fall through */ 3504 default: 3505 rc = -EINVAL; 3506 goto out; 3507 } 3508 3509 if (unlikely(copy_from_user(&buf, uarg, size))) { 3510 dev_err(dev, "%s: copy_from_user() fail " 3511 "size=%lu cmd=%d (%s) uarg=%p\n", 3512 __func__, size, cmd, decode_hioctl(cmd), uarg); 3513 rc = -EFAULT; 3514 goto out; 3515 } 3516 3517 hdr = (struct ht_cxlflash_hdr *)&buf; 3518 if (hdr->version != HT_CXLFLASH_VERSION_0) { 3519 dev_dbg(dev, "%s: Version %u not supported for %s\n", 3520 __func__, hdr->version, decode_hioctl(cmd)); 3521 rc = -EINVAL; 3522 goto out; 3523 } 3524 3525 if (hdr->rsvd[0] || hdr->rsvd[1] || hdr->return_flags) { 3526 dev_dbg(dev, "%s: Reserved/rflags populated\n", __func__); 3527 rc = -EINVAL; 3528 goto out; 3529 } 3530 3531 rc = do_ioctl(cfg, (void *)&buf); 3532 if (likely(!rc)) 3533 if (unlikely(copy_to_user(uarg, &buf, size))) { 3534 dev_err(dev, "%s: copy_to_user() fail " 3535 "size=%lu cmd=%d (%s) uarg=%p\n", 3536 __func__, size, cmd, decode_hioctl(cmd), uarg); 3537 rc = -EFAULT; 3538 } 3539 3540 /* fall through to exit */ 3541 3542 out: 3543 up_read(&cfg->ioctl_rwsem); 3544 if (unlikely(rc && known_ioctl)) 3545 dev_err(dev, "%s: ioctl %s (%08X) returned rc=%d\n", 3546 __func__, decode_hioctl(cmd), cmd, rc); 3547 else 3548 dev_dbg(dev, "%s: ioctl %s (%08X) returned rc=%d\n", 3549 __func__, decode_hioctl(cmd), cmd, rc); 3550 return rc; 3551 } 3552 3553 /* 3554 * Character device file operations 3555 */ 3556 static const struct file_operations cxlflash_chr_fops = { 3557 .owner = THIS_MODULE, 3558 .open = cxlflash_chr_open, 3559 .unlocked_ioctl = cxlflash_chr_ioctl, 3560 .compat_ioctl = cxlflash_chr_ioctl, 3561 }; 3562 3563 /** 3564 * init_chrdev() - initialize the character device for the host 3565 * @cfg: Internal structure associated with the host. 3566 * 3567 * Return: 0 on success, -errno on failure 3568 */ 3569 static int init_chrdev(struct cxlflash_cfg *cfg) 3570 { 3571 struct device *dev = &cfg->dev->dev; 3572 struct device *char_dev; 3573 dev_t devno; 3574 int minor; 3575 int rc = 0; 3576 3577 minor = cxlflash_get_minor(); 3578 if (unlikely(minor < 0)) { 3579 dev_err(dev, "%s: Exhausted allowed adapters\n", __func__); 3580 rc = -ENOSPC; 3581 goto out; 3582 } 3583 3584 devno = MKDEV(cxlflash_major, minor); 3585 cdev_init(&cfg->cdev, &cxlflash_chr_fops); 3586 3587 rc = cdev_add(&cfg->cdev, devno, 1); 3588 if (rc) { 3589 dev_err(dev, "%s: cdev_add failed rc=%d\n", __func__, rc); 3590 goto err1; 3591 } 3592 3593 char_dev = device_create(cxlflash_class, NULL, devno, 3594 NULL, "cxlflash%d", minor); 3595 if (IS_ERR(char_dev)) { 3596 rc = PTR_ERR(char_dev); 3597 dev_err(dev, "%s: device_create failed rc=%d\n", 3598 __func__, rc); 3599 goto err2; 3600 } 3601 3602 cfg->chardev = char_dev; 3603 out: 3604 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 3605 return rc; 3606 err2: 3607 cdev_del(&cfg->cdev); 3608 err1: 3609 cxlflash_put_minor(minor); 3610 goto out; 3611 } 3612 3613 /** 3614 * cxlflash_probe() - PCI entry point to add host 3615 * @pdev: PCI device associated with the host. 3616 * @dev_id: PCI device id associated with device. 3617 * 3618 * The device will initially start out in a 'probing' state and 3619 * transition to the 'normal' state at the end of a successful 3620 * probe. Should an EEH event occur during probe, the notification 3621 * thread (error_detected()) will wait until the probe handler 3622 * is nearly complete. At that time, the device will be moved to 3623 * a 'probed' state and the EEH thread woken up to drive the slot 3624 * reset and recovery (device moves to 'normal' state). Meanwhile, 3625 * the probe will be allowed to exit successfully. 3626 * 3627 * Return: 0 on success, -errno on failure 3628 */ 3629 static int cxlflash_probe(struct pci_dev *pdev, 3630 const struct pci_device_id *dev_id) 3631 { 3632 struct Scsi_Host *host; 3633 struct cxlflash_cfg *cfg = NULL; 3634 struct device *dev = &pdev->dev; 3635 struct dev_dependent_vals *ddv; 3636 int rc = 0; 3637 int k; 3638 3639 dev_dbg(&pdev->dev, "%s: Found CXLFLASH with IRQ: %d\n", 3640 __func__, pdev->irq); 3641 3642 ddv = (struct dev_dependent_vals *)dev_id->driver_data; 3643 driver_template.max_sectors = ddv->max_sectors; 3644 3645 host = scsi_host_alloc(&driver_template, sizeof(struct cxlflash_cfg)); 3646 if (!host) { 3647 dev_err(dev, "%s: scsi_host_alloc failed\n", __func__); 3648 rc = -ENOMEM; 3649 goto out; 3650 } 3651 3652 host->max_id = CXLFLASH_MAX_NUM_TARGETS_PER_BUS; 3653 host->max_lun = CXLFLASH_MAX_NUM_LUNS_PER_TARGET; 3654 host->unique_id = host->host_no; 3655 host->max_cmd_len = CXLFLASH_MAX_CDB_LEN; 3656 3657 cfg = shost_priv(host); 3658 cfg->host = host; 3659 rc = alloc_mem(cfg); 3660 if (rc) { 3661 dev_err(dev, "%s: alloc_mem failed\n", __func__); 3662 rc = -ENOMEM; 3663 scsi_host_put(cfg->host); 3664 goto out; 3665 } 3666 3667 cfg->init_state = INIT_STATE_NONE; 3668 cfg->dev = pdev; 3669 cfg->cxl_fops = cxlflash_cxl_fops; 3670 3671 /* 3672 * Promoted LUNs move to the top of the LUN table. The rest stay on 3673 * the bottom half. The bottom half grows from the end (index = 255), 3674 * whereas the top half grows from the beginning (index = 0). 3675 * 3676 * Initialize the last LUN index for all possible ports. 3677 */ 3678 cfg->promote_lun_index = 0; 3679 3680 for (k = 0; k < MAX_FC_PORTS; k++) 3681 cfg->last_lun_index[k] = CXLFLASH_NUM_VLUNS/2 - 1; 3682 3683 cfg->dev_id = (struct pci_device_id *)dev_id; 3684 3685 init_waitqueue_head(&cfg->tmf_waitq); 3686 init_waitqueue_head(&cfg->reset_waitq); 3687 3688 INIT_WORK(&cfg->work_q, cxlflash_worker_thread); 3689 cfg->lr_state = LINK_RESET_INVALID; 3690 cfg->lr_port = -1; 3691 spin_lock_init(&cfg->tmf_slock); 3692 mutex_init(&cfg->ctx_tbl_list_mutex); 3693 mutex_init(&cfg->ctx_recovery_mutex); 3694 init_rwsem(&cfg->ioctl_rwsem); 3695 INIT_LIST_HEAD(&cfg->ctx_err_recovery); 3696 INIT_LIST_HEAD(&cfg->lluns); 3697 3698 pci_set_drvdata(pdev, cfg); 3699 3700 cfg->cxl_afu = cxl_pci_to_afu(pdev); 3701 3702 rc = init_pci(cfg); 3703 if (rc) { 3704 dev_err(dev, "%s: init_pci failed rc=%d\n", __func__, rc); 3705 goto out_remove; 3706 } 3707 cfg->init_state = INIT_STATE_PCI; 3708 3709 rc = init_afu(cfg); 3710 if (rc && !wq_has_sleeper(&cfg->reset_waitq)) { 3711 dev_err(dev, "%s: init_afu failed rc=%d\n", __func__, rc); 3712 goto out_remove; 3713 } 3714 cfg->init_state = INIT_STATE_AFU; 3715 3716 rc = init_scsi(cfg); 3717 if (rc) { 3718 dev_err(dev, "%s: init_scsi failed rc=%d\n", __func__, rc); 3719 goto out_remove; 3720 } 3721 cfg->init_state = INIT_STATE_SCSI; 3722 3723 rc = init_chrdev(cfg); 3724 if (rc) { 3725 dev_err(dev, "%s: init_chrdev failed rc=%d\n", __func__, rc); 3726 goto out_remove; 3727 } 3728 cfg->init_state = INIT_STATE_CDEV; 3729 3730 if (wq_has_sleeper(&cfg->reset_waitq)) { 3731 cfg->state = STATE_PROBED; 3732 wake_up_all(&cfg->reset_waitq); 3733 } else 3734 cfg->state = STATE_NORMAL; 3735 out: 3736 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); 3737 return rc; 3738 3739 out_remove: 3740 cxlflash_remove(pdev); 3741 goto out; 3742 } 3743 3744 /** 3745 * cxlflash_pci_error_detected() - called when a PCI error is detected 3746 * @pdev: PCI device struct. 3747 * @state: PCI channel state. 3748 * 3749 * When an EEH occurs during an active reset, wait until the reset is 3750 * complete and then take action based upon the device state. 3751 * 3752 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT 3753 */ 3754 static pci_ers_result_t cxlflash_pci_error_detected(struct pci_dev *pdev, 3755 pci_channel_state_t state) 3756 { 3757 int rc = 0; 3758 struct cxlflash_cfg *cfg = pci_get_drvdata(pdev); 3759 struct device *dev = &cfg->dev->dev; 3760 3761 dev_dbg(dev, "%s: pdev=%p state=%u\n", __func__, pdev, state); 3762 3763 switch (state) { 3764 case pci_channel_io_frozen: 3765 wait_event(cfg->reset_waitq, cfg->state != STATE_RESET && 3766 cfg->state != STATE_PROBING); 3767 if (cfg->state == STATE_FAILTERM) 3768 return PCI_ERS_RESULT_DISCONNECT; 3769 3770 cfg->state = STATE_RESET; 3771 scsi_block_requests(cfg->host); 3772 drain_ioctls(cfg); 3773 rc = cxlflash_mark_contexts_error(cfg); 3774 if (unlikely(rc)) 3775 dev_err(dev, "%s: Failed to mark user contexts rc=%d\n", 3776 __func__, rc); 3777 term_afu(cfg); 3778 return PCI_ERS_RESULT_NEED_RESET; 3779 case pci_channel_io_perm_failure: 3780 cfg->state = STATE_FAILTERM; 3781 wake_up_all(&cfg->reset_waitq); 3782 scsi_unblock_requests(cfg->host); 3783 return PCI_ERS_RESULT_DISCONNECT; 3784 default: 3785 break; 3786 } 3787 return PCI_ERS_RESULT_NEED_RESET; 3788 } 3789 3790 /** 3791 * cxlflash_pci_slot_reset() - called when PCI slot has been reset 3792 * @pdev: PCI device struct. 3793 * 3794 * This routine is called by the pci error recovery code after the PCI 3795 * slot has been reset, just before we should resume normal operations. 3796 * 3797 * Return: PCI_ERS_RESULT_RECOVERED or PCI_ERS_RESULT_DISCONNECT 3798 */ 3799 static pci_ers_result_t cxlflash_pci_slot_reset(struct pci_dev *pdev) 3800 { 3801 int rc = 0; 3802 struct cxlflash_cfg *cfg = pci_get_drvdata(pdev); 3803 struct device *dev = &cfg->dev->dev; 3804 3805 dev_dbg(dev, "%s: pdev=%p\n", __func__, pdev); 3806 3807 rc = init_afu(cfg); 3808 if (unlikely(rc)) { 3809 dev_err(dev, "%s: EEH recovery failed rc=%d\n", __func__, rc); 3810 return PCI_ERS_RESULT_DISCONNECT; 3811 } 3812 3813 return PCI_ERS_RESULT_RECOVERED; 3814 } 3815 3816 /** 3817 * cxlflash_pci_resume() - called when normal operation can resume 3818 * @pdev: PCI device struct 3819 */ 3820 static void cxlflash_pci_resume(struct pci_dev *pdev) 3821 { 3822 struct cxlflash_cfg *cfg = pci_get_drvdata(pdev); 3823 struct device *dev = &cfg->dev->dev; 3824 3825 dev_dbg(dev, "%s: pdev=%p\n", __func__, pdev); 3826 3827 cfg->state = STATE_NORMAL; 3828 wake_up_all(&cfg->reset_waitq); 3829 scsi_unblock_requests(cfg->host); 3830 } 3831 3832 /** 3833 * cxlflash_devnode() - provides devtmpfs for devices in the cxlflash class 3834 * @dev: Character device. 3835 * @mode: Mode that can be used to verify access. 3836 * 3837 * Return: Allocated string describing the devtmpfs structure. 3838 */ 3839 static char *cxlflash_devnode(struct device *dev, umode_t *mode) 3840 { 3841 return kasprintf(GFP_KERNEL, "cxlflash/%s", dev_name(dev)); 3842 } 3843 3844 /** 3845 * cxlflash_class_init() - create character device class 3846 * 3847 * Return: 0 on success, -errno on failure 3848 */ 3849 static int cxlflash_class_init(void) 3850 { 3851 dev_t devno; 3852 int rc = 0; 3853 3854 rc = alloc_chrdev_region(&devno, 0, CXLFLASH_MAX_ADAPTERS, "cxlflash"); 3855 if (unlikely(rc)) { 3856 pr_err("%s: alloc_chrdev_region failed rc=%d\n", __func__, rc); 3857 goto out; 3858 } 3859 3860 cxlflash_major = MAJOR(devno); 3861 3862 cxlflash_class = class_create(THIS_MODULE, "cxlflash"); 3863 if (IS_ERR(cxlflash_class)) { 3864 rc = PTR_ERR(cxlflash_class); 3865 pr_err("%s: class_create failed rc=%d\n", __func__, rc); 3866 goto err; 3867 } 3868 3869 cxlflash_class->devnode = cxlflash_devnode; 3870 out: 3871 pr_debug("%s: returning rc=%d\n", __func__, rc); 3872 return rc; 3873 err: 3874 unregister_chrdev_region(devno, CXLFLASH_MAX_ADAPTERS); 3875 goto out; 3876 } 3877 3878 /** 3879 * cxlflash_class_exit() - destroy character device class 3880 */ 3881 static void cxlflash_class_exit(void) 3882 { 3883 dev_t devno = MKDEV(cxlflash_major, 0); 3884 3885 class_destroy(cxlflash_class); 3886 unregister_chrdev_region(devno, CXLFLASH_MAX_ADAPTERS); 3887 } 3888 3889 static const struct pci_error_handlers cxlflash_err_handler = { 3890 .error_detected = cxlflash_pci_error_detected, 3891 .slot_reset = cxlflash_pci_slot_reset, 3892 .resume = cxlflash_pci_resume, 3893 }; 3894 3895 /* 3896 * PCI device structure 3897 */ 3898 static struct pci_driver cxlflash_driver = { 3899 .name = CXLFLASH_NAME, 3900 .id_table = cxlflash_pci_table, 3901 .probe = cxlflash_probe, 3902 .remove = cxlflash_remove, 3903 .shutdown = cxlflash_remove, 3904 .err_handler = &cxlflash_err_handler, 3905 }; 3906 3907 /** 3908 * init_cxlflash() - module entry point 3909 * 3910 * Return: 0 on success, -errno on failure 3911 */ 3912 static int __init init_cxlflash(void) 3913 { 3914 int rc; 3915 3916 check_sizes(); 3917 cxlflash_list_init(); 3918 rc = cxlflash_class_init(); 3919 if (unlikely(rc)) 3920 goto out; 3921 3922 rc = pci_register_driver(&cxlflash_driver); 3923 if (unlikely(rc)) 3924 goto err; 3925 out: 3926 pr_debug("%s: returning rc=%d\n", __func__, rc); 3927 return rc; 3928 err: 3929 cxlflash_class_exit(); 3930 goto out; 3931 } 3932 3933 /** 3934 * exit_cxlflash() - module exit point 3935 */ 3936 static void __exit exit_cxlflash(void) 3937 { 3938 cxlflash_term_global_luns(); 3939 cxlflash_free_errpage(); 3940 3941 pci_unregister_driver(&cxlflash_driver); 3942 cxlflash_class_exit(); 3943 } 3944 3945 module_init(init_cxlflash); 3946 module_exit(exit_cxlflash); 3947