1 /* 2 * Copyright (C) 1999 Eric Youngdale 3 * Copyright (C) 2014 Christoph Hellwig 4 * 5 * SCSI queueing library. 6 * Initial versions: Eric Youngdale (eric@andante.org). 7 * Based upon conversations with large numbers 8 * of people at Linux Expo. 9 */ 10 11 #include <linux/bio.h> 12 #include <linux/bitops.h> 13 #include <linux/blkdev.h> 14 #include <linux/completion.h> 15 #include <linux/kernel.h> 16 #include <linux/export.h> 17 #include <linux/mempool.h> 18 #include <linux/slab.h> 19 #include <linux/init.h> 20 #include <linux/pci.h> 21 #include <linux/delay.h> 22 #include <linux/hardirq.h> 23 #include <linux/scatterlist.h> 24 #include <linux/blk-mq.h> 25 #include <linux/ratelimit.h> 26 27 #include <scsi/scsi.h> 28 #include <scsi/scsi_cmnd.h> 29 #include <scsi/scsi_dbg.h> 30 #include <scsi/scsi_device.h> 31 #include <scsi/scsi_driver.h> 32 #include <scsi/scsi_eh.h> 33 #include <scsi/scsi_host.h> 34 #include <scsi/scsi_dh.h> 35 36 #include <trace/events/scsi.h> 37 38 #include "scsi_priv.h" 39 #include "scsi_logging.h" 40 41 42 #define SG_MEMPOOL_NR ARRAY_SIZE(scsi_sg_pools) 43 #define SG_MEMPOOL_SIZE 2 44 45 struct scsi_host_sg_pool { 46 size_t size; 47 char *name; 48 struct kmem_cache *slab; 49 mempool_t *pool; 50 }; 51 52 #define SP(x) { .size = x, "sgpool-" __stringify(x) } 53 #if (SCSI_MAX_SG_SEGMENTS < 32) 54 #error SCSI_MAX_SG_SEGMENTS is too small (must be 32 or greater) 55 #endif 56 static struct scsi_host_sg_pool scsi_sg_pools[] = { 57 SP(8), 58 SP(16), 59 #if (SCSI_MAX_SG_SEGMENTS > 32) 60 SP(32), 61 #if (SCSI_MAX_SG_SEGMENTS > 64) 62 SP(64), 63 #if (SCSI_MAX_SG_SEGMENTS > 128) 64 SP(128), 65 #if (SCSI_MAX_SG_SEGMENTS > 256) 66 #error SCSI_MAX_SG_SEGMENTS is too large (256 MAX) 67 #endif 68 #endif 69 #endif 70 #endif 71 SP(SCSI_MAX_SG_SEGMENTS) 72 }; 73 #undef SP 74 75 struct kmem_cache *scsi_sdb_cache; 76 77 /* 78 * When to reinvoke queueing after a resource shortage. It's 3 msecs to 79 * not change behaviour from the previous unplug mechanism, experimentation 80 * may prove this needs changing. 81 */ 82 #define SCSI_QUEUE_DELAY 3 83 84 static void 85 scsi_set_blocked(struct scsi_cmnd *cmd, int reason) 86 { 87 struct Scsi_Host *host = cmd->device->host; 88 struct scsi_device *device = cmd->device; 89 struct scsi_target *starget = scsi_target(device); 90 91 /* 92 * Set the appropriate busy bit for the device/host. 93 * 94 * If the host/device isn't busy, assume that something actually 95 * completed, and that we should be able to queue a command now. 96 * 97 * Note that the prior mid-layer assumption that any host could 98 * always queue at least one command is now broken. The mid-layer 99 * will implement a user specifiable stall (see 100 * scsi_host.max_host_blocked and scsi_device.max_device_blocked) 101 * if a command is requeued with no other commands outstanding 102 * either for the device or for the host. 103 */ 104 switch (reason) { 105 case SCSI_MLQUEUE_HOST_BUSY: 106 atomic_set(&host->host_blocked, host->max_host_blocked); 107 break; 108 case SCSI_MLQUEUE_DEVICE_BUSY: 109 case SCSI_MLQUEUE_EH_RETRY: 110 atomic_set(&device->device_blocked, 111 device->max_device_blocked); 112 break; 113 case SCSI_MLQUEUE_TARGET_BUSY: 114 atomic_set(&starget->target_blocked, 115 starget->max_target_blocked); 116 break; 117 } 118 } 119 120 static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd) 121 { 122 struct scsi_device *sdev = cmd->device; 123 struct request_queue *q = cmd->request->q; 124 125 blk_mq_requeue_request(cmd->request); 126 blk_mq_kick_requeue_list(q); 127 put_device(&sdev->sdev_gendev); 128 } 129 130 /** 131 * __scsi_queue_insert - private queue insertion 132 * @cmd: The SCSI command being requeued 133 * @reason: The reason for the requeue 134 * @unbusy: Whether the queue should be unbusied 135 * 136 * This is a private queue insertion. The public interface 137 * scsi_queue_insert() always assumes the queue should be unbusied 138 * because it's always called before the completion. This function is 139 * for a requeue after completion, which should only occur in this 140 * file. 141 */ 142 static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy) 143 { 144 struct scsi_device *device = cmd->device; 145 struct request_queue *q = device->request_queue; 146 unsigned long flags; 147 148 SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd, 149 "Inserting command %p into mlqueue\n", cmd)); 150 151 scsi_set_blocked(cmd, reason); 152 153 /* 154 * Decrement the counters, since these commands are no longer 155 * active on the host/device. 156 */ 157 if (unbusy) 158 scsi_device_unbusy(device); 159 160 /* 161 * Requeue this command. It will go before all other commands 162 * that are already in the queue. Schedule requeue work under 163 * lock such that the kblockd_schedule_work() call happens 164 * before blk_cleanup_queue() finishes. 165 */ 166 cmd->result = 0; 167 if (q->mq_ops) { 168 scsi_mq_requeue_cmd(cmd); 169 return; 170 } 171 spin_lock_irqsave(q->queue_lock, flags); 172 blk_requeue_request(q, cmd->request); 173 kblockd_schedule_work(&device->requeue_work); 174 spin_unlock_irqrestore(q->queue_lock, flags); 175 } 176 177 /* 178 * Function: scsi_queue_insert() 179 * 180 * Purpose: Insert a command in the midlevel queue. 181 * 182 * Arguments: cmd - command that we are adding to queue. 183 * reason - why we are inserting command to queue. 184 * 185 * Lock status: Assumed that lock is not held upon entry. 186 * 187 * Returns: Nothing. 188 * 189 * Notes: We do this for one of two cases. Either the host is busy 190 * and it cannot accept any more commands for the time being, 191 * or the device returned QUEUE_FULL and can accept no more 192 * commands. 193 * Notes: This could be called either from an interrupt context or a 194 * normal process context. 195 */ 196 void scsi_queue_insert(struct scsi_cmnd *cmd, int reason) 197 { 198 __scsi_queue_insert(cmd, reason, 1); 199 } 200 /** 201 * scsi_execute - insert request and wait for the result 202 * @sdev: scsi device 203 * @cmd: scsi command 204 * @data_direction: data direction 205 * @buffer: data buffer 206 * @bufflen: len of buffer 207 * @sense: optional sense buffer 208 * @timeout: request timeout in seconds 209 * @retries: number of times to retry request 210 * @flags: or into request flags; 211 * @resid: optional residual length 212 * 213 * returns the req->errors value which is the scsi_cmnd result 214 * field. 215 */ 216 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd, 217 int data_direction, void *buffer, unsigned bufflen, 218 unsigned char *sense, int timeout, int retries, u64 flags, 219 int *resid) 220 { 221 struct request *req; 222 int write = (data_direction == DMA_TO_DEVICE); 223 int ret = DRIVER_ERROR << 24; 224 225 req = blk_get_request(sdev->request_queue, write, __GFP_RECLAIM); 226 if (IS_ERR(req)) 227 return ret; 228 blk_rq_set_block_pc(req); 229 230 if (bufflen && blk_rq_map_kern(sdev->request_queue, req, 231 buffer, bufflen, __GFP_RECLAIM)) 232 goto out; 233 234 req->cmd_len = COMMAND_SIZE(cmd[0]); 235 memcpy(req->cmd, cmd, req->cmd_len); 236 req->sense = sense; 237 req->sense_len = 0; 238 req->retries = retries; 239 req->timeout = timeout; 240 req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT; 241 242 /* 243 * head injection *required* here otherwise quiesce won't work 244 */ 245 blk_execute_rq(req->q, NULL, req, 1); 246 247 /* 248 * Some devices (USB mass-storage in particular) may transfer 249 * garbage data together with a residue indicating that the data 250 * is invalid. Prevent the garbage from being misinterpreted 251 * and prevent security leaks by zeroing out the excess data. 252 */ 253 if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen)) 254 memset(buffer + (bufflen - req->resid_len), 0, req->resid_len); 255 256 if (resid) 257 *resid = req->resid_len; 258 ret = req->errors; 259 out: 260 blk_put_request(req); 261 262 return ret; 263 } 264 EXPORT_SYMBOL(scsi_execute); 265 266 int scsi_execute_req_flags(struct scsi_device *sdev, const unsigned char *cmd, 267 int data_direction, void *buffer, unsigned bufflen, 268 struct scsi_sense_hdr *sshdr, int timeout, int retries, 269 int *resid, u64 flags) 270 { 271 char *sense = NULL; 272 int result; 273 274 if (sshdr) { 275 sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO); 276 if (!sense) 277 return DRIVER_ERROR << 24; 278 } 279 result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen, 280 sense, timeout, retries, flags, resid); 281 if (sshdr) 282 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr); 283 284 kfree(sense); 285 return result; 286 } 287 EXPORT_SYMBOL(scsi_execute_req_flags); 288 289 /* 290 * Function: scsi_init_cmd_errh() 291 * 292 * Purpose: Initialize cmd fields related to error handling. 293 * 294 * Arguments: cmd - command that is ready to be queued. 295 * 296 * Notes: This function has the job of initializing a number of 297 * fields related to error handling. Typically this will 298 * be called once for each command, as required. 299 */ 300 static void scsi_init_cmd_errh(struct scsi_cmnd *cmd) 301 { 302 cmd->serial_number = 0; 303 scsi_set_resid(cmd, 0); 304 memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE); 305 if (cmd->cmd_len == 0) 306 cmd->cmd_len = scsi_command_size(cmd->cmnd); 307 } 308 309 void scsi_device_unbusy(struct scsi_device *sdev) 310 { 311 struct Scsi_Host *shost = sdev->host; 312 struct scsi_target *starget = scsi_target(sdev); 313 unsigned long flags; 314 315 atomic_dec(&shost->host_busy); 316 if (starget->can_queue > 0) 317 atomic_dec(&starget->target_busy); 318 319 if (unlikely(scsi_host_in_recovery(shost) && 320 (shost->host_failed || shost->host_eh_scheduled))) { 321 spin_lock_irqsave(shost->host_lock, flags); 322 scsi_eh_wakeup(shost); 323 spin_unlock_irqrestore(shost->host_lock, flags); 324 } 325 326 atomic_dec(&sdev->device_busy); 327 } 328 329 static void scsi_kick_queue(struct request_queue *q) 330 { 331 if (q->mq_ops) 332 blk_mq_start_hw_queues(q); 333 else 334 blk_run_queue(q); 335 } 336 337 /* 338 * Called for single_lun devices on IO completion. Clear starget_sdev_user, 339 * and call blk_run_queue for all the scsi_devices on the target - 340 * including current_sdev first. 341 * 342 * Called with *no* scsi locks held. 343 */ 344 static void scsi_single_lun_run(struct scsi_device *current_sdev) 345 { 346 struct Scsi_Host *shost = current_sdev->host; 347 struct scsi_device *sdev, *tmp; 348 struct scsi_target *starget = scsi_target(current_sdev); 349 unsigned long flags; 350 351 spin_lock_irqsave(shost->host_lock, flags); 352 starget->starget_sdev_user = NULL; 353 spin_unlock_irqrestore(shost->host_lock, flags); 354 355 /* 356 * Call blk_run_queue for all LUNs on the target, starting with 357 * current_sdev. We race with others (to set starget_sdev_user), 358 * but in most cases, we will be first. Ideally, each LU on the 359 * target would get some limited time or requests on the target. 360 */ 361 scsi_kick_queue(current_sdev->request_queue); 362 363 spin_lock_irqsave(shost->host_lock, flags); 364 if (starget->starget_sdev_user) 365 goto out; 366 list_for_each_entry_safe(sdev, tmp, &starget->devices, 367 same_target_siblings) { 368 if (sdev == current_sdev) 369 continue; 370 if (scsi_device_get(sdev)) 371 continue; 372 373 spin_unlock_irqrestore(shost->host_lock, flags); 374 scsi_kick_queue(sdev->request_queue); 375 spin_lock_irqsave(shost->host_lock, flags); 376 377 scsi_device_put(sdev); 378 } 379 out: 380 spin_unlock_irqrestore(shost->host_lock, flags); 381 } 382 383 static inline bool scsi_device_is_busy(struct scsi_device *sdev) 384 { 385 if (atomic_read(&sdev->device_busy) >= sdev->queue_depth) 386 return true; 387 if (atomic_read(&sdev->device_blocked) > 0) 388 return true; 389 return false; 390 } 391 392 static inline bool scsi_target_is_busy(struct scsi_target *starget) 393 { 394 if (starget->can_queue > 0) { 395 if (atomic_read(&starget->target_busy) >= starget->can_queue) 396 return true; 397 if (atomic_read(&starget->target_blocked) > 0) 398 return true; 399 } 400 return false; 401 } 402 403 static inline bool scsi_host_is_busy(struct Scsi_Host *shost) 404 { 405 if (shost->can_queue > 0 && 406 atomic_read(&shost->host_busy) >= shost->can_queue) 407 return true; 408 if (atomic_read(&shost->host_blocked) > 0) 409 return true; 410 if (shost->host_self_blocked) 411 return true; 412 return false; 413 } 414 415 static void scsi_starved_list_run(struct Scsi_Host *shost) 416 { 417 LIST_HEAD(starved_list); 418 struct scsi_device *sdev; 419 unsigned long flags; 420 421 spin_lock_irqsave(shost->host_lock, flags); 422 list_splice_init(&shost->starved_list, &starved_list); 423 424 while (!list_empty(&starved_list)) { 425 struct request_queue *slq; 426 427 /* 428 * As long as shost is accepting commands and we have 429 * starved queues, call blk_run_queue. scsi_request_fn 430 * drops the queue_lock and can add us back to the 431 * starved_list. 432 * 433 * host_lock protects the starved_list and starved_entry. 434 * scsi_request_fn must get the host_lock before checking 435 * or modifying starved_list or starved_entry. 436 */ 437 if (scsi_host_is_busy(shost)) 438 break; 439 440 sdev = list_entry(starved_list.next, 441 struct scsi_device, starved_entry); 442 list_del_init(&sdev->starved_entry); 443 if (scsi_target_is_busy(scsi_target(sdev))) { 444 list_move_tail(&sdev->starved_entry, 445 &shost->starved_list); 446 continue; 447 } 448 449 /* 450 * Once we drop the host lock, a racing scsi_remove_device() 451 * call may remove the sdev from the starved list and destroy 452 * it and the queue. Mitigate by taking a reference to the 453 * queue and never touching the sdev again after we drop the 454 * host lock. Note: if __scsi_remove_device() invokes 455 * blk_cleanup_queue() before the queue is run from this 456 * function then blk_run_queue() will return immediately since 457 * blk_cleanup_queue() marks the queue with QUEUE_FLAG_DYING. 458 */ 459 slq = sdev->request_queue; 460 if (!blk_get_queue(slq)) 461 continue; 462 spin_unlock_irqrestore(shost->host_lock, flags); 463 464 scsi_kick_queue(slq); 465 blk_put_queue(slq); 466 467 spin_lock_irqsave(shost->host_lock, flags); 468 } 469 /* put any unprocessed entries back */ 470 list_splice(&starved_list, &shost->starved_list); 471 spin_unlock_irqrestore(shost->host_lock, flags); 472 } 473 474 /* 475 * Function: scsi_run_queue() 476 * 477 * Purpose: Select a proper request queue to serve next 478 * 479 * Arguments: q - last request's queue 480 * 481 * Returns: Nothing 482 * 483 * Notes: The previous command was completely finished, start 484 * a new one if possible. 485 */ 486 static void scsi_run_queue(struct request_queue *q) 487 { 488 struct scsi_device *sdev = q->queuedata; 489 490 if (scsi_target(sdev)->single_lun) 491 scsi_single_lun_run(sdev); 492 if (!list_empty(&sdev->host->starved_list)) 493 scsi_starved_list_run(sdev->host); 494 495 if (q->mq_ops) 496 blk_mq_start_stopped_hw_queues(q, false); 497 else 498 blk_run_queue(q); 499 } 500 501 void scsi_requeue_run_queue(struct work_struct *work) 502 { 503 struct scsi_device *sdev; 504 struct request_queue *q; 505 506 sdev = container_of(work, struct scsi_device, requeue_work); 507 q = sdev->request_queue; 508 scsi_run_queue(q); 509 } 510 511 /* 512 * Function: scsi_requeue_command() 513 * 514 * Purpose: Handle post-processing of completed commands. 515 * 516 * Arguments: q - queue to operate on 517 * cmd - command that may need to be requeued. 518 * 519 * Returns: Nothing 520 * 521 * Notes: After command completion, there may be blocks left 522 * over which weren't finished by the previous command 523 * this can be for a number of reasons - the main one is 524 * I/O errors in the middle of the request, in which case 525 * we need to request the blocks that come after the bad 526 * sector. 527 * Notes: Upon return, cmd is a stale pointer. 528 */ 529 static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd) 530 { 531 struct scsi_device *sdev = cmd->device; 532 struct request *req = cmd->request; 533 unsigned long flags; 534 535 spin_lock_irqsave(q->queue_lock, flags); 536 blk_unprep_request(req); 537 req->special = NULL; 538 scsi_put_command(cmd); 539 blk_requeue_request(q, req); 540 spin_unlock_irqrestore(q->queue_lock, flags); 541 542 scsi_run_queue(q); 543 544 put_device(&sdev->sdev_gendev); 545 } 546 547 void scsi_run_host_queues(struct Scsi_Host *shost) 548 { 549 struct scsi_device *sdev; 550 551 shost_for_each_device(sdev, shost) 552 scsi_run_queue(sdev->request_queue); 553 } 554 555 static inline unsigned int scsi_sgtable_index(unsigned short nents) 556 { 557 unsigned int index; 558 559 BUG_ON(nents > SCSI_MAX_SG_SEGMENTS); 560 561 if (nents <= 8) 562 index = 0; 563 else 564 index = get_count_order(nents) - 3; 565 566 return index; 567 } 568 569 static void scsi_sg_free(struct scatterlist *sgl, unsigned int nents) 570 { 571 struct scsi_host_sg_pool *sgp; 572 573 sgp = scsi_sg_pools + scsi_sgtable_index(nents); 574 mempool_free(sgl, sgp->pool); 575 } 576 577 static struct scatterlist *scsi_sg_alloc(unsigned int nents, gfp_t gfp_mask) 578 { 579 struct scsi_host_sg_pool *sgp; 580 581 sgp = scsi_sg_pools + scsi_sgtable_index(nents); 582 return mempool_alloc(sgp->pool, gfp_mask); 583 } 584 585 static void scsi_free_sgtable(struct scsi_data_buffer *sdb, bool mq) 586 { 587 if (mq && sdb->table.orig_nents <= SCSI_MAX_SG_SEGMENTS) 588 return; 589 __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, mq, scsi_sg_free); 590 } 591 592 static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents, bool mq) 593 { 594 struct scatterlist *first_chunk = NULL; 595 int ret; 596 597 BUG_ON(!nents); 598 599 if (mq) { 600 if (nents <= SCSI_MAX_SG_SEGMENTS) { 601 sdb->table.nents = sdb->table.orig_nents = nents; 602 sg_init_table(sdb->table.sgl, nents); 603 return 0; 604 } 605 first_chunk = sdb->table.sgl; 606 } 607 608 ret = __sg_alloc_table(&sdb->table, nents, SCSI_MAX_SG_SEGMENTS, 609 first_chunk, GFP_ATOMIC, scsi_sg_alloc); 610 if (unlikely(ret)) 611 scsi_free_sgtable(sdb, mq); 612 return ret; 613 } 614 615 static void scsi_uninit_cmd(struct scsi_cmnd *cmd) 616 { 617 if (cmd->request->cmd_type == REQ_TYPE_FS) { 618 struct scsi_driver *drv = scsi_cmd_to_driver(cmd); 619 620 if (drv->uninit_command) 621 drv->uninit_command(cmd); 622 } 623 } 624 625 static void scsi_mq_free_sgtables(struct scsi_cmnd *cmd) 626 { 627 if (cmd->sdb.table.nents) 628 scsi_free_sgtable(&cmd->sdb, true); 629 if (cmd->request->next_rq && cmd->request->next_rq->special) 630 scsi_free_sgtable(cmd->request->next_rq->special, true); 631 if (scsi_prot_sg_count(cmd)) 632 scsi_free_sgtable(cmd->prot_sdb, true); 633 } 634 635 static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd) 636 { 637 struct scsi_device *sdev = cmd->device; 638 struct Scsi_Host *shost = sdev->host; 639 unsigned long flags; 640 641 scsi_mq_free_sgtables(cmd); 642 scsi_uninit_cmd(cmd); 643 644 if (shost->use_cmd_list) { 645 BUG_ON(list_empty(&cmd->list)); 646 spin_lock_irqsave(&sdev->list_lock, flags); 647 list_del_init(&cmd->list); 648 spin_unlock_irqrestore(&sdev->list_lock, flags); 649 } 650 } 651 652 /* 653 * Function: scsi_release_buffers() 654 * 655 * Purpose: Free resources allocate for a scsi_command. 656 * 657 * Arguments: cmd - command that we are bailing. 658 * 659 * Lock status: Assumed that no lock is held upon entry. 660 * 661 * Returns: Nothing 662 * 663 * Notes: In the event that an upper level driver rejects a 664 * command, we must release resources allocated during 665 * the __init_io() function. Primarily this would involve 666 * the scatter-gather table. 667 */ 668 static void scsi_release_buffers(struct scsi_cmnd *cmd) 669 { 670 if (cmd->sdb.table.nents) 671 scsi_free_sgtable(&cmd->sdb, false); 672 673 memset(&cmd->sdb, 0, sizeof(cmd->sdb)); 674 675 if (scsi_prot_sg_count(cmd)) 676 scsi_free_sgtable(cmd->prot_sdb, false); 677 } 678 679 static void scsi_release_bidi_buffers(struct scsi_cmnd *cmd) 680 { 681 struct scsi_data_buffer *bidi_sdb = cmd->request->next_rq->special; 682 683 scsi_free_sgtable(bidi_sdb, false); 684 kmem_cache_free(scsi_sdb_cache, bidi_sdb); 685 cmd->request->next_rq->special = NULL; 686 } 687 688 static bool scsi_end_request(struct request *req, int error, 689 unsigned int bytes, unsigned int bidi_bytes) 690 { 691 struct scsi_cmnd *cmd = req->special; 692 struct scsi_device *sdev = cmd->device; 693 struct request_queue *q = sdev->request_queue; 694 695 if (blk_update_request(req, error, bytes)) 696 return true; 697 698 /* Bidi request must be completed as a whole */ 699 if (unlikely(bidi_bytes) && 700 blk_update_request(req->next_rq, error, bidi_bytes)) 701 return true; 702 703 if (blk_queue_add_random(q)) 704 add_disk_randomness(req->rq_disk); 705 706 if (req->mq_ctx) { 707 /* 708 * In the MQ case the command gets freed by __blk_mq_end_request, 709 * so we have to do all cleanup that depends on it earlier. 710 * 711 * We also can't kick the queues from irq context, so we 712 * will have to defer it to a workqueue. 713 */ 714 scsi_mq_uninit_cmd(cmd); 715 716 __blk_mq_end_request(req, error); 717 718 if (scsi_target(sdev)->single_lun || 719 !list_empty(&sdev->host->starved_list)) 720 kblockd_schedule_work(&sdev->requeue_work); 721 else 722 blk_mq_start_stopped_hw_queues(q, true); 723 } else { 724 unsigned long flags; 725 726 if (bidi_bytes) 727 scsi_release_bidi_buffers(cmd); 728 729 spin_lock_irqsave(q->queue_lock, flags); 730 blk_finish_request(req, error); 731 spin_unlock_irqrestore(q->queue_lock, flags); 732 733 scsi_release_buffers(cmd); 734 735 scsi_put_command(cmd); 736 scsi_run_queue(q); 737 } 738 739 put_device(&sdev->sdev_gendev); 740 return false; 741 } 742 743 /** 744 * __scsi_error_from_host_byte - translate SCSI error code into errno 745 * @cmd: SCSI command (unused) 746 * @result: scsi error code 747 * 748 * Translate SCSI error code into standard UNIX errno. 749 * Return values: 750 * -ENOLINK temporary transport failure 751 * -EREMOTEIO permanent target failure, do not retry 752 * -EBADE permanent nexus failure, retry on other path 753 * -ENOSPC No write space available 754 * -ENODATA Medium error 755 * -EIO unspecified I/O error 756 */ 757 static int __scsi_error_from_host_byte(struct scsi_cmnd *cmd, int result) 758 { 759 int error = 0; 760 761 switch(host_byte(result)) { 762 case DID_TRANSPORT_FAILFAST: 763 error = -ENOLINK; 764 break; 765 case DID_TARGET_FAILURE: 766 set_host_byte(cmd, DID_OK); 767 error = -EREMOTEIO; 768 break; 769 case DID_NEXUS_FAILURE: 770 set_host_byte(cmd, DID_OK); 771 error = -EBADE; 772 break; 773 case DID_ALLOC_FAILURE: 774 set_host_byte(cmd, DID_OK); 775 error = -ENOSPC; 776 break; 777 case DID_MEDIUM_ERROR: 778 set_host_byte(cmd, DID_OK); 779 error = -ENODATA; 780 break; 781 default: 782 error = -EIO; 783 break; 784 } 785 786 return error; 787 } 788 789 /* 790 * Function: scsi_io_completion() 791 * 792 * Purpose: Completion processing for block device I/O requests. 793 * 794 * Arguments: cmd - command that is finished. 795 * 796 * Lock status: Assumed that no lock is held upon entry. 797 * 798 * Returns: Nothing 799 * 800 * Notes: We will finish off the specified number of sectors. If we 801 * are done, the command block will be released and the queue 802 * function will be goosed. If we are not done then we have to 803 * figure out what to do next: 804 * 805 * a) We can call scsi_requeue_command(). The request 806 * will be unprepared and put back on the queue. Then 807 * a new command will be created for it. This should 808 * be used if we made forward progress, or if we want 809 * to switch from READ(10) to READ(6) for example. 810 * 811 * b) We can call __scsi_queue_insert(). The request will 812 * be put back on the queue and retried using the same 813 * command as before, possibly after a delay. 814 * 815 * c) We can call scsi_end_request() with -EIO to fail 816 * the remainder of the request. 817 */ 818 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes) 819 { 820 int result = cmd->result; 821 struct request_queue *q = cmd->device->request_queue; 822 struct request *req = cmd->request; 823 int error = 0; 824 struct scsi_sense_hdr sshdr; 825 bool sense_valid = false; 826 int sense_deferred = 0, level = 0; 827 enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY, 828 ACTION_DELAYED_RETRY} action; 829 unsigned long wait_for = (cmd->allowed + 1) * req->timeout; 830 831 if (result) { 832 sense_valid = scsi_command_normalize_sense(cmd, &sshdr); 833 if (sense_valid) 834 sense_deferred = scsi_sense_is_deferred(&sshdr); 835 } 836 837 if (req->cmd_type == REQ_TYPE_BLOCK_PC) { /* SG_IO ioctl from block level */ 838 if (result) { 839 if (sense_valid && req->sense) { 840 /* 841 * SG_IO wants current and deferred errors 842 */ 843 int len = 8 + cmd->sense_buffer[7]; 844 845 if (len > SCSI_SENSE_BUFFERSIZE) 846 len = SCSI_SENSE_BUFFERSIZE; 847 memcpy(req->sense, cmd->sense_buffer, len); 848 req->sense_len = len; 849 } 850 if (!sense_deferred) 851 error = __scsi_error_from_host_byte(cmd, result); 852 } 853 /* 854 * __scsi_error_from_host_byte may have reset the host_byte 855 */ 856 req->errors = cmd->result; 857 858 req->resid_len = scsi_get_resid(cmd); 859 860 if (scsi_bidi_cmnd(cmd)) { 861 /* 862 * Bidi commands Must be complete as a whole, 863 * both sides at once. 864 */ 865 req->next_rq->resid_len = scsi_in(cmd)->resid; 866 if (scsi_end_request(req, 0, blk_rq_bytes(req), 867 blk_rq_bytes(req->next_rq))) 868 BUG(); 869 return; 870 } 871 } else if (blk_rq_bytes(req) == 0 && result && !sense_deferred) { 872 /* 873 * Certain non BLOCK_PC requests are commands that don't 874 * actually transfer anything (FLUSH), so cannot use 875 * good_bytes != blk_rq_bytes(req) as the signal for an error. 876 * This sets the error explicitly for the problem case. 877 */ 878 error = __scsi_error_from_host_byte(cmd, result); 879 } 880 881 /* no bidi support for !REQ_TYPE_BLOCK_PC yet */ 882 BUG_ON(blk_bidi_rq(req)); 883 884 /* 885 * Next deal with any sectors which we were able to correctly 886 * handle. 887 */ 888 SCSI_LOG_HLCOMPLETE(1, scmd_printk(KERN_INFO, cmd, 889 "%u sectors total, %d bytes done.\n", 890 blk_rq_sectors(req), good_bytes)); 891 892 /* 893 * Recovered errors need reporting, but they're always treated 894 * as success, so fiddle the result code here. For BLOCK_PC 895 * we already took a copy of the original into rq->errors which 896 * is what gets returned to the user 897 */ 898 if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) { 899 /* if ATA PASS-THROUGH INFORMATION AVAILABLE skip 900 * print since caller wants ATA registers. Only occurs on 901 * SCSI ATA PASS_THROUGH commands when CK_COND=1 902 */ 903 if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d)) 904 ; 905 else if (!(req->cmd_flags & REQ_QUIET)) 906 scsi_print_sense(cmd); 907 result = 0; 908 /* BLOCK_PC may have set error */ 909 error = 0; 910 } 911 912 /* 913 * If we finished all bytes in the request we are done now. 914 */ 915 if (!scsi_end_request(req, error, good_bytes, 0)) 916 return; 917 918 /* 919 * Kill remainder if no retrys. 920 */ 921 if (error && scsi_noretry_cmd(cmd)) { 922 if (scsi_end_request(req, error, blk_rq_bytes(req), 0)) 923 BUG(); 924 return; 925 } 926 927 /* 928 * If there had been no error, but we have leftover bytes in the 929 * requeues just queue the command up again. 930 */ 931 if (result == 0) 932 goto requeue; 933 934 error = __scsi_error_from_host_byte(cmd, result); 935 936 if (host_byte(result) == DID_RESET) { 937 /* Third party bus reset or reset for error recovery 938 * reasons. Just retry the command and see what 939 * happens. 940 */ 941 action = ACTION_RETRY; 942 } else if (sense_valid && !sense_deferred) { 943 switch (sshdr.sense_key) { 944 case UNIT_ATTENTION: 945 if (cmd->device->removable) { 946 /* Detected disc change. Set a bit 947 * and quietly refuse further access. 948 */ 949 cmd->device->changed = 1; 950 action = ACTION_FAIL; 951 } else { 952 /* Must have been a power glitch, or a 953 * bus reset. Could not have been a 954 * media change, so we just retry the 955 * command and see what happens. 956 */ 957 action = ACTION_RETRY; 958 } 959 break; 960 case ILLEGAL_REQUEST: 961 /* If we had an ILLEGAL REQUEST returned, then 962 * we may have performed an unsupported 963 * command. The only thing this should be 964 * would be a ten byte read where only a six 965 * byte read was supported. Also, on a system 966 * where READ CAPACITY failed, we may have 967 * read past the end of the disk. 968 */ 969 if ((cmd->device->use_10_for_rw && 970 sshdr.asc == 0x20 && sshdr.ascq == 0x00) && 971 (cmd->cmnd[0] == READ_10 || 972 cmd->cmnd[0] == WRITE_10)) { 973 /* This will issue a new 6-byte command. */ 974 cmd->device->use_10_for_rw = 0; 975 action = ACTION_REPREP; 976 } else if (sshdr.asc == 0x10) /* DIX */ { 977 action = ACTION_FAIL; 978 error = -EILSEQ; 979 /* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */ 980 } else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) { 981 action = ACTION_FAIL; 982 error = -EREMOTEIO; 983 } else 984 action = ACTION_FAIL; 985 break; 986 case ABORTED_COMMAND: 987 action = ACTION_FAIL; 988 if (sshdr.asc == 0x10) /* DIF */ 989 error = -EILSEQ; 990 break; 991 case NOT_READY: 992 /* If the device is in the process of becoming 993 * ready, or has a temporary blockage, retry. 994 */ 995 if (sshdr.asc == 0x04) { 996 switch (sshdr.ascq) { 997 case 0x01: /* becoming ready */ 998 case 0x04: /* format in progress */ 999 case 0x05: /* rebuild in progress */ 1000 case 0x06: /* recalculation in progress */ 1001 case 0x07: /* operation in progress */ 1002 case 0x08: /* Long write in progress */ 1003 case 0x09: /* self test in progress */ 1004 case 0x14: /* space allocation in progress */ 1005 action = ACTION_DELAYED_RETRY; 1006 break; 1007 default: 1008 action = ACTION_FAIL; 1009 break; 1010 } 1011 } else 1012 action = ACTION_FAIL; 1013 break; 1014 case VOLUME_OVERFLOW: 1015 /* See SSC3rXX or current. */ 1016 action = ACTION_FAIL; 1017 break; 1018 default: 1019 action = ACTION_FAIL; 1020 break; 1021 } 1022 } else 1023 action = ACTION_FAIL; 1024 1025 if (action != ACTION_FAIL && 1026 time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) 1027 action = ACTION_FAIL; 1028 1029 switch (action) { 1030 case ACTION_FAIL: 1031 /* Give up and fail the remainder of the request */ 1032 if (!(req->cmd_flags & REQ_QUIET)) { 1033 static DEFINE_RATELIMIT_STATE(_rs, 1034 DEFAULT_RATELIMIT_INTERVAL, 1035 DEFAULT_RATELIMIT_BURST); 1036 1037 if (unlikely(scsi_logging_level)) 1038 level = SCSI_LOG_LEVEL(SCSI_LOG_MLCOMPLETE_SHIFT, 1039 SCSI_LOG_MLCOMPLETE_BITS); 1040 1041 /* 1042 * if logging is enabled the failure will be printed 1043 * in scsi_log_completion(), so avoid duplicate messages 1044 */ 1045 if (!level && __ratelimit(&_rs)) { 1046 scsi_print_result(cmd, NULL, FAILED); 1047 if (driver_byte(result) & DRIVER_SENSE) 1048 scsi_print_sense(cmd); 1049 scsi_print_command(cmd); 1050 } 1051 } 1052 if (!scsi_end_request(req, error, blk_rq_err_bytes(req), 0)) 1053 return; 1054 /*FALLTHRU*/ 1055 case ACTION_REPREP: 1056 requeue: 1057 /* Unprep the request and put it back at the head of the queue. 1058 * A new command will be prepared and issued. 1059 */ 1060 if (q->mq_ops) { 1061 cmd->request->cmd_flags &= ~REQ_DONTPREP; 1062 scsi_mq_uninit_cmd(cmd); 1063 scsi_mq_requeue_cmd(cmd); 1064 } else { 1065 scsi_release_buffers(cmd); 1066 scsi_requeue_command(q, cmd); 1067 } 1068 break; 1069 case ACTION_RETRY: 1070 /* Retry the same command immediately */ 1071 __scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0); 1072 break; 1073 case ACTION_DELAYED_RETRY: 1074 /* Retry the same command after a delay */ 1075 __scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0); 1076 break; 1077 } 1078 } 1079 1080 static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb) 1081 { 1082 int count; 1083 1084 /* 1085 * If sg table allocation fails, requeue request later. 1086 */ 1087 if (unlikely(scsi_alloc_sgtable(sdb, req->nr_phys_segments, 1088 req->mq_ctx != NULL))) 1089 return BLKPREP_DEFER; 1090 1091 /* 1092 * Next, walk the list, and fill in the addresses and sizes of 1093 * each segment. 1094 */ 1095 count = blk_rq_map_sg(req->q, req, sdb->table.sgl); 1096 BUG_ON(count > sdb->table.nents); 1097 sdb->table.nents = count; 1098 sdb->length = blk_rq_bytes(req); 1099 return BLKPREP_OK; 1100 } 1101 1102 /* 1103 * Function: scsi_init_io() 1104 * 1105 * Purpose: SCSI I/O initialize function. 1106 * 1107 * Arguments: cmd - Command descriptor we wish to initialize 1108 * 1109 * Returns: 0 on success 1110 * BLKPREP_DEFER if the failure is retryable 1111 * BLKPREP_KILL if the failure is fatal 1112 */ 1113 int scsi_init_io(struct scsi_cmnd *cmd) 1114 { 1115 struct scsi_device *sdev = cmd->device; 1116 struct request *rq = cmd->request; 1117 bool is_mq = (rq->mq_ctx != NULL); 1118 int error; 1119 1120 BUG_ON(!rq->nr_phys_segments); 1121 1122 error = scsi_init_sgtable(rq, &cmd->sdb); 1123 if (error) 1124 goto err_exit; 1125 1126 if (blk_bidi_rq(rq)) { 1127 if (!rq->q->mq_ops) { 1128 struct scsi_data_buffer *bidi_sdb = 1129 kmem_cache_zalloc(scsi_sdb_cache, GFP_ATOMIC); 1130 if (!bidi_sdb) { 1131 error = BLKPREP_DEFER; 1132 goto err_exit; 1133 } 1134 1135 rq->next_rq->special = bidi_sdb; 1136 } 1137 1138 error = scsi_init_sgtable(rq->next_rq, rq->next_rq->special); 1139 if (error) 1140 goto err_exit; 1141 } 1142 1143 if (blk_integrity_rq(rq)) { 1144 struct scsi_data_buffer *prot_sdb = cmd->prot_sdb; 1145 int ivecs, count; 1146 1147 if (prot_sdb == NULL) { 1148 /* 1149 * This can happen if someone (e.g. multipath) 1150 * queues a command to a device on an adapter 1151 * that does not support DIX. 1152 */ 1153 WARN_ON_ONCE(1); 1154 error = BLKPREP_KILL; 1155 goto err_exit; 1156 } 1157 1158 ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio); 1159 1160 if (scsi_alloc_sgtable(prot_sdb, ivecs, is_mq)) { 1161 error = BLKPREP_DEFER; 1162 goto err_exit; 1163 } 1164 1165 count = blk_rq_map_integrity_sg(rq->q, rq->bio, 1166 prot_sdb->table.sgl); 1167 BUG_ON(unlikely(count > ivecs)); 1168 BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q))); 1169 1170 cmd->prot_sdb = prot_sdb; 1171 cmd->prot_sdb->table.nents = count; 1172 } 1173 1174 return BLKPREP_OK; 1175 err_exit: 1176 if (is_mq) { 1177 scsi_mq_free_sgtables(cmd); 1178 } else { 1179 scsi_release_buffers(cmd); 1180 cmd->request->special = NULL; 1181 scsi_put_command(cmd); 1182 put_device(&sdev->sdev_gendev); 1183 } 1184 return error; 1185 } 1186 EXPORT_SYMBOL(scsi_init_io); 1187 1188 static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev, 1189 struct request *req) 1190 { 1191 struct scsi_cmnd *cmd; 1192 1193 if (!req->special) { 1194 /* Bail if we can't get a reference to the device */ 1195 if (!get_device(&sdev->sdev_gendev)) 1196 return NULL; 1197 1198 cmd = scsi_get_command(sdev, GFP_ATOMIC); 1199 if (unlikely(!cmd)) { 1200 put_device(&sdev->sdev_gendev); 1201 return NULL; 1202 } 1203 req->special = cmd; 1204 } else { 1205 cmd = req->special; 1206 } 1207 1208 /* pull a tag out of the request if we have one */ 1209 cmd->tag = req->tag; 1210 cmd->request = req; 1211 1212 cmd->cmnd = req->cmd; 1213 cmd->prot_op = SCSI_PROT_NORMAL; 1214 1215 return cmd; 1216 } 1217 1218 static int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req) 1219 { 1220 struct scsi_cmnd *cmd = req->special; 1221 1222 /* 1223 * BLOCK_PC requests may transfer data, in which case they must 1224 * a bio attached to them. Or they might contain a SCSI command 1225 * that does not transfer data, in which case they may optionally 1226 * submit a request without an attached bio. 1227 */ 1228 if (req->bio) { 1229 int ret = scsi_init_io(cmd); 1230 if (unlikely(ret)) 1231 return ret; 1232 } else { 1233 BUG_ON(blk_rq_bytes(req)); 1234 1235 memset(&cmd->sdb, 0, sizeof(cmd->sdb)); 1236 } 1237 1238 cmd->cmd_len = req->cmd_len; 1239 cmd->transfersize = blk_rq_bytes(req); 1240 cmd->allowed = req->retries; 1241 return BLKPREP_OK; 1242 } 1243 1244 /* 1245 * Setup a REQ_TYPE_FS command. These are simple request from filesystems 1246 * that still need to be translated to SCSI CDBs from the ULD. 1247 */ 1248 static int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req) 1249 { 1250 struct scsi_cmnd *cmd = req->special; 1251 1252 if (unlikely(sdev->handler && sdev->handler->prep_fn)) { 1253 int ret = sdev->handler->prep_fn(sdev, req); 1254 if (ret != BLKPREP_OK) 1255 return ret; 1256 } 1257 1258 memset(cmd->cmnd, 0, BLK_MAX_CDB); 1259 return scsi_cmd_to_driver(cmd)->init_command(cmd); 1260 } 1261 1262 static int scsi_setup_cmnd(struct scsi_device *sdev, struct request *req) 1263 { 1264 struct scsi_cmnd *cmd = req->special; 1265 1266 if (!blk_rq_bytes(req)) 1267 cmd->sc_data_direction = DMA_NONE; 1268 else if (rq_data_dir(req) == WRITE) 1269 cmd->sc_data_direction = DMA_TO_DEVICE; 1270 else 1271 cmd->sc_data_direction = DMA_FROM_DEVICE; 1272 1273 switch (req->cmd_type) { 1274 case REQ_TYPE_FS: 1275 return scsi_setup_fs_cmnd(sdev, req); 1276 case REQ_TYPE_BLOCK_PC: 1277 return scsi_setup_blk_pc_cmnd(sdev, req); 1278 default: 1279 return BLKPREP_KILL; 1280 } 1281 } 1282 1283 static int 1284 scsi_prep_state_check(struct scsi_device *sdev, struct request *req) 1285 { 1286 int ret = BLKPREP_OK; 1287 1288 /* 1289 * If the device is not in running state we will reject some 1290 * or all commands. 1291 */ 1292 if (unlikely(sdev->sdev_state != SDEV_RUNNING)) { 1293 switch (sdev->sdev_state) { 1294 case SDEV_OFFLINE: 1295 case SDEV_TRANSPORT_OFFLINE: 1296 /* 1297 * If the device is offline we refuse to process any 1298 * commands. The device must be brought online 1299 * before trying any recovery commands. 1300 */ 1301 sdev_printk(KERN_ERR, sdev, 1302 "rejecting I/O to offline device\n"); 1303 ret = BLKPREP_KILL; 1304 break; 1305 case SDEV_DEL: 1306 /* 1307 * If the device is fully deleted, we refuse to 1308 * process any commands as well. 1309 */ 1310 sdev_printk(KERN_ERR, sdev, 1311 "rejecting I/O to dead device\n"); 1312 ret = BLKPREP_KILL; 1313 break; 1314 case SDEV_BLOCK: 1315 case SDEV_CREATED_BLOCK: 1316 ret = BLKPREP_DEFER; 1317 break; 1318 case SDEV_QUIESCE: 1319 /* 1320 * If the devices is blocked we defer normal commands. 1321 */ 1322 if (!(req->cmd_flags & REQ_PREEMPT)) 1323 ret = BLKPREP_DEFER; 1324 break; 1325 default: 1326 /* 1327 * For any other not fully online state we only allow 1328 * special commands. In particular any user initiated 1329 * command is not allowed. 1330 */ 1331 if (!(req->cmd_flags & REQ_PREEMPT)) 1332 ret = BLKPREP_KILL; 1333 break; 1334 } 1335 } 1336 return ret; 1337 } 1338 1339 static int 1340 scsi_prep_return(struct request_queue *q, struct request *req, int ret) 1341 { 1342 struct scsi_device *sdev = q->queuedata; 1343 1344 switch (ret) { 1345 case BLKPREP_KILL: 1346 req->errors = DID_NO_CONNECT << 16; 1347 /* release the command and kill it */ 1348 if (req->special) { 1349 struct scsi_cmnd *cmd = req->special; 1350 scsi_release_buffers(cmd); 1351 scsi_put_command(cmd); 1352 put_device(&sdev->sdev_gendev); 1353 req->special = NULL; 1354 } 1355 break; 1356 case BLKPREP_DEFER: 1357 /* 1358 * If we defer, the blk_peek_request() returns NULL, but the 1359 * queue must be restarted, so we schedule a callback to happen 1360 * shortly. 1361 */ 1362 if (atomic_read(&sdev->device_busy) == 0) 1363 blk_delay_queue(q, SCSI_QUEUE_DELAY); 1364 break; 1365 default: 1366 req->cmd_flags |= REQ_DONTPREP; 1367 } 1368 1369 return ret; 1370 } 1371 1372 static int scsi_prep_fn(struct request_queue *q, struct request *req) 1373 { 1374 struct scsi_device *sdev = q->queuedata; 1375 struct scsi_cmnd *cmd; 1376 int ret; 1377 1378 ret = scsi_prep_state_check(sdev, req); 1379 if (ret != BLKPREP_OK) 1380 goto out; 1381 1382 cmd = scsi_get_cmd_from_req(sdev, req); 1383 if (unlikely(!cmd)) { 1384 ret = BLKPREP_DEFER; 1385 goto out; 1386 } 1387 1388 ret = scsi_setup_cmnd(sdev, req); 1389 out: 1390 return scsi_prep_return(q, req, ret); 1391 } 1392 1393 static void scsi_unprep_fn(struct request_queue *q, struct request *req) 1394 { 1395 scsi_uninit_cmd(req->special); 1396 } 1397 1398 /* 1399 * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else 1400 * return 0. 1401 * 1402 * Called with the queue_lock held. 1403 */ 1404 static inline int scsi_dev_queue_ready(struct request_queue *q, 1405 struct scsi_device *sdev) 1406 { 1407 unsigned int busy; 1408 1409 busy = atomic_inc_return(&sdev->device_busy) - 1; 1410 if (atomic_read(&sdev->device_blocked)) { 1411 if (busy) 1412 goto out_dec; 1413 1414 /* 1415 * unblock after device_blocked iterates to zero 1416 */ 1417 if (atomic_dec_return(&sdev->device_blocked) > 0) { 1418 /* 1419 * For the MQ case we take care of this in the caller. 1420 */ 1421 if (!q->mq_ops) 1422 blk_delay_queue(q, SCSI_QUEUE_DELAY); 1423 goto out_dec; 1424 } 1425 SCSI_LOG_MLQUEUE(3, sdev_printk(KERN_INFO, sdev, 1426 "unblocking device at zero depth\n")); 1427 } 1428 1429 if (busy >= sdev->queue_depth) 1430 goto out_dec; 1431 1432 return 1; 1433 out_dec: 1434 atomic_dec(&sdev->device_busy); 1435 return 0; 1436 } 1437 1438 /* 1439 * scsi_target_queue_ready: checks if there we can send commands to target 1440 * @sdev: scsi device on starget to check. 1441 */ 1442 static inline int scsi_target_queue_ready(struct Scsi_Host *shost, 1443 struct scsi_device *sdev) 1444 { 1445 struct scsi_target *starget = scsi_target(sdev); 1446 unsigned int busy; 1447 1448 if (starget->single_lun) { 1449 spin_lock_irq(shost->host_lock); 1450 if (starget->starget_sdev_user && 1451 starget->starget_sdev_user != sdev) { 1452 spin_unlock_irq(shost->host_lock); 1453 return 0; 1454 } 1455 starget->starget_sdev_user = sdev; 1456 spin_unlock_irq(shost->host_lock); 1457 } 1458 1459 if (starget->can_queue <= 0) 1460 return 1; 1461 1462 busy = atomic_inc_return(&starget->target_busy) - 1; 1463 if (atomic_read(&starget->target_blocked) > 0) { 1464 if (busy) 1465 goto starved; 1466 1467 /* 1468 * unblock after target_blocked iterates to zero 1469 */ 1470 if (atomic_dec_return(&starget->target_blocked) > 0) 1471 goto out_dec; 1472 1473 SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget, 1474 "unblocking target at zero depth\n")); 1475 } 1476 1477 if (busy >= starget->can_queue) 1478 goto starved; 1479 1480 return 1; 1481 1482 starved: 1483 spin_lock_irq(shost->host_lock); 1484 list_move_tail(&sdev->starved_entry, &shost->starved_list); 1485 spin_unlock_irq(shost->host_lock); 1486 out_dec: 1487 if (starget->can_queue > 0) 1488 atomic_dec(&starget->target_busy); 1489 return 0; 1490 } 1491 1492 /* 1493 * scsi_host_queue_ready: if we can send requests to shost, return 1 else 1494 * return 0. We must end up running the queue again whenever 0 is 1495 * returned, else IO can hang. 1496 */ 1497 static inline int scsi_host_queue_ready(struct request_queue *q, 1498 struct Scsi_Host *shost, 1499 struct scsi_device *sdev) 1500 { 1501 unsigned int busy; 1502 1503 if (scsi_host_in_recovery(shost)) 1504 return 0; 1505 1506 busy = atomic_inc_return(&shost->host_busy) - 1; 1507 if (atomic_read(&shost->host_blocked) > 0) { 1508 if (busy) 1509 goto starved; 1510 1511 /* 1512 * unblock after host_blocked iterates to zero 1513 */ 1514 if (atomic_dec_return(&shost->host_blocked) > 0) 1515 goto out_dec; 1516 1517 SCSI_LOG_MLQUEUE(3, 1518 shost_printk(KERN_INFO, shost, 1519 "unblocking host at zero depth\n")); 1520 } 1521 1522 if (shost->can_queue > 0 && busy >= shost->can_queue) 1523 goto starved; 1524 if (shost->host_self_blocked) 1525 goto starved; 1526 1527 /* We're OK to process the command, so we can't be starved */ 1528 if (!list_empty(&sdev->starved_entry)) { 1529 spin_lock_irq(shost->host_lock); 1530 if (!list_empty(&sdev->starved_entry)) 1531 list_del_init(&sdev->starved_entry); 1532 spin_unlock_irq(shost->host_lock); 1533 } 1534 1535 return 1; 1536 1537 starved: 1538 spin_lock_irq(shost->host_lock); 1539 if (list_empty(&sdev->starved_entry)) 1540 list_add_tail(&sdev->starved_entry, &shost->starved_list); 1541 spin_unlock_irq(shost->host_lock); 1542 out_dec: 1543 atomic_dec(&shost->host_busy); 1544 return 0; 1545 } 1546 1547 /* 1548 * Busy state exporting function for request stacking drivers. 1549 * 1550 * For efficiency, no lock is taken to check the busy state of 1551 * shost/starget/sdev, since the returned value is not guaranteed and 1552 * may be changed after request stacking drivers call the function, 1553 * regardless of taking lock or not. 1554 * 1555 * When scsi can't dispatch I/Os anymore and needs to kill I/Os scsi 1556 * needs to return 'not busy'. Otherwise, request stacking drivers 1557 * may hold requests forever. 1558 */ 1559 static int scsi_lld_busy(struct request_queue *q) 1560 { 1561 struct scsi_device *sdev = q->queuedata; 1562 struct Scsi_Host *shost; 1563 1564 if (blk_queue_dying(q)) 1565 return 0; 1566 1567 shost = sdev->host; 1568 1569 /* 1570 * Ignore host/starget busy state. 1571 * Since block layer does not have a concept of fairness across 1572 * multiple queues, congestion of host/starget needs to be handled 1573 * in SCSI layer. 1574 */ 1575 if (scsi_host_in_recovery(shost) || scsi_device_is_busy(sdev)) 1576 return 1; 1577 1578 return 0; 1579 } 1580 1581 /* 1582 * Kill a request for a dead device 1583 */ 1584 static void scsi_kill_request(struct request *req, struct request_queue *q) 1585 { 1586 struct scsi_cmnd *cmd = req->special; 1587 struct scsi_device *sdev; 1588 struct scsi_target *starget; 1589 struct Scsi_Host *shost; 1590 1591 blk_start_request(req); 1592 1593 scmd_printk(KERN_INFO, cmd, "killing request\n"); 1594 1595 sdev = cmd->device; 1596 starget = scsi_target(sdev); 1597 shost = sdev->host; 1598 scsi_init_cmd_errh(cmd); 1599 cmd->result = DID_NO_CONNECT << 16; 1600 atomic_inc(&cmd->device->iorequest_cnt); 1601 1602 /* 1603 * SCSI request completion path will do scsi_device_unbusy(), 1604 * bump busy counts. To bump the counters, we need to dance 1605 * with the locks as normal issue path does. 1606 */ 1607 atomic_inc(&sdev->device_busy); 1608 atomic_inc(&shost->host_busy); 1609 if (starget->can_queue > 0) 1610 atomic_inc(&starget->target_busy); 1611 1612 blk_complete_request(req); 1613 } 1614 1615 static void scsi_softirq_done(struct request *rq) 1616 { 1617 struct scsi_cmnd *cmd = rq->special; 1618 unsigned long wait_for = (cmd->allowed + 1) * rq->timeout; 1619 int disposition; 1620 1621 INIT_LIST_HEAD(&cmd->eh_entry); 1622 1623 atomic_inc(&cmd->device->iodone_cnt); 1624 if (cmd->result) 1625 atomic_inc(&cmd->device->ioerr_cnt); 1626 1627 disposition = scsi_decide_disposition(cmd); 1628 if (disposition != SUCCESS && 1629 time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) { 1630 sdev_printk(KERN_ERR, cmd->device, 1631 "timing out command, waited %lus\n", 1632 wait_for/HZ); 1633 disposition = SUCCESS; 1634 } 1635 1636 scsi_log_completion(cmd, disposition); 1637 1638 switch (disposition) { 1639 case SUCCESS: 1640 scsi_finish_command(cmd); 1641 break; 1642 case NEEDS_RETRY: 1643 scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY); 1644 break; 1645 case ADD_TO_MLQUEUE: 1646 scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY); 1647 break; 1648 default: 1649 if (!scsi_eh_scmd_add(cmd, 0)) 1650 scsi_finish_command(cmd); 1651 } 1652 } 1653 1654 /** 1655 * scsi_dispatch_command - Dispatch a command to the low-level driver. 1656 * @cmd: command block we are dispatching. 1657 * 1658 * Return: nonzero return request was rejected and device's queue needs to be 1659 * plugged. 1660 */ 1661 static int scsi_dispatch_cmd(struct scsi_cmnd *cmd) 1662 { 1663 struct Scsi_Host *host = cmd->device->host; 1664 int rtn = 0; 1665 1666 atomic_inc(&cmd->device->iorequest_cnt); 1667 1668 /* check if the device is still usable */ 1669 if (unlikely(cmd->device->sdev_state == SDEV_DEL)) { 1670 /* in SDEV_DEL we error all commands. DID_NO_CONNECT 1671 * returns an immediate error upwards, and signals 1672 * that the device is no longer present */ 1673 cmd->result = DID_NO_CONNECT << 16; 1674 goto done; 1675 } 1676 1677 /* Check to see if the scsi lld made this device blocked. */ 1678 if (unlikely(scsi_device_blocked(cmd->device))) { 1679 /* 1680 * in blocked state, the command is just put back on 1681 * the device queue. The suspend state has already 1682 * blocked the queue so future requests should not 1683 * occur until the device transitions out of the 1684 * suspend state. 1685 */ 1686 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd, 1687 "queuecommand : device blocked\n")); 1688 return SCSI_MLQUEUE_DEVICE_BUSY; 1689 } 1690 1691 /* Store the LUN value in cmnd, if needed. */ 1692 if (cmd->device->lun_in_cdb) 1693 cmd->cmnd[1] = (cmd->cmnd[1] & 0x1f) | 1694 (cmd->device->lun << 5 & 0xe0); 1695 1696 scsi_log_send(cmd); 1697 1698 /* 1699 * Before we queue this command, check if the command 1700 * length exceeds what the host adapter can handle. 1701 */ 1702 if (cmd->cmd_len > cmd->device->host->max_cmd_len) { 1703 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd, 1704 "queuecommand : command too long. " 1705 "cdb_size=%d host->max_cmd_len=%d\n", 1706 cmd->cmd_len, cmd->device->host->max_cmd_len)); 1707 cmd->result = (DID_ABORT << 16); 1708 goto done; 1709 } 1710 1711 if (unlikely(host->shost_state == SHOST_DEL)) { 1712 cmd->result = (DID_NO_CONNECT << 16); 1713 goto done; 1714 1715 } 1716 1717 trace_scsi_dispatch_cmd_start(cmd); 1718 rtn = host->hostt->queuecommand(host, cmd); 1719 if (rtn) { 1720 trace_scsi_dispatch_cmd_error(cmd, rtn); 1721 if (rtn != SCSI_MLQUEUE_DEVICE_BUSY && 1722 rtn != SCSI_MLQUEUE_TARGET_BUSY) 1723 rtn = SCSI_MLQUEUE_HOST_BUSY; 1724 1725 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd, 1726 "queuecommand : request rejected\n")); 1727 } 1728 1729 return rtn; 1730 done: 1731 cmd->scsi_done(cmd); 1732 return 0; 1733 } 1734 1735 /** 1736 * scsi_done - Invoke completion on finished SCSI command. 1737 * @cmd: The SCSI Command for which a low-level device driver (LLDD) gives 1738 * ownership back to SCSI Core -- i.e. the LLDD has finished with it. 1739 * 1740 * Description: This function is the mid-level's (SCSI Core) interrupt routine, 1741 * which regains ownership of the SCSI command (de facto) from a LLDD, and 1742 * calls blk_complete_request() for further processing. 1743 * 1744 * This function is interrupt context safe. 1745 */ 1746 static void scsi_done(struct scsi_cmnd *cmd) 1747 { 1748 trace_scsi_dispatch_cmd_done(cmd); 1749 blk_complete_request(cmd->request); 1750 } 1751 1752 /* 1753 * Function: scsi_request_fn() 1754 * 1755 * Purpose: Main strategy routine for SCSI. 1756 * 1757 * Arguments: q - Pointer to actual queue. 1758 * 1759 * Returns: Nothing 1760 * 1761 * Lock status: IO request lock assumed to be held when called. 1762 */ 1763 static void scsi_request_fn(struct request_queue *q) 1764 __releases(q->queue_lock) 1765 __acquires(q->queue_lock) 1766 { 1767 struct scsi_device *sdev = q->queuedata; 1768 struct Scsi_Host *shost; 1769 struct scsi_cmnd *cmd; 1770 struct request *req; 1771 1772 /* 1773 * To start with, we keep looping until the queue is empty, or until 1774 * the host is no longer able to accept any more requests. 1775 */ 1776 shost = sdev->host; 1777 for (;;) { 1778 int rtn; 1779 /* 1780 * get next queueable request. We do this early to make sure 1781 * that the request is fully prepared even if we cannot 1782 * accept it. 1783 */ 1784 req = blk_peek_request(q); 1785 if (!req) 1786 break; 1787 1788 if (unlikely(!scsi_device_online(sdev))) { 1789 sdev_printk(KERN_ERR, sdev, 1790 "rejecting I/O to offline device\n"); 1791 scsi_kill_request(req, q); 1792 continue; 1793 } 1794 1795 if (!scsi_dev_queue_ready(q, sdev)) 1796 break; 1797 1798 /* 1799 * Remove the request from the request list. 1800 */ 1801 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req))) 1802 blk_start_request(req); 1803 1804 spin_unlock_irq(q->queue_lock); 1805 cmd = req->special; 1806 if (unlikely(cmd == NULL)) { 1807 printk(KERN_CRIT "impossible request in %s.\n" 1808 "please mail a stack trace to " 1809 "linux-scsi@vger.kernel.org\n", 1810 __func__); 1811 blk_dump_rq_flags(req, "foo"); 1812 BUG(); 1813 } 1814 1815 /* 1816 * We hit this when the driver is using a host wide 1817 * tag map. For device level tag maps the queue_depth check 1818 * in the device ready fn would prevent us from trying 1819 * to allocate a tag. Since the map is a shared host resource 1820 * we add the dev to the starved list so it eventually gets 1821 * a run when a tag is freed. 1822 */ 1823 if (blk_queue_tagged(q) && !(req->cmd_flags & REQ_QUEUED)) { 1824 spin_lock_irq(shost->host_lock); 1825 if (list_empty(&sdev->starved_entry)) 1826 list_add_tail(&sdev->starved_entry, 1827 &shost->starved_list); 1828 spin_unlock_irq(shost->host_lock); 1829 goto not_ready; 1830 } 1831 1832 if (!scsi_target_queue_ready(shost, sdev)) 1833 goto not_ready; 1834 1835 if (!scsi_host_queue_ready(q, shost, sdev)) 1836 goto host_not_ready; 1837 1838 if (sdev->simple_tags) 1839 cmd->flags |= SCMD_TAGGED; 1840 else 1841 cmd->flags &= ~SCMD_TAGGED; 1842 1843 /* 1844 * Finally, initialize any error handling parameters, and set up 1845 * the timers for timeouts. 1846 */ 1847 scsi_init_cmd_errh(cmd); 1848 1849 /* 1850 * Dispatch the command to the low-level driver. 1851 */ 1852 cmd->scsi_done = scsi_done; 1853 rtn = scsi_dispatch_cmd(cmd); 1854 if (rtn) { 1855 scsi_queue_insert(cmd, rtn); 1856 spin_lock_irq(q->queue_lock); 1857 goto out_delay; 1858 } 1859 spin_lock_irq(q->queue_lock); 1860 } 1861 1862 return; 1863 1864 host_not_ready: 1865 if (scsi_target(sdev)->can_queue > 0) 1866 atomic_dec(&scsi_target(sdev)->target_busy); 1867 not_ready: 1868 /* 1869 * lock q, handle tag, requeue req, and decrement device_busy. We 1870 * must return with queue_lock held. 1871 * 1872 * Decrementing device_busy without checking it is OK, as all such 1873 * cases (host limits or settings) should run the queue at some 1874 * later time. 1875 */ 1876 spin_lock_irq(q->queue_lock); 1877 blk_requeue_request(q, req); 1878 atomic_dec(&sdev->device_busy); 1879 out_delay: 1880 if (!atomic_read(&sdev->device_busy) && !scsi_device_blocked(sdev)) 1881 blk_delay_queue(q, SCSI_QUEUE_DELAY); 1882 } 1883 1884 static inline int prep_to_mq(int ret) 1885 { 1886 switch (ret) { 1887 case BLKPREP_OK: 1888 return 0; 1889 case BLKPREP_DEFER: 1890 return BLK_MQ_RQ_QUEUE_BUSY; 1891 default: 1892 return BLK_MQ_RQ_QUEUE_ERROR; 1893 } 1894 } 1895 1896 static int scsi_mq_prep_fn(struct request *req) 1897 { 1898 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); 1899 struct scsi_device *sdev = req->q->queuedata; 1900 struct Scsi_Host *shost = sdev->host; 1901 unsigned char *sense_buf = cmd->sense_buffer; 1902 struct scatterlist *sg; 1903 1904 memset(cmd, 0, sizeof(struct scsi_cmnd)); 1905 1906 req->special = cmd; 1907 1908 cmd->request = req; 1909 cmd->device = sdev; 1910 cmd->sense_buffer = sense_buf; 1911 1912 cmd->tag = req->tag; 1913 1914 cmd->cmnd = req->cmd; 1915 cmd->prot_op = SCSI_PROT_NORMAL; 1916 1917 INIT_LIST_HEAD(&cmd->list); 1918 INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler); 1919 cmd->jiffies_at_alloc = jiffies; 1920 1921 if (shost->use_cmd_list) { 1922 spin_lock_irq(&sdev->list_lock); 1923 list_add_tail(&cmd->list, &sdev->cmd_list); 1924 spin_unlock_irq(&sdev->list_lock); 1925 } 1926 1927 sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size; 1928 cmd->sdb.table.sgl = sg; 1929 1930 if (scsi_host_get_prot(shost)) { 1931 cmd->prot_sdb = (void *)sg + 1932 min_t(unsigned int, 1933 shost->sg_tablesize, SCSI_MAX_SG_SEGMENTS) * 1934 sizeof(struct scatterlist); 1935 memset(cmd->prot_sdb, 0, sizeof(struct scsi_data_buffer)); 1936 1937 cmd->prot_sdb->table.sgl = 1938 (struct scatterlist *)(cmd->prot_sdb + 1); 1939 } 1940 1941 if (blk_bidi_rq(req)) { 1942 struct request *next_rq = req->next_rq; 1943 struct scsi_data_buffer *bidi_sdb = blk_mq_rq_to_pdu(next_rq); 1944 1945 memset(bidi_sdb, 0, sizeof(struct scsi_data_buffer)); 1946 bidi_sdb->table.sgl = 1947 (struct scatterlist *)(bidi_sdb + 1); 1948 1949 next_rq->special = bidi_sdb; 1950 } 1951 1952 blk_mq_start_request(req); 1953 1954 return scsi_setup_cmnd(sdev, req); 1955 } 1956 1957 static void scsi_mq_done(struct scsi_cmnd *cmd) 1958 { 1959 trace_scsi_dispatch_cmd_done(cmd); 1960 blk_mq_complete_request(cmd->request, cmd->request->errors); 1961 } 1962 1963 static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx, 1964 const struct blk_mq_queue_data *bd) 1965 { 1966 struct request *req = bd->rq; 1967 struct request_queue *q = req->q; 1968 struct scsi_device *sdev = q->queuedata; 1969 struct Scsi_Host *shost = sdev->host; 1970 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); 1971 int ret; 1972 int reason; 1973 1974 ret = prep_to_mq(scsi_prep_state_check(sdev, req)); 1975 if (ret) 1976 goto out; 1977 1978 ret = BLK_MQ_RQ_QUEUE_BUSY; 1979 if (!get_device(&sdev->sdev_gendev)) 1980 goto out; 1981 1982 if (!scsi_dev_queue_ready(q, sdev)) 1983 goto out_put_device; 1984 if (!scsi_target_queue_ready(shost, sdev)) 1985 goto out_dec_device_busy; 1986 if (!scsi_host_queue_ready(q, shost, sdev)) 1987 goto out_dec_target_busy; 1988 1989 1990 if (!(req->cmd_flags & REQ_DONTPREP)) { 1991 ret = prep_to_mq(scsi_mq_prep_fn(req)); 1992 if (ret) 1993 goto out_dec_host_busy; 1994 req->cmd_flags |= REQ_DONTPREP; 1995 } else { 1996 blk_mq_start_request(req); 1997 } 1998 1999 if (sdev->simple_tags) 2000 cmd->flags |= SCMD_TAGGED; 2001 else 2002 cmd->flags &= ~SCMD_TAGGED; 2003 2004 scsi_init_cmd_errh(cmd); 2005 cmd->scsi_done = scsi_mq_done; 2006 2007 reason = scsi_dispatch_cmd(cmd); 2008 if (reason) { 2009 scsi_set_blocked(cmd, reason); 2010 ret = BLK_MQ_RQ_QUEUE_BUSY; 2011 goto out_dec_host_busy; 2012 } 2013 2014 return BLK_MQ_RQ_QUEUE_OK; 2015 2016 out_dec_host_busy: 2017 atomic_dec(&shost->host_busy); 2018 out_dec_target_busy: 2019 if (scsi_target(sdev)->can_queue > 0) 2020 atomic_dec(&scsi_target(sdev)->target_busy); 2021 out_dec_device_busy: 2022 atomic_dec(&sdev->device_busy); 2023 out_put_device: 2024 put_device(&sdev->sdev_gendev); 2025 out: 2026 switch (ret) { 2027 case BLK_MQ_RQ_QUEUE_BUSY: 2028 blk_mq_stop_hw_queue(hctx); 2029 if (atomic_read(&sdev->device_busy) == 0 && 2030 !scsi_device_blocked(sdev)) 2031 blk_mq_delay_queue(hctx, SCSI_QUEUE_DELAY); 2032 break; 2033 case BLK_MQ_RQ_QUEUE_ERROR: 2034 /* 2035 * Make sure to release all allocated ressources when 2036 * we hit an error, as we will never see this command 2037 * again. 2038 */ 2039 if (req->cmd_flags & REQ_DONTPREP) 2040 scsi_mq_uninit_cmd(cmd); 2041 break; 2042 default: 2043 break; 2044 } 2045 return ret; 2046 } 2047 2048 static enum blk_eh_timer_return scsi_timeout(struct request *req, 2049 bool reserved) 2050 { 2051 if (reserved) 2052 return BLK_EH_RESET_TIMER; 2053 return scsi_times_out(req); 2054 } 2055 2056 static int scsi_init_request(void *data, struct request *rq, 2057 unsigned int hctx_idx, unsigned int request_idx, 2058 unsigned int numa_node) 2059 { 2060 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 2061 2062 cmd->sense_buffer = kzalloc_node(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL, 2063 numa_node); 2064 if (!cmd->sense_buffer) 2065 return -ENOMEM; 2066 return 0; 2067 } 2068 2069 static void scsi_exit_request(void *data, struct request *rq, 2070 unsigned int hctx_idx, unsigned int request_idx) 2071 { 2072 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 2073 2074 kfree(cmd->sense_buffer); 2075 } 2076 2077 static u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost) 2078 { 2079 struct device *host_dev; 2080 u64 bounce_limit = 0xffffffff; 2081 2082 if (shost->unchecked_isa_dma) 2083 return BLK_BOUNCE_ISA; 2084 /* 2085 * Platforms with virtual-DMA translation 2086 * hardware have no practical limit. 2087 */ 2088 if (!PCI_DMA_BUS_IS_PHYS) 2089 return BLK_BOUNCE_ANY; 2090 2091 host_dev = scsi_get_device(shost); 2092 if (host_dev && host_dev->dma_mask) 2093 bounce_limit = (u64)dma_max_pfn(host_dev) << PAGE_SHIFT; 2094 2095 return bounce_limit; 2096 } 2097 2098 static void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q) 2099 { 2100 struct device *dev = shost->dma_dev; 2101 2102 /* 2103 * this limit is imposed by hardware restrictions 2104 */ 2105 blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize, 2106 SCSI_MAX_SG_CHAIN_SEGMENTS)); 2107 2108 if (scsi_host_prot_dma(shost)) { 2109 shost->sg_prot_tablesize = 2110 min_not_zero(shost->sg_prot_tablesize, 2111 (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS); 2112 BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize); 2113 blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize); 2114 } 2115 2116 blk_queue_max_hw_sectors(q, shost->max_sectors); 2117 blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost)); 2118 blk_queue_segment_boundary(q, shost->dma_boundary); 2119 dma_set_seg_boundary(dev, shost->dma_boundary); 2120 2121 blk_queue_max_segment_size(q, dma_get_max_seg_size(dev)); 2122 2123 if (!shost->use_clustering) 2124 q->limits.cluster = 0; 2125 2126 /* 2127 * set a reasonable default alignment on word boundaries: the 2128 * host and device may alter it using 2129 * blk_queue_update_dma_alignment() later. 2130 */ 2131 blk_queue_dma_alignment(q, 0x03); 2132 } 2133 2134 struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost, 2135 request_fn_proc *request_fn) 2136 { 2137 struct request_queue *q; 2138 2139 q = blk_init_queue(request_fn, NULL); 2140 if (!q) 2141 return NULL; 2142 __scsi_init_queue(shost, q); 2143 return q; 2144 } 2145 EXPORT_SYMBOL(__scsi_alloc_queue); 2146 2147 struct request_queue *scsi_alloc_queue(struct scsi_device *sdev) 2148 { 2149 struct request_queue *q; 2150 2151 q = __scsi_alloc_queue(sdev->host, scsi_request_fn); 2152 if (!q) 2153 return NULL; 2154 2155 blk_queue_prep_rq(q, scsi_prep_fn); 2156 blk_queue_unprep_rq(q, scsi_unprep_fn); 2157 blk_queue_softirq_done(q, scsi_softirq_done); 2158 blk_queue_rq_timed_out(q, scsi_times_out); 2159 blk_queue_lld_busy(q, scsi_lld_busy); 2160 return q; 2161 } 2162 2163 static struct blk_mq_ops scsi_mq_ops = { 2164 .map_queue = blk_mq_map_queue, 2165 .queue_rq = scsi_queue_rq, 2166 .complete = scsi_softirq_done, 2167 .timeout = scsi_timeout, 2168 .init_request = scsi_init_request, 2169 .exit_request = scsi_exit_request, 2170 }; 2171 2172 struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev) 2173 { 2174 sdev->request_queue = blk_mq_init_queue(&sdev->host->tag_set); 2175 if (IS_ERR(sdev->request_queue)) 2176 return NULL; 2177 2178 sdev->request_queue->queuedata = sdev; 2179 __scsi_init_queue(sdev->host, sdev->request_queue); 2180 return sdev->request_queue; 2181 } 2182 2183 int scsi_mq_setup_tags(struct Scsi_Host *shost) 2184 { 2185 unsigned int cmd_size, sgl_size, tbl_size; 2186 2187 tbl_size = shost->sg_tablesize; 2188 if (tbl_size > SCSI_MAX_SG_SEGMENTS) 2189 tbl_size = SCSI_MAX_SG_SEGMENTS; 2190 sgl_size = tbl_size * sizeof(struct scatterlist); 2191 cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size + sgl_size; 2192 if (scsi_host_get_prot(shost)) 2193 cmd_size += sizeof(struct scsi_data_buffer) + sgl_size; 2194 2195 memset(&shost->tag_set, 0, sizeof(shost->tag_set)); 2196 shost->tag_set.ops = &scsi_mq_ops; 2197 shost->tag_set.nr_hw_queues = shost->nr_hw_queues ? : 1; 2198 shost->tag_set.queue_depth = shost->can_queue; 2199 shost->tag_set.cmd_size = cmd_size; 2200 shost->tag_set.numa_node = NUMA_NO_NODE; 2201 shost->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; 2202 shost->tag_set.flags |= 2203 BLK_ALLOC_POLICY_TO_MQ_FLAG(shost->hostt->tag_alloc_policy); 2204 shost->tag_set.driver_data = shost; 2205 2206 return blk_mq_alloc_tag_set(&shost->tag_set); 2207 } 2208 2209 void scsi_mq_destroy_tags(struct Scsi_Host *shost) 2210 { 2211 blk_mq_free_tag_set(&shost->tag_set); 2212 } 2213 2214 /* 2215 * Function: scsi_block_requests() 2216 * 2217 * Purpose: Utility function used by low-level drivers to prevent further 2218 * commands from being queued to the device. 2219 * 2220 * Arguments: shost - Host in question 2221 * 2222 * Returns: Nothing 2223 * 2224 * Lock status: No locks are assumed held. 2225 * 2226 * Notes: There is no timer nor any other means by which the requests 2227 * get unblocked other than the low-level driver calling 2228 * scsi_unblock_requests(). 2229 */ 2230 void scsi_block_requests(struct Scsi_Host *shost) 2231 { 2232 shost->host_self_blocked = 1; 2233 } 2234 EXPORT_SYMBOL(scsi_block_requests); 2235 2236 /* 2237 * Function: scsi_unblock_requests() 2238 * 2239 * Purpose: Utility function used by low-level drivers to allow further 2240 * commands from being queued to the device. 2241 * 2242 * Arguments: shost - Host in question 2243 * 2244 * Returns: Nothing 2245 * 2246 * Lock status: No locks are assumed held. 2247 * 2248 * Notes: There is no timer nor any other means by which the requests 2249 * get unblocked other than the low-level driver calling 2250 * scsi_unblock_requests(). 2251 * 2252 * This is done as an API function so that changes to the 2253 * internals of the scsi mid-layer won't require wholesale 2254 * changes to drivers that use this feature. 2255 */ 2256 void scsi_unblock_requests(struct Scsi_Host *shost) 2257 { 2258 shost->host_self_blocked = 0; 2259 scsi_run_host_queues(shost); 2260 } 2261 EXPORT_SYMBOL(scsi_unblock_requests); 2262 2263 int __init scsi_init_queue(void) 2264 { 2265 int i; 2266 2267 scsi_sdb_cache = kmem_cache_create("scsi_data_buffer", 2268 sizeof(struct scsi_data_buffer), 2269 0, 0, NULL); 2270 if (!scsi_sdb_cache) { 2271 printk(KERN_ERR "SCSI: can't init scsi sdb cache\n"); 2272 return -ENOMEM; 2273 } 2274 2275 for (i = 0; i < SG_MEMPOOL_NR; i++) { 2276 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i; 2277 int size = sgp->size * sizeof(struct scatterlist); 2278 2279 sgp->slab = kmem_cache_create(sgp->name, size, 0, 2280 SLAB_HWCACHE_ALIGN, NULL); 2281 if (!sgp->slab) { 2282 printk(KERN_ERR "SCSI: can't init sg slab %s\n", 2283 sgp->name); 2284 goto cleanup_sdb; 2285 } 2286 2287 sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE, 2288 sgp->slab); 2289 if (!sgp->pool) { 2290 printk(KERN_ERR "SCSI: can't init sg mempool %s\n", 2291 sgp->name); 2292 goto cleanup_sdb; 2293 } 2294 } 2295 2296 return 0; 2297 2298 cleanup_sdb: 2299 for (i = 0; i < SG_MEMPOOL_NR; i++) { 2300 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i; 2301 if (sgp->pool) 2302 mempool_destroy(sgp->pool); 2303 if (sgp->slab) 2304 kmem_cache_destroy(sgp->slab); 2305 } 2306 kmem_cache_destroy(scsi_sdb_cache); 2307 2308 return -ENOMEM; 2309 } 2310 2311 void scsi_exit_queue(void) 2312 { 2313 int i; 2314 2315 kmem_cache_destroy(scsi_sdb_cache); 2316 2317 for (i = 0; i < SG_MEMPOOL_NR; i++) { 2318 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i; 2319 mempool_destroy(sgp->pool); 2320 kmem_cache_destroy(sgp->slab); 2321 } 2322 } 2323 2324 /** 2325 * scsi_mode_select - issue a mode select 2326 * @sdev: SCSI device to be queried 2327 * @pf: Page format bit (1 == standard, 0 == vendor specific) 2328 * @sp: Save page bit (0 == don't save, 1 == save) 2329 * @modepage: mode page being requested 2330 * @buffer: request buffer (may not be smaller than eight bytes) 2331 * @len: length of request buffer. 2332 * @timeout: command timeout 2333 * @retries: number of retries before failing 2334 * @data: returns a structure abstracting the mode header data 2335 * @sshdr: place to put sense data (or NULL if no sense to be collected). 2336 * must be SCSI_SENSE_BUFFERSIZE big. 2337 * 2338 * Returns zero if successful; negative error number or scsi 2339 * status on error 2340 * 2341 */ 2342 int 2343 scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage, 2344 unsigned char *buffer, int len, int timeout, int retries, 2345 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr) 2346 { 2347 unsigned char cmd[10]; 2348 unsigned char *real_buffer; 2349 int ret; 2350 2351 memset(cmd, 0, sizeof(cmd)); 2352 cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0); 2353 2354 if (sdev->use_10_for_ms) { 2355 if (len > 65535) 2356 return -EINVAL; 2357 real_buffer = kmalloc(8 + len, GFP_KERNEL); 2358 if (!real_buffer) 2359 return -ENOMEM; 2360 memcpy(real_buffer + 8, buffer, len); 2361 len += 8; 2362 real_buffer[0] = 0; 2363 real_buffer[1] = 0; 2364 real_buffer[2] = data->medium_type; 2365 real_buffer[3] = data->device_specific; 2366 real_buffer[4] = data->longlba ? 0x01 : 0; 2367 real_buffer[5] = 0; 2368 real_buffer[6] = data->block_descriptor_length >> 8; 2369 real_buffer[7] = data->block_descriptor_length; 2370 2371 cmd[0] = MODE_SELECT_10; 2372 cmd[7] = len >> 8; 2373 cmd[8] = len; 2374 } else { 2375 if (len > 255 || data->block_descriptor_length > 255 || 2376 data->longlba) 2377 return -EINVAL; 2378 2379 real_buffer = kmalloc(4 + len, GFP_KERNEL); 2380 if (!real_buffer) 2381 return -ENOMEM; 2382 memcpy(real_buffer + 4, buffer, len); 2383 len += 4; 2384 real_buffer[0] = 0; 2385 real_buffer[1] = data->medium_type; 2386 real_buffer[2] = data->device_specific; 2387 real_buffer[3] = data->block_descriptor_length; 2388 2389 2390 cmd[0] = MODE_SELECT; 2391 cmd[4] = len; 2392 } 2393 2394 ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len, 2395 sshdr, timeout, retries, NULL); 2396 kfree(real_buffer); 2397 return ret; 2398 } 2399 EXPORT_SYMBOL_GPL(scsi_mode_select); 2400 2401 /** 2402 * scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary. 2403 * @sdev: SCSI device to be queried 2404 * @dbd: set if mode sense will allow block descriptors to be returned 2405 * @modepage: mode page being requested 2406 * @buffer: request buffer (may not be smaller than eight bytes) 2407 * @len: length of request buffer. 2408 * @timeout: command timeout 2409 * @retries: number of retries before failing 2410 * @data: returns a structure abstracting the mode header data 2411 * @sshdr: place to put sense data (or NULL if no sense to be collected). 2412 * must be SCSI_SENSE_BUFFERSIZE big. 2413 * 2414 * Returns zero if unsuccessful, or the header offset (either 4 2415 * or 8 depending on whether a six or ten byte command was 2416 * issued) if successful. 2417 */ 2418 int 2419 scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage, 2420 unsigned char *buffer, int len, int timeout, int retries, 2421 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr) 2422 { 2423 unsigned char cmd[12]; 2424 int use_10_for_ms; 2425 int header_length; 2426 int result, retry_count = retries; 2427 struct scsi_sense_hdr my_sshdr; 2428 2429 memset(data, 0, sizeof(*data)); 2430 memset(&cmd[0], 0, 12); 2431 cmd[1] = dbd & 0x18; /* allows DBD and LLBA bits */ 2432 cmd[2] = modepage; 2433 2434 /* caller might not be interested in sense, but we need it */ 2435 if (!sshdr) 2436 sshdr = &my_sshdr; 2437 2438 retry: 2439 use_10_for_ms = sdev->use_10_for_ms; 2440 2441 if (use_10_for_ms) { 2442 if (len < 8) 2443 len = 8; 2444 2445 cmd[0] = MODE_SENSE_10; 2446 cmd[8] = len; 2447 header_length = 8; 2448 } else { 2449 if (len < 4) 2450 len = 4; 2451 2452 cmd[0] = MODE_SENSE; 2453 cmd[4] = len; 2454 header_length = 4; 2455 } 2456 2457 memset(buffer, 0, len); 2458 2459 result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len, 2460 sshdr, timeout, retries, NULL); 2461 2462 /* This code looks awful: what it's doing is making sure an 2463 * ILLEGAL REQUEST sense return identifies the actual command 2464 * byte as the problem. MODE_SENSE commands can return 2465 * ILLEGAL REQUEST if the code page isn't supported */ 2466 2467 if (use_10_for_ms && !scsi_status_is_good(result) && 2468 (driver_byte(result) & DRIVER_SENSE)) { 2469 if (scsi_sense_valid(sshdr)) { 2470 if ((sshdr->sense_key == ILLEGAL_REQUEST) && 2471 (sshdr->asc == 0x20) && (sshdr->ascq == 0)) { 2472 /* 2473 * Invalid command operation code 2474 */ 2475 sdev->use_10_for_ms = 0; 2476 goto retry; 2477 } 2478 } 2479 } 2480 2481 if(scsi_status_is_good(result)) { 2482 if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b && 2483 (modepage == 6 || modepage == 8))) { 2484 /* Initio breakage? */ 2485 header_length = 0; 2486 data->length = 13; 2487 data->medium_type = 0; 2488 data->device_specific = 0; 2489 data->longlba = 0; 2490 data->block_descriptor_length = 0; 2491 } else if(use_10_for_ms) { 2492 data->length = buffer[0]*256 + buffer[1] + 2; 2493 data->medium_type = buffer[2]; 2494 data->device_specific = buffer[3]; 2495 data->longlba = buffer[4] & 0x01; 2496 data->block_descriptor_length = buffer[6]*256 2497 + buffer[7]; 2498 } else { 2499 data->length = buffer[0] + 1; 2500 data->medium_type = buffer[1]; 2501 data->device_specific = buffer[2]; 2502 data->block_descriptor_length = buffer[3]; 2503 } 2504 data->header_length = header_length; 2505 } else if ((status_byte(result) == CHECK_CONDITION) && 2506 scsi_sense_valid(sshdr) && 2507 sshdr->sense_key == UNIT_ATTENTION && retry_count) { 2508 retry_count--; 2509 goto retry; 2510 } 2511 2512 return result; 2513 } 2514 EXPORT_SYMBOL(scsi_mode_sense); 2515 2516 /** 2517 * scsi_test_unit_ready - test if unit is ready 2518 * @sdev: scsi device to change the state of. 2519 * @timeout: command timeout 2520 * @retries: number of retries before failing 2521 * @sshdr_external: Optional pointer to struct scsi_sense_hdr for 2522 * returning sense. Make sure that this is cleared before passing 2523 * in. 2524 * 2525 * Returns zero if unsuccessful or an error if TUR failed. For 2526 * removable media, UNIT_ATTENTION sets ->changed flag. 2527 **/ 2528 int 2529 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries, 2530 struct scsi_sense_hdr *sshdr_external) 2531 { 2532 char cmd[] = { 2533 TEST_UNIT_READY, 0, 0, 0, 0, 0, 2534 }; 2535 struct scsi_sense_hdr *sshdr; 2536 int result; 2537 2538 if (!sshdr_external) 2539 sshdr = kzalloc(sizeof(*sshdr), GFP_KERNEL); 2540 else 2541 sshdr = sshdr_external; 2542 2543 /* try to eat the UNIT_ATTENTION if there are enough retries */ 2544 do { 2545 result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr, 2546 timeout, retries, NULL); 2547 if (sdev->removable && scsi_sense_valid(sshdr) && 2548 sshdr->sense_key == UNIT_ATTENTION) 2549 sdev->changed = 1; 2550 } while (scsi_sense_valid(sshdr) && 2551 sshdr->sense_key == UNIT_ATTENTION && --retries); 2552 2553 if (!sshdr_external) 2554 kfree(sshdr); 2555 return result; 2556 } 2557 EXPORT_SYMBOL(scsi_test_unit_ready); 2558 2559 /** 2560 * scsi_device_set_state - Take the given device through the device state model. 2561 * @sdev: scsi device to change the state of. 2562 * @state: state to change to. 2563 * 2564 * Returns zero if unsuccessful or an error if the requested 2565 * transition is illegal. 2566 */ 2567 int 2568 scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state) 2569 { 2570 enum scsi_device_state oldstate = sdev->sdev_state; 2571 2572 if (state == oldstate) 2573 return 0; 2574 2575 switch (state) { 2576 case SDEV_CREATED: 2577 switch (oldstate) { 2578 case SDEV_CREATED_BLOCK: 2579 break; 2580 default: 2581 goto illegal; 2582 } 2583 break; 2584 2585 case SDEV_RUNNING: 2586 switch (oldstate) { 2587 case SDEV_CREATED: 2588 case SDEV_OFFLINE: 2589 case SDEV_TRANSPORT_OFFLINE: 2590 case SDEV_QUIESCE: 2591 case SDEV_BLOCK: 2592 break; 2593 default: 2594 goto illegal; 2595 } 2596 break; 2597 2598 case SDEV_QUIESCE: 2599 switch (oldstate) { 2600 case SDEV_RUNNING: 2601 case SDEV_OFFLINE: 2602 case SDEV_TRANSPORT_OFFLINE: 2603 break; 2604 default: 2605 goto illegal; 2606 } 2607 break; 2608 2609 case SDEV_OFFLINE: 2610 case SDEV_TRANSPORT_OFFLINE: 2611 switch (oldstate) { 2612 case SDEV_CREATED: 2613 case SDEV_RUNNING: 2614 case SDEV_QUIESCE: 2615 case SDEV_BLOCK: 2616 break; 2617 default: 2618 goto illegal; 2619 } 2620 break; 2621 2622 case SDEV_BLOCK: 2623 switch (oldstate) { 2624 case SDEV_RUNNING: 2625 case SDEV_CREATED_BLOCK: 2626 break; 2627 default: 2628 goto illegal; 2629 } 2630 break; 2631 2632 case SDEV_CREATED_BLOCK: 2633 switch (oldstate) { 2634 case SDEV_CREATED: 2635 break; 2636 default: 2637 goto illegal; 2638 } 2639 break; 2640 2641 case SDEV_CANCEL: 2642 switch (oldstate) { 2643 case SDEV_CREATED: 2644 case SDEV_RUNNING: 2645 case SDEV_QUIESCE: 2646 case SDEV_OFFLINE: 2647 case SDEV_TRANSPORT_OFFLINE: 2648 case SDEV_BLOCK: 2649 break; 2650 default: 2651 goto illegal; 2652 } 2653 break; 2654 2655 case SDEV_DEL: 2656 switch (oldstate) { 2657 case SDEV_CREATED: 2658 case SDEV_RUNNING: 2659 case SDEV_OFFLINE: 2660 case SDEV_TRANSPORT_OFFLINE: 2661 case SDEV_CANCEL: 2662 case SDEV_CREATED_BLOCK: 2663 break; 2664 default: 2665 goto illegal; 2666 } 2667 break; 2668 2669 } 2670 sdev->sdev_state = state; 2671 return 0; 2672 2673 illegal: 2674 SCSI_LOG_ERROR_RECOVERY(1, 2675 sdev_printk(KERN_ERR, sdev, 2676 "Illegal state transition %s->%s", 2677 scsi_device_state_name(oldstate), 2678 scsi_device_state_name(state)) 2679 ); 2680 return -EINVAL; 2681 } 2682 EXPORT_SYMBOL(scsi_device_set_state); 2683 2684 /** 2685 * sdev_evt_emit - emit a single SCSI device uevent 2686 * @sdev: associated SCSI device 2687 * @evt: event to emit 2688 * 2689 * Send a single uevent (scsi_event) to the associated scsi_device. 2690 */ 2691 static void scsi_evt_emit(struct scsi_device *sdev, struct scsi_event *evt) 2692 { 2693 int idx = 0; 2694 char *envp[3]; 2695 2696 switch (evt->evt_type) { 2697 case SDEV_EVT_MEDIA_CHANGE: 2698 envp[idx++] = "SDEV_MEDIA_CHANGE=1"; 2699 break; 2700 case SDEV_EVT_INQUIRY_CHANGE_REPORTED: 2701 envp[idx++] = "SDEV_UA=INQUIRY_DATA_HAS_CHANGED"; 2702 break; 2703 case SDEV_EVT_CAPACITY_CHANGE_REPORTED: 2704 envp[idx++] = "SDEV_UA=CAPACITY_DATA_HAS_CHANGED"; 2705 break; 2706 case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED: 2707 envp[idx++] = "SDEV_UA=THIN_PROVISIONING_SOFT_THRESHOLD_REACHED"; 2708 break; 2709 case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED: 2710 envp[idx++] = "SDEV_UA=MODE_PARAMETERS_CHANGED"; 2711 break; 2712 case SDEV_EVT_LUN_CHANGE_REPORTED: 2713 envp[idx++] = "SDEV_UA=REPORTED_LUNS_DATA_HAS_CHANGED"; 2714 break; 2715 case SDEV_EVT_ALUA_STATE_CHANGE_REPORTED: 2716 envp[idx++] = "SDEV_UA=ASYMMETRIC_ACCESS_STATE_CHANGED"; 2717 break; 2718 default: 2719 /* do nothing */ 2720 break; 2721 } 2722 2723 envp[idx++] = NULL; 2724 2725 kobject_uevent_env(&sdev->sdev_gendev.kobj, KOBJ_CHANGE, envp); 2726 } 2727 2728 /** 2729 * sdev_evt_thread - send a uevent for each scsi event 2730 * @work: work struct for scsi_device 2731 * 2732 * Dispatch queued events to their associated scsi_device kobjects 2733 * as uevents. 2734 */ 2735 void scsi_evt_thread(struct work_struct *work) 2736 { 2737 struct scsi_device *sdev; 2738 enum scsi_device_event evt_type; 2739 LIST_HEAD(event_list); 2740 2741 sdev = container_of(work, struct scsi_device, event_work); 2742 2743 for (evt_type = SDEV_EVT_FIRST; evt_type <= SDEV_EVT_LAST; evt_type++) 2744 if (test_and_clear_bit(evt_type, sdev->pending_events)) 2745 sdev_evt_send_simple(sdev, evt_type, GFP_KERNEL); 2746 2747 while (1) { 2748 struct scsi_event *evt; 2749 struct list_head *this, *tmp; 2750 unsigned long flags; 2751 2752 spin_lock_irqsave(&sdev->list_lock, flags); 2753 list_splice_init(&sdev->event_list, &event_list); 2754 spin_unlock_irqrestore(&sdev->list_lock, flags); 2755 2756 if (list_empty(&event_list)) 2757 break; 2758 2759 list_for_each_safe(this, tmp, &event_list) { 2760 evt = list_entry(this, struct scsi_event, node); 2761 list_del(&evt->node); 2762 scsi_evt_emit(sdev, evt); 2763 kfree(evt); 2764 } 2765 } 2766 } 2767 2768 /** 2769 * sdev_evt_send - send asserted event to uevent thread 2770 * @sdev: scsi_device event occurred on 2771 * @evt: event to send 2772 * 2773 * Assert scsi device event asynchronously. 2774 */ 2775 void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt) 2776 { 2777 unsigned long flags; 2778 2779 #if 0 2780 /* FIXME: currently this check eliminates all media change events 2781 * for polled devices. Need to update to discriminate between AN 2782 * and polled events */ 2783 if (!test_bit(evt->evt_type, sdev->supported_events)) { 2784 kfree(evt); 2785 return; 2786 } 2787 #endif 2788 2789 spin_lock_irqsave(&sdev->list_lock, flags); 2790 list_add_tail(&evt->node, &sdev->event_list); 2791 schedule_work(&sdev->event_work); 2792 spin_unlock_irqrestore(&sdev->list_lock, flags); 2793 } 2794 EXPORT_SYMBOL_GPL(sdev_evt_send); 2795 2796 /** 2797 * sdev_evt_alloc - allocate a new scsi event 2798 * @evt_type: type of event to allocate 2799 * @gfpflags: GFP flags for allocation 2800 * 2801 * Allocates and returns a new scsi_event. 2802 */ 2803 struct scsi_event *sdev_evt_alloc(enum scsi_device_event evt_type, 2804 gfp_t gfpflags) 2805 { 2806 struct scsi_event *evt = kzalloc(sizeof(struct scsi_event), gfpflags); 2807 if (!evt) 2808 return NULL; 2809 2810 evt->evt_type = evt_type; 2811 INIT_LIST_HEAD(&evt->node); 2812 2813 /* evt_type-specific initialization, if any */ 2814 switch (evt_type) { 2815 case SDEV_EVT_MEDIA_CHANGE: 2816 case SDEV_EVT_INQUIRY_CHANGE_REPORTED: 2817 case SDEV_EVT_CAPACITY_CHANGE_REPORTED: 2818 case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED: 2819 case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED: 2820 case SDEV_EVT_LUN_CHANGE_REPORTED: 2821 case SDEV_EVT_ALUA_STATE_CHANGE_REPORTED: 2822 default: 2823 /* do nothing */ 2824 break; 2825 } 2826 2827 return evt; 2828 } 2829 EXPORT_SYMBOL_GPL(sdev_evt_alloc); 2830 2831 /** 2832 * sdev_evt_send_simple - send asserted event to uevent thread 2833 * @sdev: scsi_device event occurred on 2834 * @evt_type: type of event to send 2835 * @gfpflags: GFP flags for allocation 2836 * 2837 * Assert scsi device event asynchronously, given an event type. 2838 */ 2839 void sdev_evt_send_simple(struct scsi_device *sdev, 2840 enum scsi_device_event evt_type, gfp_t gfpflags) 2841 { 2842 struct scsi_event *evt = sdev_evt_alloc(evt_type, gfpflags); 2843 if (!evt) { 2844 sdev_printk(KERN_ERR, sdev, "event %d eaten due to OOM\n", 2845 evt_type); 2846 return; 2847 } 2848 2849 sdev_evt_send(sdev, evt); 2850 } 2851 EXPORT_SYMBOL_GPL(sdev_evt_send_simple); 2852 2853 /** 2854 * scsi_device_quiesce - Block user issued commands. 2855 * @sdev: scsi device to quiesce. 2856 * 2857 * This works by trying to transition to the SDEV_QUIESCE state 2858 * (which must be a legal transition). When the device is in this 2859 * state, only special requests will be accepted, all others will 2860 * be deferred. Since special requests may also be requeued requests, 2861 * a successful return doesn't guarantee the device will be 2862 * totally quiescent. 2863 * 2864 * Must be called with user context, may sleep. 2865 * 2866 * Returns zero if unsuccessful or an error if not. 2867 */ 2868 int 2869 scsi_device_quiesce(struct scsi_device *sdev) 2870 { 2871 int err = scsi_device_set_state(sdev, SDEV_QUIESCE); 2872 if (err) 2873 return err; 2874 2875 scsi_run_queue(sdev->request_queue); 2876 while (atomic_read(&sdev->device_busy)) { 2877 msleep_interruptible(200); 2878 scsi_run_queue(sdev->request_queue); 2879 } 2880 return 0; 2881 } 2882 EXPORT_SYMBOL(scsi_device_quiesce); 2883 2884 /** 2885 * scsi_device_resume - Restart user issued commands to a quiesced device. 2886 * @sdev: scsi device to resume. 2887 * 2888 * Moves the device from quiesced back to running and restarts the 2889 * queues. 2890 * 2891 * Must be called with user context, may sleep. 2892 */ 2893 void scsi_device_resume(struct scsi_device *sdev) 2894 { 2895 /* check if the device state was mutated prior to resume, and if 2896 * so assume the state is being managed elsewhere (for example 2897 * device deleted during suspend) 2898 */ 2899 if (sdev->sdev_state != SDEV_QUIESCE || 2900 scsi_device_set_state(sdev, SDEV_RUNNING)) 2901 return; 2902 scsi_run_queue(sdev->request_queue); 2903 } 2904 EXPORT_SYMBOL(scsi_device_resume); 2905 2906 static void 2907 device_quiesce_fn(struct scsi_device *sdev, void *data) 2908 { 2909 scsi_device_quiesce(sdev); 2910 } 2911 2912 void 2913 scsi_target_quiesce(struct scsi_target *starget) 2914 { 2915 starget_for_each_device(starget, NULL, device_quiesce_fn); 2916 } 2917 EXPORT_SYMBOL(scsi_target_quiesce); 2918 2919 static void 2920 device_resume_fn(struct scsi_device *sdev, void *data) 2921 { 2922 scsi_device_resume(sdev); 2923 } 2924 2925 void 2926 scsi_target_resume(struct scsi_target *starget) 2927 { 2928 starget_for_each_device(starget, NULL, device_resume_fn); 2929 } 2930 EXPORT_SYMBOL(scsi_target_resume); 2931 2932 /** 2933 * scsi_internal_device_block - internal function to put a device temporarily into the SDEV_BLOCK state 2934 * @sdev: device to block 2935 * 2936 * Block request made by scsi lld's to temporarily stop all 2937 * scsi commands on the specified device. Called from interrupt 2938 * or normal process context. 2939 * 2940 * Returns zero if successful or error if not 2941 * 2942 * Notes: 2943 * This routine transitions the device to the SDEV_BLOCK state 2944 * (which must be a legal transition). When the device is in this 2945 * state, all commands are deferred until the scsi lld reenables 2946 * the device with scsi_device_unblock or device_block_tmo fires. 2947 */ 2948 int 2949 scsi_internal_device_block(struct scsi_device *sdev) 2950 { 2951 struct request_queue *q = sdev->request_queue; 2952 unsigned long flags; 2953 int err = 0; 2954 2955 err = scsi_device_set_state(sdev, SDEV_BLOCK); 2956 if (err) { 2957 err = scsi_device_set_state(sdev, SDEV_CREATED_BLOCK); 2958 2959 if (err) 2960 return err; 2961 } 2962 2963 /* 2964 * The device has transitioned to SDEV_BLOCK. Stop the 2965 * block layer from calling the midlayer with this device's 2966 * request queue. 2967 */ 2968 if (q->mq_ops) { 2969 blk_mq_stop_hw_queues(q); 2970 } else { 2971 spin_lock_irqsave(q->queue_lock, flags); 2972 blk_stop_queue(q); 2973 spin_unlock_irqrestore(q->queue_lock, flags); 2974 } 2975 2976 return 0; 2977 } 2978 EXPORT_SYMBOL_GPL(scsi_internal_device_block); 2979 2980 /** 2981 * scsi_internal_device_unblock - resume a device after a block request 2982 * @sdev: device to resume 2983 * @new_state: state to set devices to after unblocking 2984 * 2985 * Called by scsi lld's or the midlayer to restart the device queue 2986 * for the previously suspended scsi device. Called from interrupt or 2987 * normal process context. 2988 * 2989 * Returns zero if successful or error if not. 2990 * 2991 * Notes: 2992 * This routine transitions the device to the SDEV_RUNNING state 2993 * or to one of the offline states (which must be a legal transition) 2994 * allowing the midlayer to goose the queue for this device. 2995 */ 2996 int 2997 scsi_internal_device_unblock(struct scsi_device *sdev, 2998 enum scsi_device_state new_state) 2999 { 3000 struct request_queue *q = sdev->request_queue; 3001 unsigned long flags; 3002 3003 /* 3004 * Try to transition the scsi device to SDEV_RUNNING or one of the 3005 * offlined states and goose the device queue if successful. 3006 */ 3007 if ((sdev->sdev_state == SDEV_BLOCK) || 3008 (sdev->sdev_state == SDEV_TRANSPORT_OFFLINE)) 3009 sdev->sdev_state = new_state; 3010 else if (sdev->sdev_state == SDEV_CREATED_BLOCK) { 3011 if (new_state == SDEV_TRANSPORT_OFFLINE || 3012 new_state == SDEV_OFFLINE) 3013 sdev->sdev_state = new_state; 3014 else 3015 sdev->sdev_state = SDEV_CREATED; 3016 } else if (sdev->sdev_state != SDEV_CANCEL && 3017 sdev->sdev_state != SDEV_OFFLINE) 3018 return -EINVAL; 3019 3020 if (q->mq_ops) { 3021 blk_mq_start_stopped_hw_queues(q, false); 3022 } else { 3023 spin_lock_irqsave(q->queue_lock, flags); 3024 blk_start_queue(q); 3025 spin_unlock_irqrestore(q->queue_lock, flags); 3026 } 3027 3028 return 0; 3029 } 3030 EXPORT_SYMBOL_GPL(scsi_internal_device_unblock); 3031 3032 static void 3033 device_block(struct scsi_device *sdev, void *data) 3034 { 3035 scsi_internal_device_block(sdev); 3036 } 3037 3038 static int 3039 target_block(struct device *dev, void *data) 3040 { 3041 if (scsi_is_target_device(dev)) 3042 starget_for_each_device(to_scsi_target(dev), NULL, 3043 device_block); 3044 return 0; 3045 } 3046 3047 void 3048 scsi_target_block(struct device *dev) 3049 { 3050 if (scsi_is_target_device(dev)) 3051 starget_for_each_device(to_scsi_target(dev), NULL, 3052 device_block); 3053 else 3054 device_for_each_child(dev, NULL, target_block); 3055 } 3056 EXPORT_SYMBOL_GPL(scsi_target_block); 3057 3058 static void 3059 device_unblock(struct scsi_device *sdev, void *data) 3060 { 3061 scsi_internal_device_unblock(sdev, *(enum scsi_device_state *)data); 3062 } 3063 3064 static int 3065 target_unblock(struct device *dev, void *data) 3066 { 3067 if (scsi_is_target_device(dev)) 3068 starget_for_each_device(to_scsi_target(dev), data, 3069 device_unblock); 3070 return 0; 3071 } 3072 3073 void 3074 scsi_target_unblock(struct device *dev, enum scsi_device_state new_state) 3075 { 3076 if (scsi_is_target_device(dev)) 3077 starget_for_each_device(to_scsi_target(dev), &new_state, 3078 device_unblock); 3079 else 3080 device_for_each_child(dev, &new_state, target_unblock); 3081 } 3082 EXPORT_SYMBOL_GPL(scsi_target_unblock); 3083 3084 /** 3085 * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt 3086 * @sgl: scatter-gather list 3087 * @sg_count: number of segments in sg 3088 * @offset: offset in bytes into sg, on return offset into the mapped area 3089 * @len: bytes to map, on return number of bytes mapped 3090 * 3091 * Returns virtual address of the start of the mapped page 3092 */ 3093 void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count, 3094 size_t *offset, size_t *len) 3095 { 3096 int i; 3097 size_t sg_len = 0, len_complete = 0; 3098 struct scatterlist *sg; 3099 struct page *page; 3100 3101 WARN_ON(!irqs_disabled()); 3102 3103 for_each_sg(sgl, sg, sg_count, i) { 3104 len_complete = sg_len; /* Complete sg-entries */ 3105 sg_len += sg->length; 3106 if (sg_len > *offset) 3107 break; 3108 } 3109 3110 if (unlikely(i == sg_count)) { 3111 printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, " 3112 "elements %d\n", 3113 __func__, sg_len, *offset, sg_count); 3114 WARN_ON(1); 3115 return NULL; 3116 } 3117 3118 /* Offset starting from the beginning of first page in this sg-entry */ 3119 *offset = *offset - len_complete + sg->offset; 3120 3121 /* Assumption: contiguous pages can be accessed as "page + i" */ 3122 page = nth_page(sg_page(sg), (*offset >> PAGE_SHIFT)); 3123 *offset &= ~PAGE_MASK; 3124 3125 /* Bytes in this sg-entry from *offset to the end of the page */ 3126 sg_len = PAGE_SIZE - *offset; 3127 if (*len > sg_len) 3128 *len = sg_len; 3129 3130 return kmap_atomic(page); 3131 } 3132 EXPORT_SYMBOL(scsi_kmap_atomic_sg); 3133 3134 /** 3135 * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously mapped with scsi_kmap_atomic_sg 3136 * @virt: virtual address to be unmapped 3137 */ 3138 void scsi_kunmap_atomic_sg(void *virt) 3139 { 3140 kunmap_atomic(virt); 3141 } 3142 EXPORT_SYMBOL(scsi_kunmap_atomic_sg); 3143 3144 void sdev_disable_disk_events(struct scsi_device *sdev) 3145 { 3146 atomic_inc(&sdev->disk_events_disable_depth); 3147 } 3148 EXPORT_SYMBOL(sdev_disable_disk_events); 3149 3150 void sdev_enable_disk_events(struct scsi_device *sdev) 3151 { 3152 if (WARN_ON_ONCE(atomic_read(&sdev->disk_events_disable_depth) <= 0)) 3153 return; 3154 atomic_dec(&sdev->disk_events_disable_depth); 3155 } 3156 EXPORT_SYMBOL(sdev_enable_disk_events); 3157