1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2016 Jakub Klama <jceel@FreeBSD.org>. 5 * Copyright (c) 2018 Marcelo Araujo <araujo@FreeBSD.org>. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer 13 * in this position and unchanged. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 33 #include <sys/param.h> 34 #include <sys/linker_set.h> 35 #include <sys/types.h> 36 #include <sys/uio.h> 37 #include <sys/time.h> 38 #include <sys/queue.h> 39 #include <sys/sbuf.h> 40 41 #include <errno.h> 42 #include <fcntl.h> 43 #include <stdio.h> 44 #include <stdlib.h> 45 #include <stdbool.h> 46 #include <string.h> 47 #include <unistd.h> 48 #include <assert.h> 49 #include <pthread.h> 50 #include <pthread_np.h> 51 52 #include <cam/scsi/scsi_all.h> 53 #include <cam/scsi/scsi_message.h> 54 #include <cam/ctl/ctl.h> 55 #include <cam/ctl/ctl_io.h> 56 #include <cam/ctl/ctl_backend.h> 57 #include <cam/ctl/ctl_ioctl.h> 58 #include <cam/ctl/ctl_util.h> 59 #include <cam/ctl/ctl_scsi_all.h> 60 #include <camlib.h> 61 62 #include "bhyverun.h" 63 #include "config.h" 64 #include "debug.h" 65 #include "pci_emul.h" 66 #include "virtio.h" 67 #include "iov.h" 68 69 #define VTSCSI_RINGSZ 64 70 #define VTSCSI_REQUESTQ 1 71 #define VTSCSI_THR_PER_Q 16 72 #define VTSCSI_MAXQ (VTSCSI_REQUESTQ + 2) 73 #define VTSCSI_MAXSEG 64 74 75 #define VTSCSI_IN_HEADER_LEN(_sc) \ 76 (sizeof(struct pci_vtscsi_req_cmd_rd) + _sc->vss_config.cdb_size) 77 78 #define VTSCSI_OUT_HEADER_LEN(_sc) \ 79 (sizeof(struct pci_vtscsi_req_cmd_wr) + _sc->vss_config.sense_size) 80 81 #define VIRTIO_SCSI_MAX_CHANNEL 0 82 #define VIRTIO_SCSI_MAX_TARGET 0 83 #define VIRTIO_SCSI_MAX_LUN 16383 84 85 #define VIRTIO_SCSI_F_INOUT (1 << 0) 86 #define VIRTIO_SCSI_F_HOTPLUG (1 << 1) 87 #define VIRTIO_SCSI_F_CHANGE (1 << 2) 88 89 static int pci_vtscsi_debug = 0; 90 #define WPRINTF(msg, params...) PRINTLN("virtio-scsi: " msg, ##params) 91 #define DPRINTF(msg, params...) if (pci_vtscsi_debug) WPRINTF(msg, ##params) 92 93 struct pci_vtscsi_config { 94 uint32_t num_queues; 95 uint32_t seg_max; 96 uint32_t max_sectors; 97 uint32_t cmd_per_lun; 98 uint32_t event_info_size; 99 uint32_t sense_size; 100 uint32_t cdb_size; 101 uint16_t max_channel; 102 uint16_t max_target; 103 uint32_t max_lun; 104 } __attribute__((packed)); 105 106 struct pci_vtscsi_queue { 107 struct pci_vtscsi_softc * vsq_sc; 108 struct vqueue_info * vsq_vq; 109 pthread_mutex_t vsq_mtx; 110 pthread_mutex_t vsq_qmtx; 111 pthread_cond_t vsq_cv; 112 STAILQ_HEAD(, pci_vtscsi_request) vsq_requests; 113 LIST_HEAD(, pci_vtscsi_worker) vsq_workers; 114 }; 115 116 struct pci_vtscsi_worker { 117 struct pci_vtscsi_queue * vsw_queue; 118 pthread_t vsw_thread; 119 bool vsw_exiting; 120 LIST_ENTRY(pci_vtscsi_worker) vsw_link; 121 }; 122 123 struct pci_vtscsi_request { 124 struct pci_vtscsi_queue * vsr_queue; 125 struct iovec vsr_iov_in[VTSCSI_MAXSEG]; 126 int vsr_niov_in; 127 struct iovec vsr_iov_out[VTSCSI_MAXSEG]; 128 int vsr_niov_out; 129 uint32_t vsr_idx; 130 STAILQ_ENTRY(pci_vtscsi_request) vsr_link; 131 }; 132 133 /* 134 * Per-device softc 135 */ 136 struct pci_vtscsi_softc { 137 struct virtio_softc vss_vs; 138 struct vqueue_info vss_vq[VTSCSI_MAXQ]; 139 struct pci_vtscsi_queue vss_queues[VTSCSI_REQUESTQ]; 140 pthread_mutex_t vss_mtx; 141 int vss_iid; 142 int vss_ctl_fd; 143 uint32_t vss_features; 144 struct pci_vtscsi_config vss_config; 145 }; 146 147 #define VIRTIO_SCSI_T_TMF 0 148 #define VIRTIO_SCSI_T_TMF_ABORT_TASK 0 149 #define VIRTIO_SCSI_T_TMF_ABORT_TASK_SET 1 150 #define VIRTIO_SCSI_T_TMF_CLEAR_ACA 2 151 #define VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET 3 152 #define VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET 4 153 #define VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET 5 154 #define VIRTIO_SCSI_T_TMF_QUERY_TASK 6 155 #define VIRTIO_SCSI_T_TMF_QUERY_TASK_SET 7 156 157 /* command-specific response values */ 158 #define VIRTIO_SCSI_S_FUNCTION_COMPLETE 0 159 #define VIRTIO_SCSI_S_FUNCTION_SUCCEEDED 10 160 #define VIRTIO_SCSI_S_FUNCTION_REJECTED 11 161 162 struct pci_vtscsi_ctrl_tmf { 163 uint32_t type; 164 uint32_t subtype; 165 uint8_t lun[8]; 166 uint64_t id; 167 uint8_t response; 168 } __attribute__((packed)); 169 170 #define VIRTIO_SCSI_T_AN_QUERY 1 171 #define VIRTIO_SCSI_EVT_ASYNC_OPERATIONAL_CHANGE 2 172 #define VIRTIO_SCSI_EVT_ASYNC_POWER_MGMT 4 173 #define VIRTIO_SCSI_EVT_ASYNC_EXTERNAL_REQUEST 8 174 #define VIRTIO_SCSI_EVT_ASYNC_MEDIA_CHANGE 16 175 #define VIRTIO_SCSI_EVT_ASYNC_MULTI_HOST 32 176 #define VIRTIO_SCSI_EVT_ASYNC_DEVICE_BUSY 64 177 178 struct pci_vtscsi_ctrl_an { 179 uint32_t type; 180 uint8_t lun[8]; 181 uint32_t event_requested; 182 uint32_t event_actual; 183 uint8_t response; 184 } __attribute__((packed)); 185 186 /* command-specific response values */ 187 #define VIRTIO_SCSI_S_OK 0 188 #define VIRTIO_SCSI_S_OVERRUN 1 189 #define VIRTIO_SCSI_S_ABORTED 2 190 #define VIRTIO_SCSI_S_BAD_TARGET 3 191 #define VIRTIO_SCSI_S_RESET 4 192 #define VIRTIO_SCSI_S_BUSY 5 193 #define VIRTIO_SCSI_S_TRANSPORT_FAILURE 6 194 #define VIRTIO_SCSI_S_TARGET_FAILURE 7 195 #define VIRTIO_SCSI_S_NEXUS_FAILURE 8 196 #define VIRTIO_SCSI_S_FAILURE 9 197 #define VIRTIO_SCSI_S_INCORRECT_LUN 12 198 199 /* task_attr */ 200 #define VIRTIO_SCSI_S_SIMPLE 0 201 #define VIRTIO_SCSI_S_ORDERED 1 202 #define VIRTIO_SCSI_S_HEAD 2 203 #define VIRTIO_SCSI_S_ACA 3 204 205 struct pci_vtscsi_event { 206 uint32_t event; 207 uint8_t lun[8]; 208 uint32_t reason; 209 } __attribute__((packed)); 210 211 struct pci_vtscsi_req_cmd_rd { 212 uint8_t lun[8]; 213 uint64_t id; 214 uint8_t task_attr; 215 uint8_t prio; 216 uint8_t crn; 217 uint8_t cdb[]; 218 } __attribute__((packed)); 219 220 struct pci_vtscsi_req_cmd_wr { 221 uint32_t sense_len; 222 uint32_t residual; 223 uint16_t status_qualifier; 224 uint8_t status; 225 uint8_t response; 226 uint8_t sense[]; 227 } __attribute__((packed)); 228 229 static void *pci_vtscsi_proc(void *); 230 static void pci_vtscsi_reset(void *); 231 static void pci_vtscsi_neg_features(void *, uint64_t); 232 static int pci_vtscsi_cfgread(void *, int, int, uint32_t *); 233 static int pci_vtscsi_cfgwrite(void *, int, int, uint32_t); 234 static inline int pci_vtscsi_get_lun(uint8_t *); 235 static int pci_vtscsi_control_handle(struct pci_vtscsi_softc *, void *, size_t); 236 static int pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *, 237 struct pci_vtscsi_ctrl_tmf *); 238 static int pci_vtscsi_an_handle(struct pci_vtscsi_softc *, 239 struct pci_vtscsi_ctrl_an *); 240 static int pci_vtscsi_request_handle(struct pci_vtscsi_queue *, struct iovec *, 241 int, struct iovec *, int); 242 static void pci_vtscsi_controlq_notify(void *, struct vqueue_info *); 243 static void pci_vtscsi_eventq_notify(void *, struct vqueue_info *); 244 static void pci_vtscsi_requestq_notify(void *, struct vqueue_info *); 245 static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *, 246 struct pci_vtscsi_queue *, int); 247 static int pci_vtscsi_init(struct pci_devinst *, nvlist_t *); 248 249 static struct virtio_consts vtscsi_vi_consts = { 250 .vc_name = "vtscsi", 251 .vc_nvq = VTSCSI_MAXQ, 252 .vc_cfgsize = sizeof(struct pci_vtscsi_config), 253 .vc_reset = pci_vtscsi_reset, 254 .vc_cfgread = pci_vtscsi_cfgread, 255 .vc_cfgwrite = pci_vtscsi_cfgwrite, 256 .vc_apply_features = pci_vtscsi_neg_features, 257 .vc_hv_caps = 0, 258 }; 259 260 static void * 261 pci_vtscsi_proc(void *arg) 262 { 263 struct pci_vtscsi_worker *worker = (struct pci_vtscsi_worker *)arg; 264 struct pci_vtscsi_queue *q = worker->vsw_queue; 265 struct pci_vtscsi_request *req; 266 int iolen; 267 268 for (;;) { 269 pthread_mutex_lock(&q->vsq_mtx); 270 271 while (STAILQ_EMPTY(&q->vsq_requests) 272 && !worker->vsw_exiting) 273 pthread_cond_wait(&q->vsq_cv, &q->vsq_mtx); 274 275 if (worker->vsw_exiting) 276 break; 277 278 req = STAILQ_FIRST(&q->vsq_requests); 279 STAILQ_REMOVE_HEAD(&q->vsq_requests, vsr_link); 280 281 pthread_mutex_unlock(&q->vsq_mtx); 282 iolen = pci_vtscsi_request_handle(q, req->vsr_iov_in, 283 req->vsr_niov_in, req->vsr_iov_out, req->vsr_niov_out); 284 285 pthread_mutex_lock(&q->vsq_qmtx); 286 vq_relchain(q->vsq_vq, req->vsr_idx, iolen); 287 vq_endchains(q->vsq_vq, 0); 288 pthread_mutex_unlock(&q->vsq_qmtx); 289 290 DPRINTF("request <idx=%d> completed", req->vsr_idx); 291 free(req); 292 } 293 294 pthread_mutex_unlock(&q->vsq_mtx); 295 return (NULL); 296 } 297 298 static void 299 pci_vtscsi_reset(void *vsc) 300 { 301 struct pci_vtscsi_softc *sc; 302 303 sc = vsc; 304 305 DPRINTF("device reset requested"); 306 vi_reset_dev(&sc->vss_vs); 307 308 /* initialize config structure */ 309 sc->vss_config = (struct pci_vtscsi_config){ 310 .num_queues = VTSCSI_REQUESTQ, 311 /* Leave room for the request and the response. */ 312 .seg_max = VTSCSI_MAXSEG - 2, 313 .max_sectors = 2, 314 .cmd_per_lun = 1, 315 .event_info_size = sizeof(struct pci_vtscsi_event), 316 .sense_size = 96, 317 .cdb_size = 32, 318 .max_channel = VIRTIO_SCSI_MAX_CHANNEL, 319 .max_target = VIRTIO_SCSI_MAX_TARGET, 320 .max_lun = VIRTIO_SCSI_MAX_LUN 321 }; 322 } 323 324 static void 325 pci_vtscsi_neg_features(void *vsc, uint64_t negotiated_features) 326 { 327 struct pci_vtscsi_softc *sc = vsc; 328 329 sc->vss_features = negotiated_features; 330 } 331 332 static int 333 pci_vtscsi_cfgread(void *vsc, int offset, int size, uint32_t *retval) 334 { 335 struct pci_vtscsi_softc *sc = vsc; 336 void *ptr; 337 338 ptr = (uint8_t *)&sc->vss_config + offset; 339 memcpy(retval, ptr, size); 340 return (0); 341 } 342 343 static int 344 pci_vtscsi_cfgwrite(void *vsc __unused, int offset __unused, int size __unused, 345 uint32_t val __unused) 346 { 347 return (0); 348 } 349 350 static inline int 351 pci_vtscsi_get_lun(uint8_t *lun) 352 { 353 354 return (((lun[2] << 8) | lun[3]) & 0x3fff); 355 } 356 357 static int 358 pci_vtscsi_control_handle(struct pci_vtscsi_softc *sc, void *buf, 359 size_t bufsize) 360 { 361 struct pci_vtscsi_ctrl_tmf *tmf; 362 struct pci_vtscsi_ctrl_an *an; 363 uint32_t type; 364 365 if (bufsize < sizeof(uint32_t)) { 366 WPRINTF("ignoring truncated control request"); 367 return (0); 368 } 369 370 type = *(uint32_t *)buf; 371 372 if (type == VIRTIO_SCSI_T_TMF) { 373 if (bufsize != sizeof(*tmf)) { 374 WPRINTF("ignoring tmf request with size %zu", bufsize); 375 return (0); 376 } 377 tmf = (struct pci_vtscsi_ctrl_tmf *)buf; 378 return (pci_vtscsi_tmf_handle(sc, tmf)); 379 } 380 381 if (type == VIRTIO_SCSI_T_AN_QUERY) { 382 if (bufsize != sizeof(*an)) { 383 WPRINTF("ignoring AN request with size %zu", bufsize); 384 return (0); 385 } 386 an = (struct pci_vtscsi_ctrl_an *)buf; 387 return (pci_vtscsi_an_handle(sc, an)); 388 } 389 390 return (0); 391 } 392 393 static int 394 pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *sc, 395 struct pci_vtscsi_ctrl_tmf *tmf) 396 { 397 union ctl_io *io; 398 int err; 399 400 io = ctl_scsi_alloc_io(sc->vss_iid); 401 ctl_scsi_zero_io(io); 402 403 io->io_hdr.io_type = CTL_IO_TASK; 404 io->io_hdr.nexus.initid = sc->vss_iid; 405 io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(tmf->lun); 406 io->taskio.tag_type = CTL_TAG_SIMPLE; 407 io->taskio.tag_num = tmf->id; 408 io->io_hdr.flags |= CTL_FLAG_USER_TAG; 409 410 switch (tmf->subtype) { 411 case VIRTIO_SCSI_T_TMF_ABORT_TASK: 412 io->taskio.task_action = CTL_TASK_ABORT_TASK; 413 break; 414 415 case VIRTIO_SCSI_T_TMF_ABORT_TASK_SET: 416 io->taskio.task_action = CTL_TASK_ABORT_TASK_SET; 417 break; 418 419 case VIRTIO_SCSI_T_TMF_CLEAR_ACA: 420 io->taskio.task_action = CTL_TASK_CLEAR_ACA; 421 break; 422 423 case VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET: 424 io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET; 425 break; 426 427 case VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET: 428 io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET; 429 break; 430 431 case VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET: 432 io->taskio.task_action = CTL_TASK_LUN_RESET; 433 break; 434 435 case VIRTIO_SCSI_T_TMF_QUERY_TASK: 436 io->taskio.task_action = CTL_TASK_QUERY_TASK; 437 break; 438 439 case VIRTIO_SCSI_T_TMF_QUERY_TASK_SET: 440 io->taskio.task_action = CTL_TASK_QUERY_TASK_SET; 441 break; 442 } 443 444 if (pci_vtscsi_debug) { 445 struct sbuf *sb = sbuf_new_auto(); 446 ctl_io_sbuf(io, sb); 447 sbuf_finish(sb); 448 DPRINTF("%s", sbuf_data(sb)); 449 sbuf_delete(sb); 450 } 451 452 err = ioctl(sc->vss_ctl_fd, CTL_IO, io); 453 if (err != 0) 454 WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); 455 456 tmf->response = io->taskio.task_status; 457 ctl_scsi_free_io(io); 458 return (1); 459 } 460 461 static int 462 pci_vtscsi_an_handle(struct pci_vtscsi_softc *sc __unused, 463 struct pci_vtscsi_ctrl_an *an __unused) 464 { 465 return (0); 466 } 467 468 static int 469 pci_vtscsi_request_handle(struct pci_vtscsi_queue *q, struct iovec *iov_in, 470 int niov_in, struct iovec *iov_out, int niov_out) 471 { 472 struct pci_vtscsi_softc *sc = q->vsq_sc; 473 struct pci_vtscsi_req_cmd_rd *cmd_rd = NULL; 474 struct pci_vtscsi_req_cmd_wr *cmd_wr; 475 struct iovec data_iov_in[VTSCSI_MAXSEG], data_iov_out[VTSCSI_MAXSEG]; 476 union ctl_io *io; 477 int data_niov_in, data_niov_out; 478 void *ext_data_ptr = NULL; 479 uint32_t ext_data_len = 0, ext_sg_entries = 0; 480 int err, nxferred; 481 482 if (count_iov(iov_out, niov_out) < VTSCSI_OUT_HEADER_LEN(sc)) { 483 WPRINTF("ignoring request with insufficient output"); 484 return (0); 485 } 486 if (count_iov(iov_in, niov_in) < VTSCSI_IN_HEADER_LEN(sc)) { 487 WPRINTF("ignoring request with incomplete header"); 488 return (0); 489 } 490 491 seek_iov(iov_in, niov_in, data_iov_in, &data_niov_in, 492 VTSCSI_IN_HEADER_LEN(sc)); 493 seek_iov(iov_out, niov_out, data_iov_out, &data_niov_out, 494 VTSCSI_OUT_HEADER_LEN(sc)); 495 496 truncate_iov(iov_in, &niov_in, VTSCSI_IN_HEADER_LEN(sc)); 497 truncate_iov(iov_out, &niov_out, VTSCSI_OUT_HEADER_LEN(sc)); 498 iov_to_buf(iov_in, niov_in, (void **)&cmd_rd); 499 500 cmd_wr = calloc(1, VTSCSI_OUT_HEADER_LEN(sc)); 501 io = ctl_scsi_alloc_io(sc->vss_iid); 502 ctl_scsi_zero_io(io); 503 504 io->io_hdr.nexus.initid = sc->vss_iid; 505 io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(cmd_rd->lun); 506 507 io->io_hdr.io_type = CTL_IO_SCSI; 508 509 if (data_niov_in > 0) { 510 ext_data_ptr = (void *)data_iov_in; 511 ext_sg_entries = data_niov_in; 512 ext_data_len = count_iov(data_iov_in, data_niov_in); 513 io->io_hdr.flags |= CTL_FLAG_DATA_OUT; 514 } else if (data_niov_out > 0) { 515 ext_data_ptr = (void *)data_iov_out; 516 ext_sg_entries = data_niov_out; 517 ext_data_len = count_iov(data_iov_out, data_niov_out); 518 io->io_hdr.flags |= CTL_FLAG_DATA_IN; 519 } 520 521 io->scsiio.sense_len = sc->vss_config.sense_size; 522 io->scsiio.tag_num = cmd_rd->id; 523 io->io_hdr.flags |= CTL_FLAG_USER_TAG; 524 switch (cmd_rd->task_attr) { 525 case VIRTIO_SCSI_S_ORDERED: 526 io->scsiio.tag_type = CTL_TAG_ORDERED; 527 break; 528 case VIRTIO_SCSI_S_HEAD: 529 io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE; 530 break; 531 case VIRTIO_SCSI_S_ACA: 532 io->scsiio.tag_type = CTL_TAG_ACA; 533 break; 534 case VIRTIO_SCSI_S_SIMPLE: 535 default: 536 io->scsiio.tag_type = CTL_TAG_SIMPLE; 537 break; 538 } 539 io->scsiio.ext_sg_entries = ext_sg_entries; 540 io->scsiio.ext_data_ptr = ext_data_ptr; 541 io->scsiio.ext_data_len = ext_data_len; 542 io->scsiio.ext_data_filled = 0; 543 io->scsiio.cdb_len = sc->vss_config.cdb_size; 544 memcpy(io->scsiio.cdb, cmd_rd->cdb, sc->vss_config.cdb_size); 545 546 if (pci_vtscsi_debug) { 547 struct sbuf *sb = sbuf_new_auto(); 548 ctl_io_sbuf(io, sb); 549 sbuf_finish(sb); 550 DPRINTF("%s", sbuf_data(sb)); 551 sbuf_delete(sb); 552 } 553 554 err = ioctl(sc->vss_ctl_fd, CTL_IO, io); 555 if (err != 0) { 556 WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); 557 cmd_wr->response = VIRTIO_SCSI_S_FAILURE; 558 } else { 559 cmd_wr->sense_len = MIN(io->scsiio.sense_len, 560 sc->vss_config.sense_size); 561 cmd_wr->residual = ext_data_len - io->scsiio.ext_data_filled; 562 cmd_wr->status = io->scsiio.scsi_status; 563 cmd_wr->response = VIRTIO_SCSI_S_OK; 564 memcpy(&cmd_wr->sense, &io->scsiio.sense_data, 565 cmd_wr->sense_len); 566 } 567 568 buf_to_iov(cmd_wr, VTSCSI_OUT_HEADER_LEN(sc), iov_out, niov_out, 0); 569 nxferred = VTSCSI_OUT_HEADER_LEN(sc) + io->scsiio.ext_data_filled; 570 free(cmd_rd); 571 free(cmd_wr); 572 ctl_scsi_free_io(io); 573 return (nxferred); 574 } 575 576 static void 577 pci_vtscsi_controlq_notify(void *vsc, struct vqueue_info *vq) 578 { 579 struct pci_vtscsi_softc *sc; 580 struct iovec iov[VTSCSI_MAXSEG]; 581 struct vi_req req; 582 void *buf = NULL; 583 size_t bufsize; 584 int iolen, n; 585 586 sc = vsc; 587 588 while (vq_has_descs(vq)) { 589 n = vq_getchain(vq, iov, VTSCSI_MAXSEG, &req); 590 assert(n >= 1 && n <= VTSCSI_MAXSEG); 591 592 bufsize = iov_to_buf(iov, n, &buf); 593 iolen = pci_vtscsi_control_handle(sc, buf, bufsize); 594 buf_to_iov((uint8_t *)buf + bufsize - iolen, iolen, iov, n, 595 bufsize - iolen); 596 597 /* 598 * Release this chain and handle more 599 */ 600 vq_relchain(vq, req.idx, iolen); 601 } 602 vq_endchains(vq, 1); /* Generate interrupt if appropriate. */ 603 free(buf); 604 } 605 606 static void 607 pci_vtscsi_eventq_notify(void *vsc __unused, struct vqueue_info *vq) 608 { 609 vq_kick_disable(vq); 610 } 611 612 static void 613 pci_vtscsi_requestq_notify(void *vsc, struct vqueue_info *vq) 614 { 615 struct pci_vtscsi_softc *sc; 616 struct pci_vtscsi_queue *q; 617 struct pci_vtscsi_request *req; 618 struct iovec iov[VTSCSI_MAXSEG]; 619 struct vi_req vireq; 620 int n; 621 622 sc = vsc; 623 q = &sc->vss_queues[vq->vq_num - 2]; 624 625 while (vq_has_descs(vq)) { 626 n = vq_getchain(vq, iov, VTSCSI_MAXSEG, &vireq); 627 assert(n >= 1 && n <= VTSCSI_MAXSEG); 628 629 req = calloc(1, sizeof(struct pci_vtscsi_request)); 630 req->vsr_idx = vireq.idx; 631 req->vsr_queue = q; 632 req->vsr_niov_in = vireq.readable; 633 req->vsr_niov_out = vireq.writable; 634 memcpy(req->vsr_iov_in, iov, 635 req->vsr_niov_in * sizeof(struct iovec)); 636 memcpy(req->vsr_iov_out, iov + vireq.readable, 637 req->vsr_niov_out * sizeof(struct iovec)); 638 639 pthread_mutex_lock(&q->vsq_mtx); 640 STAILQ_INSERT_TAIL(&q->vsq_requests, req, vsr_link); 641 pthread_cond_signal(&q->vsq_cv); 642 pthread_mutex_unlock(&q->vsq_mtx); 643 644 DPRINTF("request <idx=%d> enqueued", vireq.idx); 645 } 646 } 647 648 static int 649 pci_vtscsi_init_queue(struct pci_vtscsi_softc *sc, 650 struct pci_vtscsi_queue *queue, int num) 651 { 652 struct pci_vtscsi_worker *worker; 653 char tname[MAXCOMLEN + 1]; 654 int i; 655 656 queue->vsq_sc = sc; 657 queue->vsq_vq = &sc->vss_vq[num + 2]; 658 659 pthread_mutex_init(&queue->vsq_mtx, NULL); 660 pthread_mutex_init(&queue->vsq_qmtx, NULL); 661 pthread_cond_init(&queue->vsq_cv, NULL); 662 STAILQ_INIT(&queue->vsq_requests); 663 LIST_INIT(&queue->vsq_workers); 664 665 for (i = 0; i < VTSCSI_THR_PER_Q; i++) { 666 worker = calloc(1, sizeof(struct pci_vtscsi_worker)); 667 worker->vsw_queue = queue; 668 669 pthread_create(&worker->vsw_thread, NULL, &pci_vtscsi_proc, 670 (void *)worker); 671 672 snprintf(tname, sizeof(tname), "vtscsi:%d-%d", num, i); 673 pthread_set_name_np(worker->vsw_thread, tname); 674 LIST_INSERT_HEAD(&queue->vsq_workers, worker, vsw_link); 675 } 676 677 return (0); 678 } 679 680 static int 681 pci_vtscsi_legacy_config(nvlist_t *nvl, const char *opts) 682 { 683 char *cp, *devname; 684 685 if (opts == NULL) 686 return (0); 687 688 cp = strchr(opts, ','); 689 if (cp == NULL) { 690 set_config_value_node(nvl, "dev", opts); 691 return (0); 692 } 693 devname = strndup(opts, cp - opts); 694 set_config_value_node(nvl, "dev", devname); 695 free(devname); 696 return (pci_parse_legacy_config(nvl, cp + 1)); 697 } 698 699 static int 700 pci_vtscsi_init(struct pci_devinst *pi, nvlist_t *nvl) 701 { 702 struct pci_vtscsi_softc *sc; 703 const char *devname, *value; 704 int i; 705 706 sc = calloc(1, sizeof(struct pci_vtscsi_softc)); 707 value = get_config_value_node(nvl, "iid"); 708 if (value != NULL) 709 sc->vss_iid = strtoul(value, NULL, 10); 710 711 value = get_config_value_node(nvl, "bootindex"); 712 if (value != NULL) { 713 if (pci_emul_add_boot_device(pi, atoi(value))) { 714 EPRINTLN("Invalid bootindex %d", atoi(value)); 715 free(sc); 716 return (-1); 717 } 718 } 719 720 devname = get_config_value_node(nvl, "dev"); 721 if (devname == NULL) 722 devname = "/dev/cam/ctl"; 723 sc->vss_ctl_fd = open(devname, O_RDWR); 724 if (sc->vss_ctl_fd < 0) { 725 WPRINTF("cannot open %s: %s", devname, strerror(errno)); 726 free(sc); 727 return (1); 728 } 729 730 pthread_mutex_init(&sc->vss_mtx, NULL); 731 732 vi_softc_linkup(&sc->vss_vs, &vtscsi_vi_consts, sc, pi, sc->vss_vq); 733 sc->vss_vs.vs_mtx = &sc->vss_mtx; 734 735 /* controlq */ 736 sc->vss_vq[0].vq_qsize = VTSCSI_RINGSZ; 737 sc->vss_vq[0].vq_notify = pci_vtscsi_controlq_notify; 738 739 /* eventq */ 740 sc->vss_vq[1].vq_qsize = VTSCSI_RINGSZ; 741 sc->vss_vq[1].vq_notify = pci_vtscsi_eventq_notify; 742 743 /* request queues */ 744 for (i = 2; i < VTSCSI_MAXQ; i++) { 745 sc->vss_vq[i].vq_qsize = VTSCSI_RINGSZ; 746 sc->vss_vq[i].vq_notify = pci_vtscsi_requestq_notify; 747 pci_vtscsi_init_queue(sc, &sc->vss_queues[i - 2], i - 2); 748 } 749 750 /* initialize config space */ 751 pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_SCSI); 752 pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR); 753 pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE); 754 pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_SCSI); 755 pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR); 756 757 if (vi_intr_init(&sc->vss_vs, 1, fbsdrun_virtio_msix())) 758 return (1); 759 vi_set_io_bar(&sc->vss_vs, 0); 760 761 return (0); 762 } 763 764 765 static const struct pci_devemu pci_de_vscsi = { 766 .pe_emu = "virtio-scsi", 767 .pe_init = pci_vtscsi_init, 768 .pe_legacy_config = pci_vtscsi_legacy_config, 769 .pe_barwrite = vi_pci_write, 770 .pe_barread = vi_pci_read 771 }; 772 PCI_EMUL_SET(pci_de_vscsi); 773