1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Single Root IOV 1.0 5 * Address Translation Service 1.0 6 * 7 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 8 */ 9 10 #include <linux/pci.h> 11 #include <linux/slab.h> 12 #include <linux/export.h> 13 #include <linux/string.h> 14 #include <linux/delay.h> 15 #include "pci.h" 16 17 #define VIRTFN_ID_LEN 16 18 19 int pci_iov_virtfn_bus(struct pci_dev *dev, int vf_id) 20 { 21 if (!dev->is_physfn) 22 return -EINVAL; 23 return dev->bus->number + ((dev->devfn + dev->sriov->offset + 24 dev->sriov->stride * vf_id) >> 8); 25 } 26 27 int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id) 28 { 29 if (!dev->is_physfn) 30 return -EINVAL; 31 return (dev->devfn + dev->sriov->offset + 32 dev->sriov->stride * vf_id) & 0xff; 33 } 34 EXPORT_SYMBOL_GPL(pci_iov_virtfn_devfn); 35 36 int pci_iov_vf_id(struct pci_dev *dev) 37 { 38 struct pci_dev *pf; 39 40 if (!dev->is_virtfn) 41 return -EINVAL; 42 43 pf = pci_physfn(dev); 44 return (((dev->bus->number << 8) + dev->devfn) - 45 ((pf->bus->number << 8) + pf->devfn + pf->sriov->offset)) / 46 pf->sriov->stride; 47 } 48 EXPORT_SYMBOL_GPL(pci_iov_vf_id); 49 50 /** 51 * pci_iov_get_pf_drvdata - Return the drvdata of a PF 52 * @dev: VF pci_dev 53 * @pf_driver: Device driver required to own the PF 54 * 55 * This must be called from a context that ensures that a VF driver is attached. 56 * The value returned is invalid once the VF driver completes its remove() 57 * callback. 58 * 59 * Locking is achieved by the driver core. A VF driver cannot be probed until 60 * pci_enable_sriov() is called and pci_disable_sriov() does not return until 61 * all VF drivers have completed their remove(). 62 * 63 * The PF driver must call pci_disable_sriov() before it begins to destroy the 64 * drvdata. 65 */ 66 void *pci_iov_get_pf_drvdata(struct pci_dev *dev, struct pci_driver *pf_driver) 67 { 68 struct pci_dev *pf_dev; 69 70 if (!dev->is_virtfn) 71 return ERR_PTR(-EINVAL); 72 pf_dev = dev->physfn; 73 if (pf_dev->driver != pf_driver) 74 return ERR_PTR(-EINVAL); 75 return pci_get_drvdata(pf_dev); 76 } 77 EXPORT_SYMBOL_GPL(pci_iov_get_pf_drvdata); 78 79 /* 80 * Per SR-IOV spec sec 3.3.10 and 3.3.11, First VF Offset and VF Stride may 81 * change when NumVFs changes. 82 * 83 * Update iov->offset and iov->stride when NumVFs is written. 84 */ 85 static inline void pci_iov_set_numvfs(struct pci_dev *dev, int nr_virtfn) 86 { 87 struct pci_sriov *iov = dev->sriov; 88 89 pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn); 90 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &iov->offset); 91 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &iov->stride); 92 } 93 94 /* 95 * The PF consumes one bus number. NumVFs, First VF Offset, and VF Stride 96 * determine how many additional bus numbers will be consumed by VFs. 97 * 98 * Iterate over all valid NumVFs, validate offset and stride, and calculate 99 * the maximum number of bus numbers that could ever be required. 100 */ 101 static int compute_max_vf_buses(struct pci_dev *dev) 102 { 103 struct pci_sriov *iov = dev->sriov; 104 int nr_virtfn, busnr, rc = 0; 105 106 for (nr_virtfn = iov->total_VFs; nr_virtfn; nr_virtfn--) { 107 pci_iov_set_numvfs(dev, nr_virtfn); 108 if (!iov->offset || (nr_virtfn > 1 && !iov->stride)) { 109 rc = -EIO; 110 goto out; 111 } 112 113 busnr = pci_iov_virtfn_bus(dev, nr_virtfn - 1); 114 if (busnr > iov->max_VF_buses) 115 iov->max_VF_buses = busnr; 116 } 117 118 out: 119 pci_iov_set_numvfs(dev, 0); 120 return rc; 121 } 122 123 static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr) 124 { 125 struct pci_bus *child; 126 127 if (bus->number == busnr) 128 return bus; 129 130 child = pci_find_bus(pci_domain_nr(bus), busnr); 131 if (child) 132 return child; 133 134 child = pci_add_new_bus(bus, NULL, busnr); 135 if (!child) 136 return NULL; 137 138 pci_bus_insert_busn_res(child, busnr, busnr); 139 140 return child; 141 } 142 143 static void virtfn_remove_bus(struct pci_bus *physbus, struct pci_bus *virtbus) 144 { 145 if (physbus != virtbus && list_empty(&virtbus->devices)) 146 pci_remove_bus(virtbus); 147 } 148 149 resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno) 150 { 151 if (!dev->is_physfn) 152 return 0; 153 154 return dev->sriov->barsz[resno - PCI_IOV_RESOURCES]; 155 } 156 157 static void pci_read_vf_config_common(struct pci_dev *virtfn) 158 { 159 struct pci_dev *physfn = virtfn->physfn; 160 161 /* 162 * Some config registers are the same across all associated VFs. 163 * Read them once from VF0 so we can skip reading them from the 164 * other VFs. 165 * 166 * PCIe r4.0, sec 9.3.4.1, technically doesn't require all VFs to 167 * have the same Revision ID and Subsystem ID, but we assume they 168 * do. 169 */ 170 pci_read_config_dword(virtfn, PCI_CLASS_REVISION, 171 &physfn->sriov->class); 172 pci_read_config_byte(virtfn, PCI_HEADER_TYPE, 173 &physfn->sriov->hdr_type); 174 pci_read_config_word(virtfn, PCI_SUBSYSTEM_VENDOR_ID, 175 &physfn->sriov->subsystem_vendor); 176 pci_read_config_word(virtfn, PCI_SUBSYSTEM_ID, 177 &physfn->sriov->subsystem_device); 178 } 179 180 int pci_iov_sysfs_link(struct pci_dev *dev, 181 struct pci_dev *virtfn, int id) 182 { 183 char buf[VIRTFN_ID_LEN]; 184 int rc; 185 186 sprintf(buf, "virtfn%u", id); 187 rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf); 188 if (rc) 189 goto failed; 190 rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn"); 191 if (rc) 192 goto failed1; 193 194 kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE); 195 196 return 0; 197 198 failed1: 199 sysfs_remove_link(&dev->dev.kobj, buf); 200 failed: 201 return rc; 202 } 203 204 #ifdef CONFIG_PCI_MSI 205 static ssize_t sriov_vf_total_msix_show(struct device *dev, 206 struct device_attribute *attr, 207 char *buf) 208 { 209 struct pci_dev *pdev = to_pci_dev(dev); 210 u32 vf_total_msix = 0; 211 212 device_lock(dev); 213 if (!pdev->driver || !pdev->driver->sriov_get_vf_total_msix) 214 goto unlock; 215 216 vf_total_msix = pdev->driver->sriov_get_vf_total_msix(pdev); 217 unlock: 218 device_unlock(dev); 219 return sysfs_emit(buf, "%u\n", vf_total_msix); 220 } 221 static DEVICE_ATTR_RO(sriov_vf_total_msix); 222 223 static ssize_t sriov_vf_msix_count_store(struct device *dev, 224 struct device_attribute *attr, 225 const char *buf, size_t count) 226 { 227 struct pci_dev *vf_dev = to_pci_dev(dev); 228 struct pci_dev *pdev = pci_physfn(vf_dev); 229 int val, ret = 0; 230 231 if (kstrtoint(buf, 0, &val) < 0) 232 return -EINVAL; 233 234 if (val < 0) 235 return -EINVAL; 236 237 device_lock(&pdev->dev); 238 if (!pdev->driver || !pdev->driver->sriov_set_msix_vec_count) { 239 ret = -EOPNOTSUPP; 240 goto err_pdev; 241 } 242 243 device_lock(&vf_dev->dev); 244 if (vf_dev->driver) { 245 /* 246 * A driver is already attached to this VF and has configured 247 * itself based on the current MSI-X vector count. Changing 248 * the vector size could mess up the driver, so block it. 249 */ 250 ret = -EBUSY; 251 goto err_dev; 252 } 253 254 ret = pdev->driver->sriov_set_msix_vec_count(vf_dev, val); 255 256 err_dev: 257 device_unlock(&vf_dev->dev); 258 err_pdev: 259 device_unlock(&pdev->dev); 260 return ret ? : count; 261 } 262 static DEVICE_ATTR_WO(sriov_vf_msix_count); 263 #endif 264 265 static struct attribute *sriov_vf_dev_attrs[] = { 266 #ifdef CONFIG_PCI_MSI 267 &dev_attr_sriov_vf_msix_count.attr, 268 #endif 269 NULL, 270 }; 271 272 static umode_t sriov_vf_attrs_are_visible(struct kobject *kobj, 273 struct attribute *a, int n) 274 { 275 struct device *dev = kobj_to_dev(kobj); 276 struct pci_dev *pdev = to_pci_dev(dev); 277 278 if (!pdev->is_virtfn) 279 return 0; 280 281 return a->mode; 282 } 283 284 const struct attribute_group sriov_vf_dev_attr_group = { 285 .attrs = sriov_vf_dev_attrs, 286 .is_visible = sriov_vf_attrs_are_visible, 287 }; 288 289 int pci_iov_add_virtfn(struct pci_dev *dev, int id) 290 { 291 int i; 292 int rc = -ENOMEM; 293 u64 size; 294 struct pci_dev *virtfn; 295 struct resource *res; 296 struct pci_sriov *iov = dev->sriov; 297 struct pci_bus *bus; 298 299 bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id)); 300 if (!bus) 301 goto failed; 302 303 virtfn = pci_alloc_dev(bus); 304 if (!virtfn) 305 goto failed0; 306 307 virtfn->devfn = pci_iov_virtfn_devfn(dev, id); 308 virtfn->vendor = dev->vendor; 309 virtfn->device = iov->vf_device; 310 virtfn->is_virtfn = 1; 311 virtfn->physfn = pci_dev_get(dev); 312 virtfn->no_command_memory = 1; 313 314 if (id == 0) 315 pci_read_vf_config_common(virtfn); 316 317 rc = pci_setup_device(virtfn); 318 if (rc) 319 goto failed1; 320 321 virtfn->dev.parent = dev->dev.parent; 322 virtfn->multifunction = 0; 323 324 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 325 res = &dev->resource[i + PCI_IOV_RESOURCES]; 326 if (!res->parent) 327 continue; 328 virtfn->resource[i].name = pci_name(virtfn); 329 virtfn->resource[i].flags = res->flags; 330 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES); 331 virtfn->resource[i].start = res->start + size * id; 332 virtfn->resource[i].end = virtfn->resource[i].start + size - 1; 333 rc = request_resource(res, &virtfn->resource[i]); 334 BUG_ON(rc); 335 } 336 337 pci_device_add(virtfn, virtfn->bus); 338 rc = pci_iov_sysfs_link(dev, virtfn, id); 339 if (rc) 340 goto failed1; 341 342 pci_bus_add_device(virtfn); 343 344 return 0; 345 346 failed1: 347 pci_stop_and_remove_bus_device(virtfn); 348 pci_dev_put(dev); 349 failed0: 350 virtfn_remove_bus(dev->bus, bus); 351 failed: 352 353 return rc; 354 } 355 356 void pci_iov_remove_virtfn(struct pci_dev *dev, int id) 357 { 358 char buf[VIRTFN_ID_LEN]; 359 struct pci_dev *virtfn; 360 361 virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus), 362 pci_iov_virtfn_bus(dev, id), 363 pci_iov_virtfn_devfn(dev, id)); 364 if (!virtfn) 365 return; 366 367 sprintf(buf, "virtfn%u", id); 368 sysfs_remove_link(&dev->dev.kobj, buf); 369 /* 370 * pci_stop_dev() could have been called for this virtfn already, 371 * so the directory for the virtfn may have been removed before. 372 * Double check to avoid spurious sysfs warnings. 373 */ 374 if (virtfn->dev.kobj.sd) 375 sysfs_remove_link(&virtfn->dev.kobj, "physfn"); 376 377 pci_stop_and_remove_bus_device(virtfn); 378 virtfn_remove_bus(dev->bus, virtfn->bus); 379 380 /* balance pci_get_domain_bus_and_slot() */ 381 pci_dev_put(virtfn); 382 pci_dev_put(dev); 383 } 384 385 static ssize_t sriov_totalvfs_show(struct device *dev, 386 struct device_attribute *attr, 387 char *buf) 388 { 389 struct pci_dev *pdev = to_pci_dev(dev); 390 391 return sysfs_emit(buf, "%u\n", pci_sriov_get_totalvfs(pdev)); 392 } 393 394 static ssize_t sriov_numvfs_show(struct device *dev, 395 struct device_attribute *attr, 396 char *buf) 397 { 398 struct pci_dev *pdev = to_pci_dev(dev); 399 u16 num_vfs; 400 401 /* Serialize vs sriov_numvfs_store() so readers see valid num_VFs */ 402 device_lock(&pdev->dev); 403 num_vfs = pdev->sriov->num_VFs; 404 device_unlock(&pdev->dev); 405 406 return sysfs_emit(buf, "%u\n", num_vfs); 407 } 408 409 /* 410 * num_vfs > 0; number of VFs to enable 411 * num_vfs = 0; disable all VFs 412 * 413 * Note: SRIOV spec does not allow partial VF 414 * disable, so it's all or none. 415 */ 416 static ssize_t sriov_numvfs_store(struct device *dev, 417 struct device_attribute *attr, 418 const char *buf, size_t count) 419 { 420 struct pci_dev *pdev = to_pci_dev(dev); 421 int ret = 0; 422 u16 num_vfs; 423 424 if (kstrtou16(buf, 0, &num_vfs) < 0) 425 return -EINVAL; 426 427 if (num_vfs > pci_sriov_get_totalvfs(pdev)) 428 return -ERANGE; 429 430 device_lock(&pdev->dev); 431 432 if (num_vfs == pdev->sriov->num_VFs) 433 goto exit; 434 435 /* is PF driver loaded */ 436 if (!pdev->driver) { 437 pci_info(pdev, "no driver bound to device; cannot configure SR-IOV\n"); 438 ret = -ENOENT; 439 goto exit; 440 } 441 442 /* is PF driver loaded w/callback */ 443 if (!pdev->driver->sriov_configure) { 444 pci_info(pdev, "driver does not support SR-IOV configuration via sysfs\n"); 445 ret = -ENOENT; 446 goto exit; 447 } 448 449 if (num_vfs == 0) { 450 /* disable VFs */ 451 ret = pdev->driver->sriov_configure(pdev, 0); 452 goto exit; 453 } 454 455 /* enable VFs */ 456 if (pdev->sriov->num_VFs) { 457 pci_warn(pdev, "%d VFs already enabled. Disable before enabling %d VFs\n", 458 pdev->sriov->num_VFs, num_vfs); 459 ret = -EBUSY; 460 goto exit; 461 } 462 463 ret = pdev->driver->sriov_configure(pdev, num_vfs); 464 if (ret < 0) 465 goto exit; 466 467 if (ret != num_vfs) 468 pci_warn(pdev, "%d VFs requested; only %d enabled\n", 469 num_vfs, ret); 470 471 exit: 472 device_unlock(&pdev->dev); 473 474 if (ret < 0) 475 return ret; 476 477 return count; 478 } 479 480 static ssize_t sriov_offset_show(struct device *dev, 481 struct device_attribute *attr, 482 char *buf) 483 { 484 struct pci_dev *pdev = to_pci_dev(dev); 485 486 return sysfs_emit(buf, "%u\n", pdev->sriov->offset); 487 } 488 489 static ssize_t sriov_stride_show(struct device *dev, 490 struct device_attribute *attr, 491 char *buf) 492 { 493 struct pci_dev *pdev = to_pci_dev(dev); 494 495 return sysfs_emit(buf, "%u\n", pdev->sriov->stride); 496 } 497 498 static ssize_t sriov_vf_device_show(struct device *dev, 499 struct device_attribute *attr, 500 char *buf) 501 { 502 struct pci_dev *pdev = to_pci_dev(dev); 503 504 return sysfs_emit(buf, "%x\n", pdev->sriov->vf_device); 505 } 506 507 static ssize_t sriov_drivers_autoprobe_show(struct device *dev, 508 struct device_attribute *attr, 509 char *buf) 510 { 511 struct pci_dev *pdev = to_pci_dev(dev); 512 513 return sysfs_emit(buf, "%u\n", pdev->sriov->drivers_autoprobe); 514 } 515 516 static ssize_t sriov_drivers_autoprobe_store(struct device *dev, 517 struct device_attribute *attr, 518 const char *buf, size_t count) 519 { 520 struct pci_dev *pdev = to_pci_dev(dev); 521 bool drivers_autoprobe; 522 523 if (kstrtobool(buf, &drivers_autoprobe) < 0) 524 return -EINVAL; 525 526 pdev->sriov->drivers_autoprobe = drivers_autoprobe; 527 528 return count; 529 } 530 531 static DEVICE_ATTR_RO(sriov_totalvfs); 532 static DEVICE_ATTR_RW(sriov_numvfs); 533 static DEVICE_ATTR_RO(sriov_offset); 534 static DEVICE_ATTR_RO(sriov_stride); 535 static DEVICE_ATTR_RO(sriov_vf_device); 536 static DEVICE_ATTR_RW(sriov_drivers_autoprobe); 537 538 static struct attribute *sriov_pf_dev_attrs[] = { 539 &dev_attr_sriov_totalvfs.attr, 540 &dev_attr_sriov_numvfs.attr, 541 &dev_attr_sriov_offset.attr, 542 &dev_attr_sriov_stride.attr, 543 &dev_attr_sriov_vf_device.attr, 544 &dev_attr_sriov_drivers_autoprobe.attr, 545 #ifdef CONFIG_PCI_MSI 546 &dev_attr_sriov_vf_total_msix.attr, 547 #endif 548 NULL, 549 }; 550 551 static umode_t sriov_pf_attrs_are_visible(struct kobject *kobj, 552 struct attribute *a, int n) 553 { 554 struct device *dev = kobj_to_dev(kobj); 555 556 if (!dev_is_pf(dev)) 557 return 0; 558 559 return a->mode; 560 } 561 562 const struct attribute_group sriov_pf_dev_attr_group = { 563 .attrs = sriov_pf_dev_attrs, 564 .is_visible = sriov_pf_attrs_are_visible, 565 }; 566 567 int __weak pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) 568 { 569 return 0; 570 } 571 572 int __weak pcibios_sriov_disable(struct pci_dev *pdev) 573 { 574 return 0; 575 } 576 577 static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs) 578 { 579 unsigned int i; 580 int rc; 581 582 if (dev->no_vf_scan) 583 return 0; 584 585 for (i = 0; i < num_vfs; i++) { 586 rc = pci_iov_add_virtfn(dev, i); 587 if (rc) 588 goto failed; 589 } 590 return 0; 591 failed: 592 while (i--) 593 pci_iov_remove_virtfn(dev, i); 594 595 return rc; 596 } 597 598 static int sriov_enable(struct pci_dev *dev, int nr_virtfn) 599 { 600 int rc; 601 int i; 602 int nres; 603 u16 initial; 604 struct resource *res; 605 struct pci_dev *pdev; 606 struct pci_sriov *iov = dev->sriov; 607 int bars = 0; 608 int bus; 609 610 if (!nr_virtfn) 611 return 0; 612 613 if (iov->num_VFs) 614 return -EINVAL; 615 616 pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial); 617 if (initial > iov->total_VFs || 618 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total_VFs))) 619 return -EIO; 620 621 if (nr_virtfn < 0 || nr_virtfn > iov->total_VFs || 622 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial))) 623 return -EINVAL; 624 625 nres = 0; 626 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 627 bars |= (1 << (i + PCI_IOV_RESOURCES)); 628 res = &dev->resource[i + PCI_IOV_RESOURCES]; 629 if (res->parent) 630 nres++; 631 } 632 if (nres != iov->nres) { 633 pci_err(dev, "not enough MMIO resources for SR-IOV\n"); 634 return -ENOMEM; 635 } 636 637 bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1); 638 if (bus > dev->bus->busn_res.end) { 639 pci_err(dev, "can't enable %d VFs (bus %02x out of range of %pR)\n", 640 nr_virtfn, bus, &dev->bus->busn_res); 641 return -ENOMEM; 642 } 643 644 if (pci_enable_resources(dev, bars)) { 645 pci_err(dev, "SR-IOV: IOV BARS not allocated\n"); 646 return -ENOMEM; 647 } 648 649 if (iov->link != dev->devfn) { 650 pdev = pci_get_slot(dev->bus, iov->link); 651 if (!pdev) 652 return -ENODEV; 653 654 if (!pdev->is_physfn) { 655 pci_dev_put(pdev); 656 return -ENOSYS; 657 } 658 659 rc = sysfs_create_link(&dev->dev.kobj, 660 &pdev->dev.kobj, "dep_link"); 661 pci_dev_put(pdev); 662 if (rc) 663 return rc; 664 } 665 666 iov->initial_VFs = initial; 667 if (nr_virtfn < initial) 668 initial = nr_virtfn; 669 670 rc = pcibios_sriov_enable(dev, initial); 671 if (rc) { 672 pci_err(dev, "failure %d from pcibios_sriov_enable()\n", rc); 673 goto err_pcibios; 674 } 675 676 pci_iov_set_numvfs(dev, nr_virtfn); 677 iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE; 678 pci_cfg_access_lock(dev); 679 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 680 msleep(100); 681 pci_cfg_access_unlock(dev); 682 683 rc = sriov_add_vfs(dev, initial); 684 if (rc) 685 goto err_pcibios; 686 687 kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE); 688 iov->num_VFs = nr_virtfn; 689 690 return 0; 691 692 err_pcibios: 693 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 694 pci_cfg_access_lock(dev); 695 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 696 ssleep(1); 697 pci_cfg_access_unlock(dev); 698 699 pcibios_sriov_disable(dev); 700 701 if (iov->link != dev->devfn) 702 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 703 704 pci_iov_set_numvfs(dev, 0); 705 return rc; 706 } 707 708 static void sriov_del_vfs(struct pci_dev *dev) 709 { 710 struct pci_sriov *iov = dev->sriov; 711 int i; 712 713 for (i = 0; i < iov->num_VFs; i++) 714 pci_iov_remove_virtfn(dev, i); 715 } 716 717 static void sriov_disable(struct pci_dev *dev) 718 { 719 struct pci_sriov *iov = dev->sriov; 720 721 if (!iov->num_VFs) 722 return; 723 724 sriov_del_vfs(dev); 725 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 726 pci_cfg_access_lock(dev); 727 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 728 ssleep(1); 729 pci_cfg_access_unlock(dev); 730 731 pcibios_sriov_disable(dev); 732 733 if (iov->link != dev->devfn) 734 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 735 736 iov->num_VFs = 0; 737 pci_iov_set_numvfs(dev, 0); 738 } 739 740 static int sriov_init(struct pci_dev *dev, int pos) 741 { 742 int i, bar64; 743 int rc; 744 int nres; 745 u32 pgsz; 746 u16 ctrl, total; 747 struct pci_sriov *iov; 748 struct resource *res; 749 struct pci_dev *pdev; 750 751 pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl); 752 if (ctrl & PCI_SRIOV_CTRL_VFE) { 753 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0); 754 ssleep(1); 755 } 756 757 ctrl = 0; 758 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 759 if (pdev->is_physfn) 760 goto found; 761 762 pdev = NULL; 763 if (pci_ari_enabled(dev->bus)) 764 ctrl |= PCI_SRIOV_CTRL_ARI; 765 766 found: 767 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl); 768 769 pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total); 770 if (!total) 771 return 0; 772 773 pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz); 774 i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0; 775 pgsz &= ~((1 << i) - 1); 776 if (!pgsz) 777 return -EIO; 778 779 pgsz &= ~(pgsz - 1); 780 pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz); 781 782 iov = kzalloc(sizeof(*iov), GFP_KERNEL); 783 if (!iov) 784 return -ENOMEM; 785 786 nres = 0; 787 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 788 res = &dev->resource[i + PCI_IOV_RESOURCES]; 789 /* 790 * If it is already FIXED, don't change it, something 791 * (perhaps EA or header fixups) wants it this way. 792 */ 793 if (res->flags & IORESOURCE_PCI_FIXED) 794 bar64 = (res->flags & IORESOURCE_MEM_64) ? 1 : 0; 795 else 796 bar64 = __pci_read_base(dev, pci_bar_unknown, res, 797 pos + PCI_SRIOV_BAR + i * 4); 798 if (!res->flags) 799 continue; 800 if (resource_size(res) & (PAGE_SIZE - 1)) { 801 rc = -EIO; 802 goto failed; 803 } 804 iov->barsz[i] = resource_size(res); 805 res->end = res->start + resource_size(res) * total - 1; 806 pci_info(dev, "VF(n) BAR%d space: %pR (contains BAR%d for %d VFs)\n", 807 i, res, i, total); 808 i += bar64; 809 nres++; 810 } 811 812 iov->pos = pos; 813 iov->nres = nres; 814 iov->ctrl = ctrl; 815 iov->total_VFs = total; 816 iov->driver_max_VFs = total; 817 pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &iov->vf_device); 818 iov->pgsz = pgsz; 819 iov->self = dev; 820 iov->drivers_autoprobe = true; 821 pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap); 822 pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link); 823 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) 824 iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link); 825 826 if (pdev) 827 iov->dev = pci_dev_get(pdev); 828 else 829 iov->dev = dev; 830 831 dev->sriov = iov; 832 dev->is_physfn = 1; 833 rc = compute_max_vf_buses(dev); 834 if (rc) 835 goto fail_max_buses; 836 837 return 0; 838 839 fail_max_buses: 840 dev->sriov = NULL; 841 dev->is_physfn = 0; 842 failed: 843 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 844 res = &dev->resource[i + PCI_IOV_RESOURCES]; 845 res->flags = 0; 846 } 847 848 kfree(iov); 849 return rc; 850 } 851 852 static void sriov_release(struct pci_dev *dev) 853 { 854 BUG_ON(dev->sriov->num_VFs); 855 856 if (dev != dev->sriov->dev) 857 pci_dev_put(dev->sriov->dev); 858 859 kfree(dev->sriov); 860 dev->sriov = NULL; 861 } 862 863 static void sriov_restore_state(struct pci_dev *dev) 864 { 865 int i; 866 u16 ctrl; 867 struct pci_sriov *iov = dev->sriov; 868 869 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl); 870 if (ctrl & PCI_SRIOV_CTRL_VFE) 871 return; 872 873 /* 874 * Restore PCI_SRIOV_CTRL_ARI before pci_iov_set_numvfs() because 875 * it reads offset & stride, which depend on PCI_SRIOV_CTRL_ARI. 876 */ 877 ctrl &= ~PCI_SRIOV_CTRL_ARI; 878 ctrl |= iov->ctrl & PCI_SRIOV_CTRL_ARI; 879 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, ctrl); 880 881 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) 882 pci_update_resource(dev, i + PCI_IOV_RESOURCES); 883 884 pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz); 885 pci_iov_set_numvfs(dev, iov->num_VFs); 886 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 887 if (iov->ctrl & PCI_SRIOV_CTRL_VFE) 888 msleep(100); 889 } 890 891 /** 892 * pci_iov_init - initialize the IOV capability 893 * @dev: the PCI device 894 * 895 * Returns 0 on success, or negative on failure. 896 */ 897 int pci_iov_init(struct pci_dev *dev) 898 { 899 int pos; 900 901 if (!pci_is_pcie(dev)) 902 return -ENODEV; 903 904 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV); 905 if (pos) 906 return sriov_init(dev, pos); 907 908 return -ENODEV; 909 } 910 911 /** 912 * pci_iov_release - release resources used by the IOV capability 913 * @dev: the PCI device 914 */ 915 void pci_iov_release(struct pci_dev *dev) 916 { 917 if (dev->is_physfn) 918 sriov_release(dev); 919 } 920 921 /** 922 * pci_iov_remove - clean up SR-IOV state after PF driver is detached 923 * @dev: the PCI device 924 */ 925 void pci_iov_remove(struct pci_dev *dev) 926 { 927 struct pci_sriov *iov = dev->sriov; 928 929 if (!dev->is_physfn) 930 return; 931 932 iov->driver_max_VFs = iov->total_VFs; 933 if (iov->num_VFs) 934 pci_warn(dev, "driver left SR-IOV enabled after remove\n"); 935 } 936 937 /** 938 * pci_iov_update_resource - update a VF BAR 939 * @dev: the PCI device 940 * @resno: the resource number 941 * 942 * Update a VF BAR in the SR-IOV capability of a PF. 943 */ 944 void pci_iov_update_resource(struct pci_dev *dev, int resno) 945 { 946 struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL; 947 struct resource *res = dev->resource + resno; 948 int vf_bar = resno - PCI_IOV_RESOURCES; 949 struct pci_bus_region region; 950 u16 cmd; 951 u32 new; 952 int reg; 953 954 /* 955 * The generic pci_restore_bars() path calls this for all devices, 956 * including VFs and non-SR-IOV devices. If this is not a PF, we 957 * have nothing to do. 958 */ 959 if (!iov) 960 return; 961 962 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &cmd); 963 if ((cmd & PCI_SRIOV_CTRL_VFE) && (cmd & PCI_SRIOV_CTRL_MSE)) { 964 dev_WARN(&dev->dev, "can't update enabled VF BAR%d %pR\n", 965 vf_bar, res); 966 return; 967 } 968 969 /* 970 * Ignore unimplemented BARs, unused resource slots for 64-bit 971 * BARs, and non-movable resources, e.g., those described via 972 * Enhanced Allocation. 973 */ 974 if (!res->flags) 975 return; 976 977 if (res->flags & IORESOURCE_UNSET) 978 return; 979 980 if (res->flags & IORESOURCE_PCI_FIXED) 981 return; 982 983 pcibios_resource_to_bus(dev->bus, ®ion, res); 984 new = region.start; 985 new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK; 986 987 reg = iov->pos + PCI_SRIOV_BAR + 4 * vf_bar; 988 pci_write_config_dword(dev, reg, new); 989 if (res->flags & IORESOURCE_MEM_64) { 990 new = region.start >> 16 >> 16; 991 pci_write_config_dword(dev, reg + 4, new); 992 } 993 } 994 995 resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev, 996 int resno) 997 { 998 return pci_iov_resource_size(dev, resno); 999 } 1000 1001 /** 1002 * pci_sriov_resource_alignment - get resource alignment for VF BAR 1003 * @dev: the PCI device 1004 * @resno: the resource number 1005 * 1006 * Returns the alignment of the VF BAR found in the SR-IOV capability. 1007 * This is not the same as the resource size which is defined as 1008 * the VF BAR size multiplied by the number of VFs. The alignment 1009 * is just the VF BAR size. 1010 */ 1011 resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno) 1012 { 1013 return pcibios_iov_resource_alignment(dev, resno); 1014 } 1015 1016 /** 1017 * pci_restore_iov_state - restore the state of the IOV capability 1018 * @dev: the PCI device 1019 */ 1020 void pci_restore_iov_state(struct pci_dev *dev) 1021 { 1022 if (dev->is_physfn) 1023 sriov_restore_state(dev); 1024 } 1025 1026 /** 1027 * pci_vf_drivers_autoprobe - set PF property drivers_autoprobe for VFs 1028 * @dev: the PCI device 1029 * @auto_probe: set VF drivers auto probe flag 1030 */ 1031 void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool auto_probe) 1032 { 1033 if (dev->is_physfn) 1034 dev->sriov->drivers_autoprobe = auto_probe; 1035 } 1036 1037 /** 1038 * pci_iov_bus_range - find bus range used by Virtual Function 1039 * @bus: the PCI bus 1040 * 1041 * Returns max number of buses (exclude current one) used by Virtual 1042 * Functions. 1043 */ 1044 int pci_iov_bus_range(struct pci_bus *bus) 1045 { 1046 int max = 0; 1047 struct pci_dev *dev; 1048 1049 list_for_each_entry(dev, &bus->devices, bus_list) { 1050 if (!dev->is_physfn) 1051 continue; 1052 if (dev->sriov->max_VF_buses > max) 1053 max = dev->sriov->max_VF_buses; 1054 } 1055 1056 return max ? max - bus->number : 0; 1057 } 1058 1059 /** 1060 * pci_enable_sriov - enable the SR-IOV capability 1061 * @dev: the PCI device 1062 * @nr_virtfn: number of virtual functions to enable 1063 * 1064 * Returns 0 on success, or negative on failure. 1065 */ 1066 int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) 1067 { 1068 might_sleep(); 1069 1070 if (!dev->is_physfn) 1071 return -ENOSYS; 1072 1073 return sriov_enable(dev, nr_virtfn); 1074 } 1075 EXPORT_SYMBOL_GPL(pci_enable_sriov); 1076 1077 /** 1078 * pci_disable_sriov - disable the SR-IOV capability 1079 * @dev: the PCI device 1080 */ 1081 void pci_disable_sriov(struct pci_dev *dev) 1082 { 1083 might_sleep(); 1084 1085 if (!dev->is_physfn) 1086 return; 1087 1088 sriov_disable(dev); 1089 } 1090 EXPORT_SYMBOL_GPL(pci_disable_sriov); 1091 1092 /** 1093 * pci_num_vf - return number of VFs associated with a PF device_release_driver 1094 * @dev: the PCI device 1095 * 1096 * Returns number of VFs, or 0 if SR-IOV is not enabled. 1097 */ 1098 int pci_num_vf(struct pci_dev *dev) 1099 { 1100 if (!dev->is_physfn) 1101 return 0; 1102 1103 return dev->sriov->num_VFs; 1104 } 1105 EXPORT_SYMBOL_GPL(pci_num_vf); 1106 1107 /** 1108 * pci_vfs_assigned - returns number of VFs are assigned to a guest 1109 * @dev: the PCI device 1110 * 1111 * Returns number of VFs belonging to this device that are assigned to a guest. 1112 * If device is not a physical function returns 0. 1113 */ 1114 int pci_vfs_assigned(struct pci_dev *dev) 1115 { 1116 struct pci_dev *vfdev; 1117 unsigned int vfs_assigned = 0; 1118 unsigned short dev_id; 1119 1120 /* only search if we are a PF */ 1121 if (!dev->is_physfn) 1122 return 0; 1123 1124 /* 1125 * determine the device ID for the VFs, the vendor ID will be the 1126 * same as the PF so there is no need to check for that one 1127 */ 1128 dev_id = dev->sriov->vf_device; 1129 1130 /* loop through all the VFs to see if we own any that are assigned */ 1131 vfdev = pci_get_device(dev->vendor, dev_id, NULL); 1132 while (vfdev) { 1133 /* 1134 * It is considered assigned if it is a virtual function with 1135 * our dev as the physical function and the assigned bit is set 1136 */ 1137 if (vfdev->is_virtfn && (vfdev->physfn == dev) && 1138 pci_is_dev_assigned(vfdev)) 1139 vfs_assigned++; 1140 1141 vfdev = pci_get_device(dev->vendor, dev_id, vfdev); 1142 } 1143 1144 return vfs_assigned; 1145 } 1146 EXPORT_SYMBOL_GPL(pci_vfs_assigned); 1147 1148 /** 1149 * pci_sriov_set_totalvfs -- reduce the TotalVFs available 1150 * @dev: the PCI PF device 1151 * @numvfs: number that should be used for TotalVFs supported 1152 * 1153 * Should be called from PF driver's probe routine with 1154 * device's mutex held. 1155 * 1156 * Returns 0 if PF is an SRIOV-capable device and 1157 * value of numvfs valid. If not a PF return -ENOSYS; 1158 * if numvfs is invalid return -EINVAL; 1159 * if VFs already enabled, return -EBUSY. 1160 */ 1161 int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs) 1162 { 1163 if (!dev->is_physfn) 1164 return -ENOSYS; 1165 1166 if (numvfs > dev->sriov->total_VFs) 1167 return -EINVAL; 1168 1169 /* Shouldn't change if VFs already enabled */ 1170 if (dev->sriov->ctrl & PCI_SRIOV_CTRL_VFE) 1171 return -EBUSY; 1172 1173 dev->sriov->driver_max_VFs = numvfs; 1174 return 0; 1175 } 1176 EXPORT_SYMBOL_GPL(pci_sriov_set_totalvfs); 1177 1178 /** 1179 * pci_sriov_get_totalvfs -- get total VFs supported on this device 1180 * @dev: the PCI PF device 1181 * 1182 * For a PCIe device with SRIOV support, return the PCIe 1183 * SRIOV capability value of TotalVFs or the value of driver_max_VFs 1184 * if the driver reduced it. Otherwise 0. 1185 */ 1186 int pci_sriov_get_totalvfs(struct pci_dev *dev) 1187 { 1188 if (!dev->is_physfn) 1189 return 0; 1190 1191 return dev->sriov->driver_max_VFs; 1192 } 1193 EXPORT_SYMBOL_GPL(pci_sriov_get_totalvfs); 1194 1195 /** 1196 * pci_sriov_configure_simple - helper to configure SR-IOV 1197 * @dev: the PCI device 1198 * @nr_virtfn: number of virtual functions to enable, 0 to disable 1199 * 1200 * Enable or disable SR-IOV for devices that don't require any PF setup 1201 * before enabling SR-IOV. Return value is negative on error, or number of 1202 * VFs allocated on success. 1203 */ 1204 int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn) 1205 { 1206 int rc; 1207 1208 might_sleep(); 1209 1210 if (!dev->is_physfn) 1211 return -ENODEV; 1212 1213 if (pci_vfs_assigned(dev)) { 1214 pci_warn(dev, "Cannot modify SR-IOV while VFs are assigned\n"); 1215 return -EPERM; 1216 } 1217 1218 if (nr_virtfn == 0) { 1219 sriov_disable(dev); 1220 return 0; 1221 } 1222 1223 rc = sriov_enable(dev, nr_virtfn); 1224 if (rc < 0) 1225 return rc; 1226 1227 return nr_virtfn; 1228 } 1229 EXPORT_SYMBOL_GPL(pci_sriov_configure_simple); 1230