1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Endpoint Function Driver to implement Non-Transparent Bridge functionality 4 * 5 * Copyright (C) 2020 Texas Instruments 6 * Author: Kishon Vijay Abraham I <kishon@ti.com> 7 */ 8 9 /* 10 * The PCI NTB function driver configures the SoC with multiple PCIe Endpoint 11 * (EP) controller instances (see diagram below) in such a way that 12 * transactions from one EP controller are routed to the other EP controller. 13 * Once PCI NTB function driver configures the SoC with multiple EP instances, 14 * HOST1 and HOST2 can communicate with each other using SoC as a bridge. 15 * 16 * +-------------+ +-------------+ 17 * | | | | 18 * | HOST1 | | HOST2 | 19 * | | | | 20 * +------^------+ +------^------+ 21 * | | 22 * | | 23 * +---------|-------------------------------------------------|---------+ 24 * | +------v------+ +------v------+ | 25 * | | | | | | 26 * | | EP | | EP | | 27 * | | CONTROLLER1 | | CONTROLLER2 | | 28 * | | <-----------------------------------> | | 29 * | | | | | | 30 * | | | | | | 31 * | | | SoC With Multiple EP Instances | | | 32 * | | | (Configured using NTB Function) | | | 33 * | +-------------+ +-------------+ | 34 * +---------------------------------------------------------------------+ 35 */ 36 37 #include <linux/delay.h> 38 #include <linux/io.h> 39 #include <linux/module.h> 40 #include <linux/slab.h> 41 42 #include <linux/pci-epc.h> 43 #include <linux/pci-epf.h> 44 45 static struct workqueue_struct *kpcintb_workqueue; 46 47 #define COMMAND_CONFIGURE_DOORBELL 1 48 #define COMMAND_TEARDOWN_DOORBELL 2 49 #define COMMAND_CONFIGURE_MW 3 50 #define COMMAND_TEARDOWN_MW 4 51 #define COMMAND_LINK_UP 5 52 #define COMMAND_LINK_DOWN 6 53 54 #define COMMAND_STATUS_OK 1 55 #define COMMAND_STATUS_ERROR 2 56 57 #define LINK_STATUS_UP BIT(0) 58 59 #define SPAD_COUNT 64 60 #define DB_COUNT 4 61 #define NTB_MW_OFFSET 2 62 #define DB_COUNT_MASK GENMASK(15, 0) 63 #define MSIX_ENABLE BIT(16) 64 #define MAX_DB_COUNT 32 65 #define MAX_MW 4 66 67 enum epf_ntb_bar { 68 BAR_CONFIG, 69 BAR_PEER_SPAD, 70 BAR_DB_MW1, 71 BAR_MW2, 72 BAR_MW3, 73 BAR_MW4, 74 }; 75 76 struct epf_ntb { 77 u32 num_mws; 78 u32 db_count; 79 u32 spad_count; 80 struct pci_epf *epf; 81 u64 mws_size[MAX_MW]; 82 struct config_group group; 83 struct epf_ntb_epc *epc[2]; 84 }; 85 86 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group) 87 88 struct epf_ntb_epc { 89 u8 func_no; 90 u8 vfunc_no; 91 bool linkup; 92 bool is_msix; 93 int msix_bar; 94 u32 spad_size; 95 struct pci_epc *epc; 96 struct epf_ntb *epf_ntb; 97 void __iomem *mw_addr[6]; 98 size_t msix_table_offset; 99 struct epf_ntb_ctrl *reg; 100 struct pci_epf_bar *epf_bar; 101 enum pci_barno epf_ntb_bar[6]; 102 struct delayed_work cmd_handler; 103 enum pci_epc_interface_type type; 104 const struct pci_epc_features *epc_features; 105 }; 106 107 struct epf_ntb_ctrl { 108 u32 command; 109 u32 argument; 110 u16 command_status; 111 u16 link_status; 112 u32 topology; 113 u64 addr; 114 u64 size; 115 u32 num_mws; 116 u32 mw1_offset; 117 u32 spad_offset; 118 u32 spad_count; 119 u32 db_entry_size; 120 u32 db_data[MAX_DB_COUNT]; 121 u32 db_offset[MAX_DB_COUNT]; 122 } __packed; 123 124 static struct pci_epf_header epf_ntb_header = { 125 .vendorid = PCI_ANY_ID, 126 .deviceid = PCI_ANY_ID, 127 .baseclass_code = PCI_BASE_CLASS_MEMORY, 128 .interrupt_pin = PCI_INTERRUPT_INTA, 129 }; 130 131 /** 132 * epf_ntb_link_up() - Raise link_up interrupt to both the hosts 133 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 134 * @link_up: true or false indicating Link is UP or Down 135 * 136 * Once NTB function in HOST1 and the NTB function in HOST2 invoke 137 * ntb_link_enable(), this NTB function driver will trigger a link event to 138 * the NTB client in both the hosts. 139 */ 140 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up) 141 { 142 enum pci_epc_interface_type type; 143 enum pci_epc_irq_type irq_type; 144 struct epf_ntb_epc *ntb_epc; 145 struct epf_ntb_ctrl *ctrl; 146 struct pci_epc *epc; 147 u8 func_no, vfunc_no; 148 bool is_msix; 149 int ret; 150 151 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 152 ntb_epc = ntb->epc[type]; 153 epc = ntb_epc->epc; 154 func_no = ntb_epc->func_no; 155 vfunc_no = ntb_epc->vfunc_no; 156 is_msix = ntb_epc->is_msix; 157 ctrl = ntb_epc->reg; 158 if (link_up) 159 ctrl->link_status |= LINK_STATUS_UP; 160 else 161 ctrl->link_status &= ~LINK_STATUS_UP; 162 irq_type = is_msix ? PCI_EPC_IRQ_MSIX : PCI_EPC_IRQ_MSI; 163 ret = pci_epc_raise_irq(epc, func_no, vfunc_no, irq_type, 1); 164 if (ret) { 165 dev_err(&epc->dev, 166 "%s intf: Failed to raise Link Up IRQ\n", 167 pci_epc_interface_string(type)); 168 return ret; 169 } 170 } 171 172 return 0; 173 } 174 175 /** 176 * epf_ntb_configure_mw() - Configure the Outbound Address Space for one host 177 * to access the memory window of other host 178 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 179 * @type: PRIMARY interface or SECONDARY interface 180 * @mw: Index of the memory window (either 0, 1, 2 or 3) 181 * 182 * +-----------------+ +---->+----------------+-----------+-----------------+ 183 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | 184 * +-----------------+ | +----------------+ +-----------------+ 185 * | BAR1 | | | Doorbell 2 +---------+ | | 186 * +-----------------+----+ +----------------+ | | | 187 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ 188 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | 189 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 190 * +-----------------+ | |----------------+ | | | | 191 * | BAR4 | | | | | | +-----------------+ 192 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| 193 * | BAR5 | | | | | | +-----------------+ 194 * +-----------------+ +---->-----------------+ | | | | 195 * EP CONTROLLER 1 | | | | +-----------------+ 196 * | | | +---->+ MSI|X ADDRESS 4 | 197 * +----------------+ | +-----------------+ 198 * (A) EP CONTROLLER 2 | | | 199 * (OB SPACE) | | | 200 * +-------> MW1 | 201 * | | 202 * | | 203 * (B) +-----------------+ 204 * | | 205 * | | 206 * | | 207 * | | 208 * | | 209 * +-----------------+ 210 * PCI Address Space 211 * (Managed by HOST2) 212 * 213 * This function performs stage (B) in the above diagram (see MW1) i.e., map OB 214 * address space of memory window to PCI address space. 215 * 216 * This operation requires 3 parameters 217 * 1) Address in the outbound address space 218 * 2) Address in the PCI Address space 219 * 3) Size of the address region to be mapped 220 * 221 * The address in the outbound address space (for MW1, MW2, MW3 and MW4) is 222 * stored in epf_bar corresponding to BAR_DB_MW1 for MW1 and BAR_MW2, BAR_MW3 223 * BAR_MW4 for rest of the BARs of epf_ntb_epc that is connected to HOST1. This 224 * is populated in epf_ntb_alloc_peer_mem() in this driver. 225 * 226 * The address and size of the PCI address region that has to be mapped would 227 * be provided by HOST2 in ctrl->addr and ctrl->size of epf_ntb_epc that is 228 * connected to HOST2. 229 * 230 * Please note Memory window1 (MW1) and Doorbell registers together will be 231 * mapped to a single BAR (BAR2) above for 32-bit BARs. The exact BAR that's 232 * used for Memory window (MW) can be obtained from epf_ntb_bar[BAR_DB_MW1], 233 * epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2]. 234 */ 235 static int epf_ntb_configure_mw(struct epf_ntb *ntb, 236 enum pci_epc_interface_type type, u32 mw) 237 { 238 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 239 struct pci_epf_bar *peer_epf_bar; 240 enum pci_barno peer_barno; 241 struct epf_ntb_ctrl *ctrl; 242 phys_addr_t phys_addr; 243 u8 func_no, vfunc_no; 244 struct pci_epc *epc; 245 u64 addr, size; 246 int ret = 0; 247 248 ntb_epc = ntb->epc[type]; 249 epc = ntb_epc->epc; 250 251 peer_ntb_epc = ntb->epc[!type]; 252 peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET]; 253 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 254 255 phys_addr = peer_epf_bar->phys_addr; 256 ctrl = ntb_epc->reg; 257 addr = ctrl->addr; 258 size = ctrl->size; 259 if (mw + NTB_MW_OFFSET == BAR_DB_MW1) 260 phys_addr += ctrl->mw1_offset; 261 262 if (size > ntb->mws_size[mw]) { 263 dev_err(&epc->dev, 264 "%s intf: MW: %d Req Sz:%llxx > Supported Sz:%llx\n", 265 pci_epc_interface_string(type), mw, size, 266 ntb->mws_size[mw]); 267 ret = -EINVAL; 268 goto err_invalid_size; 269 } 270 271 func_no = ntb_epc->func_no; 272 vfunc_no = ntb_epc->vfunc_no; 273 274 ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, addr, size); 275 if (ret) 276 dev_err(&epc->dev, 277 "%s intf: Failed to map memory window %d address\n", 278 pci_epc_interface_string(type), mw); 279 280 err_invalid_size: 281 282 return ret; 283 } 284 285 /** 286 * epf_ntb_teardown_mw() - Teardown the configured OB ATU 287 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 288 * @type: PRIMARY interface or SECONDARY interface 289 * @mw: Index of the memory window (either 0, 1, 2 or 3) 290 * 291 * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using 292 * pci_epc_unmap_addr() 293 */ 294 static void epf_ntb_teardown_mw(struct epf_ntb *ntb, 295 enum pci_epc_interface_type type, u32 mw) 296 { 297 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 298 struct pci_epf_bar *peer_epf_bar; 299 enum pci_barno peer_barno; 300 struct epf_ntb_ctrl *ctrl; 301 phys_addr_t phys_addr; 302 u8 func_no, vfunc_no; 303 struct pci_epc *epc; 304 305 ntb_epc = ntb->epc[type]; 306 epc = ntb_epc->epc; 307 308 peer_ntb_epc = ntb->epc[!type]; 309 peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET]; 310 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 311 312 phys_addr = peer_epf_bar->phys_addr; 313 ctrl = ntb_epc->reg; 314 if (mw + NTB_MW_OFFSET == BAR_DB_MW1) 315 phys_addr += ctrl->mw1_offset; 316 func_no = ntb_epc->func_no; 317 vfunc_no = ntb_epc->vfunc_no; 318 319 pci_epc_unmap_addr(epc, func_no, vfunc_no, phys_addr); 320 } 321 322 /** 323 * epf_ntb_configure_msi() - Map OB address space to MSI address 324 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 325 * @type: PRIMARY interface or SECONDARY interface 326 * @db_count: Number of doorbell interrupts to map 327 * 328 *+-----------------+ +----->+----------------+-----------+-----------------+ 329 *| BAR0 | | | Doorbell 1 +---+-------> MSI ADDRESS | 330 *+-----------------+ | +----------------+ | +-----------------+ 331 *| BAR1 | | | Doorbell 2 +---+ | | 332 *+-----------------+----+ +----------------+ | | | 333 *| BAR2 | | Doorbell 3 +---+ | | 334 *+-----------------+----+ +----------------+ | | | 335 *| BAR3 | | | Doorbell 4 +---+ | | 336 *+-----------------+ | |----------------+ | | 337 *| BAR4 | | | | | | 338 *+-----------------+ | | MW1 | | | 339 *| BAR5 | | | | | | 340 *+-----------------+ +----->-----------------+ | | 341 * EP CONTROLLER 1 | | | | 342 * | | | | 343 * +----------------+ +-----------------+ 344 * (A) EP CONTROLLER 2 | | 345 * (OB SPACE) | | 346 * | MW1 | 347 * | | 348 * | | 349 * (B) +-----------------+ 350 * | | 351 * | | 352 * | | 353 * | | 354 * | | 355 * +-----------------+ 356 * PCI Address Space 357 * (Managed by HOST2) 358 * 359 * 360 * This function performs stage (B) in the above diagram (see Doorbell 1, 361 * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to 362 * doorbell to MSI address in PCI address space. 363 * 364 * This operation requires 3 parameters 365 * 1) Address reserved for doorbell in the outbound address space 366 * 2) MSI-X address in the PCIe Address space 367 * 3) Number of MSI-X interrupts that has to be configured 368 * 369 * The address in the outbound address space (for the Doorbell) is stored in 370 * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to 371 * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along 372 * with address for MW1. 373 * 374 * pci_epc_map_msi_irq() takes the MSI address from MSI capability register 375 * and maps the OB address (obtained in epf_ntb_alloc_peer_mem()) to the MSI 376 * address. 377 * 378 * epf_ntb_configure_msi() also stores the MSI data to raise each interrupt 379 * in db_data of the peer's control region. This helps the peer to raise 380 * doorbell of the other host by writing db_data to the BAR corresponding to 381 * BAR_DB_MW1. 382 */ 383 static int epf_ntb_configure_msi(struct epf_ntb *ntb, 384 enum pci_epc_interface_type type, u16 db_count) 385 { 386 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 387 u32 db_entry_size, db_data, db_offset; 388 struct pci_epf_bar *peer_epf_bar; 389 struct epf_ntb_ctrl *peer_ctrl; 390 enum pci_barno peer_barno; 391 phys_addr_t phys_addr; 392 u8 func_no, vfunc_no; 393 struct pci_epc *epc; 394 int ret, i; 395 396 ntb_epc = ntb->epc[type]; 397 epc = ntb_epc->epc; 398 399 peer_ntb_epc = ntb->epc[!type]; 400 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; 401 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 402 peer_ctrl = peer_ntb_epc->reg; 403 db_entry_size = peer_ctrl->db_entry_size; 404 405 phys_addr = peer_epf_bar->phys_addr; 406 func_no = ntb_epc->func_no; 407 vfunc_no = ntb_epc->vfunc_no; 408 409 ret = pci_epc_map_msi_irq(epc, func_no, vfunc_no, phys_addr, db_count, 410 db_entry_size, &db_data, &db_offset); 411 if (ret) { 412 dev_err(&epc->dev, "%s intf: Failed to map MSI IRQ\n", 413 pci_epc_interface_string(type)); 414 return ret; 415 } 416 417 for (i = 0; i < db_count; i++) { 418 peer_ctrl->db_data[i] = db_data | i; 419 peer_ctrl->db_offset[i] = db_offset; 420 } 421 422 return 0; 423 } 424 425 /** 426 * epf_ntb_configure_msix() - Map OB address space to MSI-X address 427 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 428 * @type: PRIMARY interface or SECONDARY interface 429 * @db_count: Number of doorbell interrupts to map 430 * 431 *+-----------------+ +----->+----------------+-----------+-----------------+ 432 *| BAR0 | | | Doorbell 1 +-----------> MSI-X ADDRESS 1 | 433 *+-----------------+ | +----------------+ +-----------------+ 434 *| BAR1 | | | Doorbell 2 +---------+ | | 435 *+-----------------+----+ +----------------+ | | | 436 *| BAR2 | | Doorbell 3 +-------+ | +-----------------+ 437 *+-----------------+----+ +----------------+ | +-> MSI-X ADDRESS 2 | 438 *| BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 439 *+-----------------+ | |----------------+ | | | | 440 *| BAR4 | | | | | | +-----------------+ 441 *+-----------------+ | | MW1 + | +-->+ MSI-X ADDRESS 3|| 442 *| BAR5 | | | | | +-----------------+ 443 *+-----------------+ +----->-----------------+ | | | 444 * EP CONTROLLER 1 | | | +-----------------+ 445 * | | +---->+ MSI-X ADDRESS 4 | 446 * +----------------+ +-----------------+ 447 * (A) EP CONTROLLER 2 | | 448 * (OB SPACE) | | 449 * | MW1 | 450 * | | 451 * | | 452 * (B) +-----------------+ 453 * | | 454 * | | 455 * | | 456 * | | 457 * | | 458 * +-----------------+ 459 * PCI Address Space 460 * (Managed by HOST2) 461 * 462 * This function performs stage (B) in the above diagram (see Doorbell 1, 463 * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to 464 * doorbell to MSI-X address in PCI address space. 465 * 466 * This operation requires 3 parameters 467 * 1) Address reserved for doorbell in the outbound address space 468 * 2) MSI-X address in the PCIe Address space 469 * 3) Number of MSI-X interrupts that has to be configured 470 * 471 * The address in the outbound address space (for the Doorbell) is stored in 472 * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to 473 * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along 474 * with address for MW1. 475 * 476 * The MSI-X address is in the MSI-X table of EP CONTROLLER 2 and 477 * the count of doorbell is in ctrl->argument of epf_ntb_epc that is connected 478 * to HOST2. MSI-X table is stored memory mapped to ntb_epc->msix_bar and the 479 * offset is in ntb_epc->msix_table_offset. From this epf_ntb_configure_msix() 480 * gets the MSI-X address and data. 481 * 482 * epf_ntb_configure_msix() also stores the MSI-X data to raise each interrupt 483 * in db_data of the peer's control region. This helps the peer to raise 484 * doorbell of the other host by writing db_data to the BAR corresponding to 485 * BAR_DB_MW1. 486 */ 487 static int epf_ntb_configure_msix(struct epf_ntb *ntb, 488 enum pci_epc_interface_type type, 489 u16 db_count) 490 { 491 const struct pci_epc_features *epc_features; 492 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 493 struct pci_epf_bar *peer_epf_bar, *epf_bar; 494 struct pci_epf_msix_tbl *msix_tbl; 495 struct epf_ntb_ctrl *peer_ctrl; 496 u32 db_entry_size, msg_data; 497 enum pci_barno peer_barno; 498 phys_addr_t phys_addr; 499 u8 func_no, vfunc_no; 500 struct pci_epc *epc; 501 size_t align; 502 u64 msg_addr; 503 int ret, i; 504 505 ntb_epc = ntb->epc[type]; 506 epc = ntb_epc->epc; 507 508 epf_bar = &ntb_epc->epf_bar[ntb_epc->msix_bar]; 509 msix_tbl = epf_bar->addr + ntb_epc->msix_table_offset; 510 511 peer_ntb_epc = ntb->epc[!type]; 512 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; 513 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 514 phys_addr = peer_epf_bar->phys_addr; 515 peer_ctrl = peer_ntb_epc->reg; 516 epc_features = ntb_epc->epc_features; 517 align = epc_features->align; 518 519 func_no = ntb_epc->func_no; 520 vfunc_no = ntb_epc->vfunc_no; 521 db_entry_size = peer_ctrl->db_entry_size; 522 523 for (i = 0; i < db_count; i++) { 524 msg_addr = ALIGN_DOWN(msix_tbl[i].msg_addr, align); 525 msg_data = msix_tbl[i].msg_data; 526 ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, msg_addr, 527 db_entry_size); 528 if (ret) { 529 dev_err(&epc->dev, 530 "%s intf: Failed to configure MSI-X IRQ\n", 531 pci_epc_interface_string(type)); 532 return ret; 533 } 534 phys_addr = phys_addr + db_entry_size; 535 peer_ctrl->db_data[i] = msg_data; 536 peer_ctrl->db_offset[i] = msix_tbl[i].msg_addr & (align - 1); 537 } 538 ntb_epc->is_msix = true; 539 540 return 0; 541 } 542 543 /** 544 * epf_ntb_configure_db() - Configure the Outbound Address Space for one host 545 * to ring the doorbell of other host 546 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 547 * @type: PRIMARY interface or SECONDARY interface 548 * @db_count: Count of the number of doorbells that has to be configured 549 * @msix: Indicates whether MSI-X or MSI should be used 550 * 551 * Invokes epf_ntb_configure_msix() or epf_ntb_configure_msi() required for 552 * one HOST to ring the doorbell of other HOST. 553 */ 554 static int epf_ntb_configure_db(struct epf_ntb *ntb, 555 enum pci_epc_interface_type type, 556 u16 db_count, bool msix) 557 { 558 struct epf_ntb_epc *ntb_epc; 559 struct pci_epc *epc; 560 int ret; 561 562 if (db_count > MAX_DB_COUNT) 563 return -EINVAL; 564 565 ntb_epc = ntb->epc[type]; 566 epc = ntb_epc->epc; 567 568 if (msix) 569 ret = epf_ntb_configure_msix(ntb, type, db_count); 570 else 571 ret = epf_ntb_configure_msi(ntb, type, db_count); 572 573 if (ret) 574 dev_err(&epc->dev, "%s intf: Failed to configure DB\n", 575 pci_epc_interface_string(type)); 576 577 return ret; 578 } 579 580 /** 581 * epf_ntb_teardown_db() - Unmap address in OB address space to MSI/MSI-X 582 * address 583 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 584 * @type: PRIMARY interface or SECONDARY interface 585 * 586 * Invoke pci_epc_unmap_addr() to unmap OB address to MSI/MSI-X address. 587 */ 588 static void 589 epf_ntb_teardown_db(struct epf_ntb *ntb, enum pci_epc_interface_type type) 590 { 591 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 592 struct pci_epf_bar *peer_epf_bar; 593 enum pci_barno peer_barno; 594 phys_addr_t phys_addr; 595 u8 func_no, vfunc_no; 596 struct pci_epc *epc; 597 598 ntb_epc = ntb->epc[type]; 599 epc = ntb_epc->epc; 600 601 peer_ntb_epc = ntb->epc[!type]; 602 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; 603 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 604 phys_addr = peer_epf_bar->phys_addr; 605 func_no = ntb_epc->func_no; 606 vfunc_no = ntb_epc->vfunc_no; 607 608 pci_epc_unmap_addr(epc, func_no, vfunc_no, phys_addr); 609 } 610 611 /** 612 * epf_ntb_cmd_handler() - Handle commands provided by the NTB Host 613 * @work: work_struct for the two epf_ntb_epc (PRIMARY and SECONDARY) 614 * 615 * Workqueue function that gets invoked for the two epf_ntb_epc 616 * periodically (once every 5ms) to see if it has received any commands 617 * from NTB host. The host can send commands to configure doorbell or 618 * configure memory window or to update link status. 619 */ 620 static void epf_ntb_cmd_handler(struct work_struct *work) 621 { 622 enum pci_epc_interface_type type; 623 struct epf_ntb_epc *ntb_epc; 624 struct epf_ntb_ctrl *ctrl; 625 u32 command, argument; 626 struct epf_ntb *ntb; 627 struct device *dev; 628 u16 db_count; 629 bool is_msix; 630 int ret; 631 632 ntb_epc = container_of(work, struct epf_ntb_epc, cmd_handler.work); 633 ctrl = ntb_epc->reg; 634 command = ctrl->command; 635 if (!command) 636 goto reset_handler; 637 argument = ctrl->argument; 638 639 ctrl->command = 0; 640 ctrl->argument = 0; 641 642 ctrl = ntb_epc->reg; 643 type = ntb_epc->type; 644 ntb = ntb_epc->epf_ntb; 645 dev = &ntb->epf->dev; 646 647 switch (command) { 648 case COMMAND_CONFIGURE_DOORBELL: 649 db_count = argument & DB_COUNT_MASK; 650 is_msix = argument & MSIX_ENABLE; 651 ret = epf_ntb_configure_db(ntb, type, db_count, is_msix); 652 if (ret < 0) 653 ctrl->command_status = COMMAND_STATUS_ERROR; 654 else 655 ctrl->command_status = COMMAND_STATUS_OK; 656 break; 657 case COMMAND_TEARDOWN_DOORBELL: 658 epf_ntb_teardown_db(ntb, type); 659 ctrl->command_status = COMMAND_STATUS_OK; 660 break; 661 case COMMAND_CONFIGURE_MW: 662 ret = epf_ntb_configure_mw(ntb, type, argument); 663 if (ret < 0) 664 ctrl->command_status = COMMAND_STATUS_ERROR; 665 else 666 ctrl->command_status = COMMAND_STATUS_OK; 667 break; 668 case COMMAND_TEARDOWN_MW: 669 epf_ntb_teardown_mw(ntb, type, argument); 670 ctrl->command_status = COMMAND_STATUS_OK; 671 break; 672 case COMMAND_LINK_UP: 673 ntb_epc->linkup = true; 674 if (ntb->epc[PRIMARY_INTERFACE]->linkup && 675 ntb->epc[SECONDARY_INTERFACE]->linkup) { 676 ret = epf_ntb_link_up(ntb, true); 677 if (ret < 0) 678 ctrl->command_status = COMMAND_STATUS_ERROR; 679 else 680 ctrl->command_status = COMMAND_STATUS_OK; 681 goto reset_handler; 682 } 683 ctrl->command_status = COMMAND_STATUS_OK; 684 break; 685 case COMMAND_LINK_DOWN: 686 ntb_epc->linkup = false; 687 ret = epf_ntb_link_up(ntb, false); 688 if (ret < 0) 689 ctrl->command_status = COMMAND_STATUS_ERROR; 690 else 691 ctrl->command_status = COMMAND_STATUS_OK; 692 break; 693 default: 694 dev_err(dev, "%s intf UNKNOWN command: %d\n", 695 pci_epc_interface_string(type), command); 696 break; 697 } 698 699 reset_handler: 700 queue_delayed_work(kpcintb_workqueue, &ntb_epc->cmd_handler, 701 msecs_to_jiffies(5)); 702 } 703 704 /** 705 * epf_ntb_peer_spad_bar_clear() - Clear Peer Scratchpad BAR 706 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound 707 * address. 708 * 709 *+-----------------+------->+------------------+ +-----------------+ 710 *| BAR0 | | CONFIG REGION | | BAR0 | 711 *+-----------------+----+ +------------------+<-------+-----------------+ 712 *| BAR1 | | |SCRATCHPAD REGION | | BAR1 | 713 *+-----------------+ +-->+------------------+<-------+-----------------+ 714 *| BAR2 | Local Memory | BAR2 | 715 *+-----------------+ +-----------------+ 716 *| BAR3 | | BAR3 | 717 *+-----------------+ +-----------------+ 718 *| BAR4 | | BAR4 | 719 *+-----------------+ +-----------------+ 720 *| BAR5 | | BAR5 | 721 *+-----------------+ +-----------------+ 722 * EP CONTROLLER 1 EP CONTROLLER 2 723 * 724 * Clear BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad 725 * region. While BAR1 is the default peer scratchpad BAR, an NTB could have 726 * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs). 727 * This function can get the exact BAR used for peer scratchpad from 728 * epf_ntb_bar[BAR_PEER_SPAD]. 729 * 730 * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function 731 * gets the address of peer scratchpad from 732 * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]. 733 */ 734 static void epf_ntb_peer_spad_bar_clear(struct epf_ntb_epc *ntb_epc) 735 { 736 struct pci_epf_bar *epf_bar; 737 enum pci_barno barno; 738 u8 func_no, vfunc_no; 739 struct pci_epc *epc; 740 741 epc = ntb_epc->epc; 742 func_no = ntb_epc->func_no; 743 vfunc_no = ntb_epc->vfunc_no; 744 barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; 745 epf_bar = &ntb_epc->epf_bar[barno]; 746 pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar); 747 } 748 749 /** 750 * epf_ntb_peer_spad_bar_set() - Set peer scratchpad BAR 751 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 752 * @type: PRIMARY interface or SECONDARY interface 753 * 754 *+-----------------+------->+------------------+ +-----------------+ 755 *| BAR0 | | CONFIG REGION | | BAR0 | 756 *+-----------------+----+ +------------------+<-------+-----------------+ 757 *| BAR1 | | |SCRATCHPAD REGION | | BAR1 | 758 *+-----------------+ +-->+------------------+<-------+-----------------+ 759 *| BAR2 | Local Memory | BAR2 | 760 *+-----------------+ +-----------------+ 761 *| BAR3 | | BAR3 | 762 *+-----------------+ +-----------------+ 763 *| BAR4 | | BAR4 | 764 *+-----------------+ +-----------------+ 765 *| BAR5 | | BAR5 | 766 *+-----------------+ +-----------------+ 767 * EP CONTROLLER 1 EP CONTROLLER 2 768 * 769 * Set BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad 770 * region. While BAR1 is the default peer scratchpad BAR, an NTB could have 771 * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs). 772 * This function can get the exact BAR used for peer scratchpad from 773 * epf_ntb_bar[BAR_PEER_SPAD]. 774 * 775 * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function 776 * gets the address of peer scratchpad from 777 * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]. 778 */ 779 static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb, 780 enum pci_epc_interface_type type) 781 { 782 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 783 struct pci_epf_bar *peer_epf_bar, *epf_bar; 784 enum pci_barno peer_barno, barno; 785 u32 peer_spad_offset; 786 u8 func_no, vfunc_no; 787 struct pci_epc *epc; 788 struct device *dev; 789 int ret; 790 791 dev = &ntb->epf->dev; 792 793 peer_ntb_epc = ntb->epc[!type]; 794 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]; 795 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 796 797 ntb_epc = ntb->epc[type]; 798 barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; 799 epf_bar = &ntb_epc->epf_bar[barno]; 800 func_no = ntb_epc->func_no; 801 vfunc_no = ntb_epc->vfunc_no; 802 epc = ntb_epc->epc; 803 804 peer_spad_offset = peer_ntb_epc->reg->spad_offset; 805 epf_bar->phys_addr = peer_epf_bar->phys_addr + peer_spad_offset; 806 epf_bar->size = peer_ntb_epc->spad_size; 807 epf_bar->barno = barno; 808 epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; 809 810 ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar); 811 if (ret) { 812 dev_err(dev, "%s intf: peer SPAD BAR set failed\n", 813 pci_epc_interface_string(type)); 814 return ret; 815 } 816 817 return 0; 818 } 819 820 /** 821 * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR 822 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound 823 * address. 824 * 825 * +-----------------+------->+------------------+ +-----------------+ 826 * | BAR0 | | CONFIG REGION | | BAR0 | 827 * +-----------------+----+ +------------------+<-------+-----------------+ 828 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | 829 * +-----------------+ +-->+------------------+<-------+-----------------+ 830 * | BAR2 | Local Memory | BAR2 | 831 * +-----------------+ +-----------------+ 832 * | BAR3 | | BAR3 | 833 * +-----------------+ +-----------------+ 834 * | BAR4 | | BAR4 | 835 * +-----------------+ +-----------------+ 836 * | BAR5 | | BAR5 | 837 * +-----------------+ +-----------------+ 838 * EP CONTROLLER 1 EP CONTROLLER 2 839 * 840 * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and 841 * self scratchpad region (removes inbound ATU configuration). While BAR0 is 842 * the default self scratchpad BAR, an NTB could have other BARs for self 843 * scratchpad (because of reserved BARs). This function can get the exact BAR 844 * used for self scratchpad from epf_ntb_bar[BAR_CONFIG]. 845 * 846 * Please note the self scratchpad region and config region is combined to 847 * a single region and mapped using the same BAR. Also note HOST2's peer 848 * scratchpad is HOST1's self scratchpad. 849 */ 850 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb_epc *ntb_epc) 851 { 852 struct pci_epf_bar *epf_bar; 853 enum pci_barno barno; 854 u8 func_no, vfunc_no; 855 struct pci_epc *epc; 856 857 epc = ntb_epc->epc; 858 func_no = ntb_epc->func_no; 859 vfunc_no = ntb_epc->vfunc_no; 860 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; 861 epf_bar = &ntb_epc->epf_bar[barno]; 862 pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar); 863 } 864 865 /** 866 * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR 867 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound 868 * address. 869 * 870 * +-----------------+------->+------------------+ +-----------------+ 871 * | BAR0 | | CONFIG REGION | | BAR0 | 872 * +-----------------+----+ +------------------+<-------+-----------------+ 873 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | 874 * +-----------------+ +-->+------------------+<-------+-----------------+ 875 * | BAR2 | Local Memory | BAR2 | 876 * +-----------------+ +-----------------+ 877 * | BAR3 | | BAR3 | 878 * +-----------------+ +-----------------+ 879 * | BAR4 | | BAR4 | 880 * +-----------------+ +-----------------+ 881 * | BAR5 | | BAR5 | 882 * +-----------------+ +-----------------+ 883 * EP CONTROLLER 1 EP CONTROLLER 2 884 * 885 * Map BAR0 of EP CONTROLLER 1 which contains the HOST1's config and 886 * self scratchpad region. While BAR0 is the default self scratchpad BAR, an 887 * NTB could have other BARs for self scratchpad (because of reserved BARs). 888 * This function can get the exact BAR used for self scratchpad from 889 * epf_ntb_bar[BAR_CONFIG]. 890 * 891 * Please note the self scratchpad region and config region is combined to 892 * a single region and mapped using the same BAR. Also note HOST2's peer 893 * scratchpad is HOST1's self scratchpad. 894 */ 895 static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc) 896 { 897 struct pci_epf_bar *epf_bar; 898 enum pci_barno barno; 899 u8 func_no, vfunc_no; 900 struct epf_ntb *ntb; 901 struct pci_epc *epc; 902 struct device *dev; 903 int ret; 904 905 ntb = ntb_epc->epf_ntb; 906 dev = &ntb->epf->dev; 907 908 epc = ntb_epc->epc; 909 func_no = ntb_epc->func_no; 910 vfunc_no = ntb_epc->vfunc_no; 911 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; 912 epf_bar = &ntb_epc->epf_bar[barno]; 913 914 ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar); 915 if (ret) { 916 dev_err(dev, "%s inft: Config/Status/SPAD BAR set failed\n", 917 pci_epc_interface_string(ntb_epc->type)); 918 return ret; 919 } 920 921 return 0; 922 } 923 924 /** 925 * epf_ntb_config_spad_bar_free() - Free the physical memory associated with 926 * config + scratchpad region 927 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 928 * 929 * +-----------------+------->+------------------+ +-----------------+ 930 * | BAR0 | | CONFIG REGION | | BAR0 | 931 * +-----------------+----+ +------------------+<-------+-----------------+ 932 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | 933 * +-----------------+ +-->+------------------+<-------+-----------------+ 934 * | BAR2 | Local Memory | BAR2 | 935 * +-----------------+ +-----------------+ 936 * | BAR3 | | BAR3 | 937 * +-----------------+ +-----------------+ 938 * | BAR4 | | BAR4 | 939 * +-----------------+ +-----------------+ 940 * | BAR5 | | BAR5 | 941 * +-----------------+ +-----------------+ 942 * EP CONTROLLER 1 EP CONTROLLER 2 943 * 944 * Free the Local Memory mentioned in the above diagram. After invoking this 945 * function, any of config + self scratchpad region of HOST1 or peer scratchpad 946 * region of HOST2 should not be accessed. 947 */ 948 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb) 949 { 950 enum pci_epc_interface_type type; 951 struct epf_ntb_epc *ntb_epc; 952 enum pci_barno barno; 953 struct pci_epf *epf; 954 955 epf = ntb->epf; 956 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 957 ntb_epc = ntb->epc[type]; 958 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; 959 if (ntb_epc->reg) 960 pci_epf_free_space(epf, ntb_epc->reg, barno, type); 961 } 962 } 963 964 /** 965 * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad 966 * region 967 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 968 * @type: PRIMARY interface or SECONDARY interface 969 * 970 * +-----------------+------->+------------------+ +-----------------+ 971 * | BAR0 | | CONFIG REGION | | BAR0 | 972 * +-----------------+----+ +------------------+<-------+-----------------+ 973 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | 974 * +-----------------+ +-->+------------------+<-------+-----------------+ 975 * | BAR2 | Local Memory | BAR2 | 976 * +-----------------+ +-----------------+ 977 * | BAR3 | | BAR3 | 978 * +-----------------+ +-----------------+ 979 * | BAR4 | | BAR4 | 980 * +-----------------+ +-----------------+ 981 * | BAR5 | | BAR5 | 982 * +-----------------+ +-----------------+ 983 * EP CONTROLLER 1 EP CONTROLLER 2 984 * 985 * Allocate the Local Memory mentioned in the above diagram. The size of 986 * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION 987 * is obtained from "spad-count" configfs entry. 988 * 989 * The size of both config region and scratchpad region has to be aligned, 990 * since the scratchpad region will also be mapped as PEER SCRATCHPAD of 991 * other host using a separate BAR. 992 */ 993 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb, 994 enum pci_epc_interface_type type) 995 { 996 const struct pci_epc_features *peer_epc_features, *epc_features; 997 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 998 size_t msix_table_size, pba_size, align; 999 enum pci_barno peer_barno, barno; 1000 struct epf_ntb_ctrl *ctrl; 1001 u32 spad_size, ctrl_size; 1002 u64 size, peer_size; 1003 struct pci_epf *epf; 1004 struct device *dev; 1005 bool msix_capable; 1006 u32 spad_count; 1007 void *base; 1008 1009 epf = ntb->epf; 1010 dev = &epf->dev; 1011 ntb_epc = ntb->epc[type]; 1012 1013 epc_features = ntb_epc->epc_features; 1014 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; 1015 size = epc_features->bar_fixed_size[barno]; 1016 align = epc_features->align; 1017 1018 peer_ntb_epc = ntb->epc[!type]; 1019 peer_epc_features = peer_ntb_epc->epc_features; 1020 peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; 1021 peer_size = peer_epc_features->bar_fixed_size[peer_barno]; 1022 1023 /* Check if epc_features is populated incorrectly */ 1024 if ((!IS_ALIGNED(size, align))) 1025 return -EINVAL; 1026 1027 spad_count = ntb->spad_count; 1028 1029 ctrl_size = sizeof(struct epf_ntb_ctrl); 1030 spad_size = spad_count * 4; 1031 1032 msix_capable = epc_features->msix_capable; 1033 if (msix_capable) { 1034 msix_table_size = PCI_MSIX_ENTRY_SIZE * ntb->db_count; 1035 ctrl_size = ALIGN(ctrl_size, 8); 1036 ntb_epc->msix_table_offset = ctrl_size; 1037 ntb_epc->msix_bar = barno; 1038 /* Align to QWORD or 8 Bytes */ 1039 pba_size = ALIGN(DIV_ROUND_UP(ntb->db_count, 8), 8); 1040 ctrl_size = ctrl_size + msix_table_size + pba_size; 1041 } 1042 1043 if (!align) { 1044 ctrl_size = roundup_pow_of_two(ctrl_size); 1045 spad_size = roundup_pow_of_two(spad_size); 1046 } else { 1047 ctrl_size = ALIGN(ctrl_size, align); 1048 spad_size = ALIGN(spad_size, align); 1049 } 1050 1051 if (peer_size) { 1052 if (peer_size < spad_size) 1053 spad_count = peer_size / 4; 1054 spad_size = peer_size; 1055 } 1056 1057 /* 1058 * In order to make sure SPAD offset is aligned to its size, 1059 * expand control region size to the size of SPAD if SPAD size 1060 * is greater than control region size. 1061 */ 1062 if (spad_size > ctrl_size) 1063 ctrl_size = spad_size; 1064 1065 if (!size) 1066 size = ctrl_size + spad_size; 1067 else if (size < ctrl_size + spad_size) 1068 return -EINVAL; 1069 1070 base = pci_epf_alloc_space(epf, size, barno, align, type); 1071 if (!base) { 1072 dev_err(dev, "%s intf: Config/Status/SPAD alloc region fail\n", 1073 pci_epc_interface_string(type)); 1074 return -ENOMEM; 1075 } 1076 1077 ntb_epc->reg = base; 1078 1079 ctrl = ntb_epc->reg; 1080 ctrl->spad_offset = ctrl_size; 1081 ctrl->spad_count = spad_count; 1082 ctrl->num_mws = ntb->num_mws; 1083 ctrl->db_entry_size = align ? align : 4; 1084 ntb_epc->spad_size = spad_size; 1085 1086 return 0; 1087 } 1088 1089 /** 1090 * epf_ntb_config_spad_bar_alloc_interface() - Allocate memory for config + 1091 * scratchpad region for each of PRIMARY and SECONDARY interface 1092 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1093 * 1094 * Wrapper for epf_ntb_config_spad_bar_alloc() which allocates memory for 1095 * config + scratchpad region for a specific interface 1096 */ 1097 static int epf_ntb_config_spad_bar_alloc_interface(struct epf_ntb *ntb) 1098 { 1099 enum pci_epc_interface_type type; 1100 struct device *dev; 1101 int ret; 1102 1103 dev = &ntb->epf->dev; 1104 1105 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 1106 ret = epf_ntb_config_spad_bar_alloc(ntb, type); 1107 if (ret) { 1108 dev_err(dev, "%s intf: Config/SPAD BAR alloc failed\n", 1109 pci_epc_interface_string(type)); 1110 return ret; 1111 } 1112 } 1113 1114 return 0; 1115 } 1116 1117 /** 1118 * epf_ntb_free_peer_mem() - Free memory allocated in peers outbound address 1119 * space 1120 * @ntb_epc: EPC associated with one of the HOST which holds peers outbound 1121 * address regions 1122 * 1123 * +-----------------+ +---->+----------------+-----------+-----------------+ 1124 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | 1125 * +-----------------+ | +----------------+ +-----------------+ 1126 * | BAR1 | | | Doorbell 2 +---------+ | | 1127 * +-----------------+----+ +----------------+ | | | 1128 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ 1129 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | 1130 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 1131 * +-----------------+ | |----------------+ | | | | 1132 * | BAR4 | | | | | | +-----------------+ 1133 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| 1134 * | BAR5 | | | | | | +-----------------+ 1135 * +-----------------+ +---->-----------------+ | | | | 1136 * EP CONTROLLER 1 | | | | +-----------------+ 1137 * | | | +---->+ MSI|X ADDRESS 4 | 1138 * +----------------+ | +-----------------+ 1139 * (A) EP CONTROLLER 2 | | | 1140 * (OB SPACE) | | | 1141 * +-------> MW1 | 1142 * | | 1143 * | | 1144 * (B) +-----------------+ 1145 * | | 1146 * | | 1147 * | | 1148 * | | 1149 * | | 1150 * +-----------------+ 1151 * PCI Address Space 1152 * (Managed by HOST2) 1153 * 1154 * Free memory allocated in EP CONTROLLER 2 (OB SPACE) in the above diagram. 1155 * It'll free Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, 1156 * MW4). 1157 */ 1158 static void epf_ntb_free_peer_mem(struct epf_ntb_epc *ntb_epc) 1159 { 1160 struct pci_epf_bar *epf_bar; 1161 void __iomem *mw_addr; 1162 phys_addr_t phys_addr; 1163 enum epf_ntb_bar bar; 1164 enum pci_barno barno; 1165 struct pci_epc *epc; 1166 size_t size; 1167 1168 epc = ntb_epc->epc; 1169 1170 for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) { 1171 barno = ntb_epc->epf_ntb_bar[bar]; 1172 mw_addr = ntb_epc->mw_addr[barno]; 1173 epf_bar = &ntb_epc->epf_bar[barno]; 1174 phys_addr = epf_bar->phys_addr; 1175 size = epf_bar->size; 1176 if (mw_addr) { 1177 pci_epc_mem_free_addr(epc, phys_addr, mw_addr, size); 1178 ntb_epc->mw_addr[barno] = NULL; 1179 } 1180 } 1181 } 1182 1183 /** 1184 * epf_ntb_db_mw_bar_clear() - Clear doorbell and memory BAR 1185 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound 1186 * address 1187 * 1188 * +-----------------+ +---->+----------------+-----------+-----------------+ 1189 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | 1190 * +-----------------+ | +----------------+ +-----------------+ 1191 * | BAR1 | | | Doorbell 2 +---------+ | | 1192 * +-----------------+----+ +----------------+ | | | 1193 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ 1194 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | 1195 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 1196 * +-----------------+ | |----------------+ | | | | 1197 * | BAR4 | | | | | | +-----------------+ 1198 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| 1199 * | BAR5 | | | | | | +-----------------+ 1200 * +-----------------+ +---->-----------------+ | | | | 1201 * EP CONTROLLER 1 | | | | +-----------------+ 1202 * | | | +---->+ MSI|X ADDRESS 4 | 1203 * +----------------+ | +-----------------+ 1204 * (A) EP CONTROLLER 2 | | | 1205 * (OB SPACE) | | | 1206 * +-------> MW1 | 1207 * | | 1208 * | | 1209 * (B) +-----------------+ 1210 * | | 1211 * | | 1212 * | | 1213 * | | 1214 * | | 1215 * +-----------------+ 1216 * PCI Address Space 1217 * (Managed by HOST2) 1218 * 1219 * Clear doorbell and memory BARs (remove inbound ATU configuration). In the above 1220 * diagram it clears BAR2 TO BAR5 of EP CONTROLLER 1 (Doorbell BAR, MW1 BAR, MW2 1221 * BAR, MW3 BAR and MW4 BAR). 1222 */ 1223 static void epf_ntb_db_mw_bar_clear(struct epf_ntb_epc *ntb_epc) 1224 { 1225 struct pci_epf_bar *epf_bar; 1226 enum epf_ntb_bar bar; 1227 enum pci_barno barno; 1228 u8 func_no, vfunc_no; 1229 struct pci_epc *epc; 1230 1231 epc = ntb_epc->epc; 1232 1233 func_no = ntb_epc->func_no; 1234 vfunc_no = ntb_epc->vfunc_no; 1235 1236 for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) { 1237 barno = ntb_epc->epf_ntb_bar[bar]; 1238 epf_bar = &ntb_epc->epf_bar[barno]; 1239 pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar); 1240 } 1241 } 1242 1243 /** 1244 * epf_ntb_db_mw_bar_cleanup() - Clear doorbell/memory BAR and free memory 1245 * allocated in peers outbound address space 1246 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1247 * @type: PRIMARY interface or SECONDARY interface 1248 * 1249 * Wrapper for epf_ntb_db_mw_bar_clear() to clear HOST1's BAR and 1250 * epf_ntb_free_peer_mem() which frees up HOST2 outbound memory. 1251 */ 1252 static void epf_ntb_db_mw_bar_cleanup(struct epf_ntb *ntb, 1253 enum pci_epc_interface_type type) 1254 { 1255 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 1256 1257 ntb_epc = ntb->epc[type]; 1258 peer_ntb_epc = ntb->epc[!type]; 1259 1260 epf_ntb_db_mw_bar_clear(ntb_epc); 1261 epf_ntb_free_peer_mem(peer_ntb_epc); 1262 } 1263 1264 /** 1265 * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability 1266 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1267 * @type: PRIMARY interface or SECONDARY interface 1268 * 1269 * Configure MSI/MSI-X capability for each interface with number of 1270 * interrupts equal to "db_count" configfs entry. 1271 */ 1272 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb, 1273 enum pci_epc_interface_type type) 1274 { 1275 const struct pci_epc_features *epc_features; 1276 bool msix_capable, msi_capable; 1277 struct epf_ntb_epc *ntb_epc; 1278 u8 func_no, vfunc_no; 1279 struct pci_epc *epc; 1280 struct device *dev; 1281 u32 db_count; 1282 int ret; 1283 1284 ntb_epc = ntb->epc[type]; 1285 dev = &ntb->epf->dev; 1286 1287 epc_features = ntb_epc->epc_features; 1288 msix_capable = epc_features->msix_capable; 1289 msi_capable = epc_features->msi_capable; 1290 1291 if (!(msix_capable || msi_capable)) { 1292 dev_err(dev, "MSI or MSI-X is required for doorbell\n"); 1293 return -EINVAL; 1294 } 1295 1296 func_no = ntb_epc->func_no; 1297 vfunc_no = ntb_epc->vfunc_no; 1298 1299 db_count = ntb->db_count; 1300 if (db_count > MAX_DB_COUNT) { 1301 dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT); 1302 return -EINVAL; 1303 } 1304 1305 ntb->db_count = db_count; 1306 epc = ntb_epc->epc; 1307 1308 if (msi_capable) { 1309 ret = pci_epc_set_msi(epc, func_no, vfunc_no, db_count); 1310 if (ret) { 1311 dev_err(dev, "%s intf: MSI configuration failed\n", 1312 pci_epc_interface_string(type)); 1313 return ret; 1314 } 1315 } 1316 1317 if (msix_capable) { 1318 ret = pci_epc_set_msix(epc, func_no, vfunc_no, db_count, 1319 ntb_epc->msix_bar, 1320 ntb_epc->msix_table_offset); 1321 if (ret) { 1322 dev_err(dev, "MSI configuration failed\n"); 1323 return ret; 1324 } 1325 } 1326 1327 return 0; 1328 } 1329 1330 /** 1331 * epf_ntb_alloc_peer_mem() - Allocate memory in peer's outbound address space 1332 * @dev: The PCI device. 1333 * @ntb_epc: EPC associated with one of the HOST whose BAR holds peer's outbound 1334 * address 1335 * @bar: BAR of @ntb_epc in for which memory has to be allocated (could be 1336 * BAR_DB_MW1, BAR_MW2, BAR_MW3, BAR_MW4) 1337 * @peer_ntb_epc: EPC associated with HOST whose outbound address space is 1338 * used by @ntb_epc 1339 * @size: Size of the address region that has to be allocated in peers OB SPACE 1340 * 1341 * 1342 * +-----------------+ +---->+----------------+-----------+-----------------+ 1343 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | 1344 * +-----------------+ | +----------------+ +-----------------+ 1345 * | BAR1 | | | Doorbell 2 +---------+ | | 1346 * +-----------------+----+ +----------------+ | | | 1347 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ 1348 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | 1349 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 1350 * +-----------------+ | |----------------+ | | | | 1351 * | BAR4 | | | | | | +-----------------+ 1352 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| 1353 * | BAR5 | | | | | | +-----------------+ 1354 * +-----------------+ +---->-----------------+ | | | | 1355 * EP CONTROLLER 1 | | | | +-----------------+ 1356 * | | | +---->+ MSI|X ADDRESS 4 | 1357 * +----------------+ | +-----------------+ 1358 * (A) EP CONTROLLER 2 | | | 1359 * (OB SPACE) | | | 1360 * +-------> MW1 | 1361 * | | 1362 * | | 1363 * (B) +-----------------+ 1364 * | | 1365 * | | 1366 * | | 1367 * | | 1368 * | | 1369 * +-----------------+ 1370 * PCI Address Space 1371 * (Managed by HOST2) 1372 * 1373 * Allocate memory in OB space of EP CONTROLLER 2 in the above diagram. Allocate 1374 * for Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, MW4). 1375 */ 1376 static int epf_ntb_alloc_peer_mem(struct device *dev, 1377 struct epf_ntb_epc *ntb_epc, 1378 enum epf_ntb_bar bar, 1379 struct epf_ntb_epc *peer_ntb_epc, 1380 size_t size) 1381 { 1382 const struct pci_epc_features *epc_features; 1383 struct pci_epf_bar *epf_bar; 1384 struct pci_epc *peer_epc; 1385 phys_addr_t phys_addr; 1386 void __iomem *mw_addr; 1387 enum pci_barno barno; 1388 size_t align; 1389 1390 epc_features = ntb_epc->epc_features; 1391 align = epc_features->align; 1392 1393 if (size < 128) 1394 size = 128; 1395 1396 if (align) 1397 size = ALIGN(size, align); 1398 else 1399 size = roundup_pow_of_two(size); 1400 1401 peer_epc = peer_ntb_epc->epc; 1402 mw_addr = pci_epc_mem_alloc_addr(peer_epc, &phys_addr, size); 1403 if (!mw_addr) { 1404 dev_err(dev, "%s intf: Failed to allocate OB address\n", 1405 pci_epc_interface_string(peer_ntb_epc->type)); 1406 return -ENOMEM; 1407 } 1408 1409 barno = ntb_epc->epf_ntb_bar[bar]; 1410 epf_bar = &ntb_epc->epf_bar[barno]; 1411 ntb_epc->mw_addr[barno] = mw_addr; 1412 1413 epf_bar->phys_addr = phys_addr; 1414 epf_bar->size = size; 1415 epf_bar->barno = barno; 1416 epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; 1417 1418 return 0; 1419 } 1420 1421 /** 1422 * epf_ntb_db_mw_bar_init() - Configure Doorbell and Memory window BARs 1423 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1424 * @type: PRIMARY interface or SECONDARY interface 1425 * 1426 * Wrapper for epf_ntb_alloc_peer_mem() and pci_epc_set_bar() that allocates 1427 * memory in OB address space of HOST2 and configures BAR of HOST1 1428 */ 1429 static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb, 1430 enum pci_epc_interface_type type) 1431 { 1432 const struct pci_epc_features *epc_features; 1433 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 1434 struct pci_epf_bar *epf_bar; 1435 struct epf_ntb_ctrl *ctrl; 1436 u32 num_mws, db_count; 1437 enum epf_ntb_bar bar; 1438 enum pci_barno barno; 1439 u8 func_no, vfunc_no; 1440 struct pci_epc *epc; 1441 struct device *dev; 1442 size_t align; 1443 int ret, i; 1444 u64 size; 1445 1446 ntb_epc = ntb->epc[type]; 1447 peer_ntb_epc = ntb->epc[!type]; 1448 1449 dev = &ntb->epf->dev; 1450 epc_features = ntb_epc->epc_features; 1451 align = epc_features->align; 1452 func_no = ntb_epc->func_no; 1453 vfunc_no = ntb_epc->vfunc_no; 1454 epc = ntb_epc->epc; 1455 num_mws = ntb->num_mws; 1456 db_count = ntb->db_count; 1457 1458 for (bar = BAR_DB_MW1, i = 0; i < num_mws; bar++, i++) { 1459 if (bar == BAR_DB_MW1) { 1460 align = align ? align : 4; 1461 size = db_count * align; 1462 size = ALIGN(size, ntb->mws_size[i]); 1463 ctrl = ntb_epc->reg; 1464 ctrl->mw1_offset = size; 1465 size += ntb->mws_size[i]; 1466 } else { 1467 size = ntb->mws_size[i]; 1468 } 1469 1470 ret = epf_ntb_alloc_peer_mem(dev, ntb_epc, bar, 1471 peer_ntb_epc, size); 1472 if (ret) { 1473 dev_err(dev, "%s intf: DoorBell mem alloc failed\n", 1474 pci_epc_interface_string(type)); 1475 goto err_alloc_peer_mem; 1476 } 1477 1478 barno = ntb_epc->epf_ntb_bar[bar]; 1479 epf_bar = &ntb_epc->epf_bar[barno]; 1480 1481 ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar); 1482 if (ret) { 1483 dev_err(dev, "%s intf: DoorBell BAR set failed\n", 1484 pci_epc_interface_string(type)); 1485 goto err_alloc_peer_mem; 1486 } 1487 } 1488 1489 return 0; 1490 1491 err_alloc_peer_mem: 1492 epf_ntb_db_mw_bar_cleanup(ntb, type); 1493 1494 return ret; 1495 } 1496 1497 /** 1498 * epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface 1499 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1500 * @type: PRIMARY interface or SECONDARY interface 1501 * 1502 * Unbind NTB function device from EPC and relinquish reference to pci_epc 1503 * for each of the interface. 1504 */ 1505 static void epf_ntb_epc_destroy_interface(struct epf_ntb *ntb, 1506 enum pci_epc_interface_type type) 1507 { 1508 struct epf_ntb_epc *ntb_epc; 1509 struct pci_epc *epc; 1510 struct pci_epf *epf; 1511 1512 if (type < 0) 1513 return; 1514 1515 epf = ntb->epf; 1516 ntb_epc = ntb->epc[type]; 1517 if (!ntb_epc) 1518 return; 1519 epc = ntb_epc->epc; 1520 pci_epc_remove_epf(epc, epf, type); 1521 pci_epc_put(epc); 1522 } 1523 1524 /** 1525 * epf_ntb_epc_destroy() - Cleanup NTB EPC interface 1526 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1527 * 1528 * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces 1529 */ 1530 static void epf_ntb_epc_destroy(struct epf_ntb *ntb) 1531 { 1532 enum pci_epc_interface_type type; 1533 1534 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) 1535 epf_ntb_epc_destroy_interface(ntb, type); 1536 } 1537 1538 /** 1539 * epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface 1540 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1541 * @epc: struct pci_epc to which a particular NTB interface should be associated 1542 * @type: PRIMARY interface or SECONDARY interface 1543 * 1544 * Allocate memory for NTB EPC interface and initialize it. 1545 */ 1546 static int epf_ntb_epc_create_interface(struct epf_ntb *ntb, 1547 struct pci_epc *epc, 1548 enum pci_epc_interface_type type) 1549 { 1550 const struct pci_epc_features *epc_features; 1551 struct pci_epf_bar *epf_bar; 1552 struct epf_ntb_epc *ntb_epc; 1553 u8 func_no, vfunc_no; 1554 struct pci_epf *epf; 1555 struct device *dev; 1556 1557 dev = &ntb->epf->dev; 1558 1559 ntb_epc = devm_kzalloc(dev, sizeof(*ntb_epc), GFP_KERNEL); 1560 if (!ntb_epc) 1561 return -ENOMEM; 1562 1563 epf = ntb->epf; 1564 vfunc_no = epf->vfunc_no; 1565 if (type == PRIMARY_INTERFACE) { 1566 func_no = epf->func_no; 1567 epf_bar = epf->bar; 1568 } else { 1569 func_no = epf->sec_epc_func_no; 1570 epf_bar = epf->sec_epc_bar; 1571 } 1572 1573 ntb_epc->linkup = false; 1574 ntb_epc->epc = epc; 1575 ntb_epc->func_no = func_no; 1576 ntb_epc->vfunc_no = vfunc_no; 1577 ntb_epc->type = type; 1578 ntb_epc->epf_bar = epf_bar; 1579 ntb_epc->epf_ntb = ntb; 1580 1581 epc_features = pci_epc_get_features(epc, func_no, vfunc_no); 1582 if (!epc_features) 1583 return -EINVAL; 1584 ntb_epc->epc_features = epc_features; 1585 1586 ntb->epc[type] = ntb_epc; 1587 1588 return 0; 1589 } 1590 1591 /** 1592 * epf_ntb_epc_create() - Create and initialize NTB EPC interface 1593 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1594 * 1595 * Get a reference to EPC device and bind NTB function device to that EPC 1596 * for each of the interface. It is also a wrapper to 1597 * epf_ntb_epc_create_interface() to allocate memory for NTB EPC interface 1598 * and initialize it 1599 */ 1600 static int epf_ntb_epc_create(struct epf_ntb *ntb) 1601 { 1602 struct pci_epf *epf; 1603 struct device *dev; 1604 int ret; 1605 1606 epf = ntb->epf; 1607 dev = &epf->dev; 1608 1609 ret = epf_ntb_epc_create_interface(ntb, epf->epc, PRIMARY_INTERFACE); 1610 if (ret) { 1611 dev_err(dev, "PRIMARY intf: Fail to create NTB EPC\n"); 1612 return ret; 1613 } 1614 1615 ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc, 1616 SECONDARY_INTERFACE); 1617 if (ret) { 1618 dev_err(dev, "SECONDARY intf: Fail to create NTB EPC\n"); 1619 goto err_epc_create; 1620 } 1621 1622 return 0; 1623 1624 err_epc_create: 1625 epf_ntb_epc_destroy_interface(ntb, PRIMARY_INTERFACE); 1626 1627 return ret; 1628 } 1629 1630 /** 1631 * epf_ntb_init_epc_bar_interface() - Identify BARs to be used for each of 1632 * the NTB constructs (scratchpad region, doorbell, memorywindow) 1633 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1634 * @type: PRIMARY interface or SECONDARY interface 1635 * 1636 * Identify the free BARs to be used for each of BAR_CONFIG, BAR_PEER_SPAD, 1637 * BAR_DB_MW1, BAR_MW2, BAR_MW3 and BAR_MW4. 1638 */ 1639 static int epf_ntb_init_epc_bar_interface(struct epf_ntb *ntb, 1640 enum pci_epc_interface_type type) 1641 { 1642 const struct pci_epc_features *epc_features; 1643 struct epf_ntb_epc *ntb_epc; 1644 enum pci_barno barno; 1645 enum epf_ntb_bar bar; 1646 struct device *dev; 1647 u32 num_mws; 1648 int i; 1649 1650 barno = BAR_0; 1651 ntb_epc = ntb->epc[type]; 1652 num_mws = ntb->num_mws; 1653 dev = &ntb->epf->dev; 1654 epc_features = ntb_epc->epc_features; 1655 1656 /* These are required BARs which are mandatory for NTB functionality */ 1657 for (bar = BAR_CONFIG; bar <= BAR_DB_MW1; bar++, barno++) { 1658 barno = pci_epc_get_next_free_bar(epc_features, barno); 1659 if (barno < 0) { 1660 dev_err(dev, "%s intf: Fail to get NTB function BAR\n", 1661 pci_epc_interface_string(type)); 1662 return barno; 1663 } 1664 ntb_epc->epf_ntb_bar[bar] = barno; 1665 } 1666 1667 /* These are optional BARs which don't impact NTB functionality */ 1668 for (bar = BAR_MW2, i = 1; i < num_mws; bar++, barno++, i++) { 1669 barno = pci_epc_get_next_free_bar(epc_features, barno); 1670 if (barno < 0) { 1671 ntb->num_mws = i; 1672 dev_dbg(dev, "BAR not available for > MW%d\n", i + 1); 1673 } 1674 ntb_epc->epf_ntb_bar[bar] = barno; 1675 } 1676 1677 return 0; 1678 } 1679 1680 /** 1681 * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB 1682 * constructs (scratchpad region, doorbell, memorywindow) 1683 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1684 * 1685 * Wrapper to epf_ntb_init_epc_bar_interface() to identify the free BARs 1686 * to be used for each of BAR_CONFIG, BAR_PEER_SPAD, BAR_DB_MW1, BAR_MW2, 1687 * BAR_MW3 and BAR_MW4 for all the interfaces. 1688 */ 1689 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb) 1690 { 1691 enum pci_epc_interface_type type; 1692 struct device *dev; 1693 int ret; 1694 1695 dev = &ntb->epf->dev; 1696 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 1697 ret = epf_ntb_init_epc_bar_interface(ntb, type); 1698 if (ret) { 1699 dev_err(dev, "Fail to init EPC bar for %s interface\n", 1700 pci_epc_interface_string(type)); 1701 return ret; 1702 } 1703 } 1704 1705 return 0; 1706 } 1707 1708 /** 1709 * epf_ntb_epc_init_interface() - Initialize NTB interface 1710 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1711 * @type: PRIMARY interface or SECONDARY interface 1712 * 1713 * Wrapper to initialize a particular EPC interface and start the workqueue 1714 * to check for commands from host. This function will write to the 1715 * EP controller HW for configuring it. 1716 */ 1717 static int epf_ntb_epc_init_interface(struct epf_ntb *ntb, 1718 enum pci_epc_interface_type type) 1719 { 1720 struct epf_ntb_epc *ntb_epc; 1721 u8 func_no, vfunc_no; 1722 struct pci_epc *epc; 1723 struct pci_epf *epf; 1724 struct device *dev; 1725 int ret; 1726 1727 ntb_epc = ntb->epc[type]; 1728 epf = ntb->epf; 1729 dev = &epf->dev; 1730 epc = ntb_epc->epc; 1731 func_no = ntb_epc->func_no; 1732 vfunc_no = ntb_epc->vfunc_no; 1733 1734 ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]); 1735 if (ret) { 1736 dev_err(dev, "%s intf: Config/self SPAD BAR init failed\n", 1737 pci_epc_interface_string(type)); 1738 return ret; 1739 } 1740 1741 ret = epf_ntb_peer_spad_bar_set(ntb, type); 1742 if (ret) { 1743 dev_err(dev, "%s intf: Peer SPAD BAR init failed\n", 1744 pci_epc_interface_string(type)); 1745 goto err_peer_spad_bar_init; 1746 } 1747 1748 ret = epf_ntb_configure_interrupt(ntb, type); 1749 if (ret) { 1750 dev_err(dev, "%s intf: Interrupt configuration failed\n", 1751 pci_epc_interface_string(type)); 1752 goto err_peer_spad_bar_init; 1753 } 1754 1755 ret = epf_ntb_db_mw_bar_init(ntb, type); 1756 if (ret) { 1757 dev_err(dev, "%s intf: DB/MW BAR init failed\n", 1758 pci_epc_interface_string(type)); 1759 goto err_db_mw_bar_init; 1760 } 1761 1762 if (vfunc_no <= 1) { 1763 ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header); 1764 if (ret) { 1765 dev_err(dev, "%s intf: Configuration header write failed\n", 1766 pci_epc_interface_string(type)); 1767 goto err_write_header; 1768 } 1769 } 1770 1771 INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler); 1772 queue_work(kpcintb_workqueue, &ntb->epc[type]->cmd_handler.work); 1773 1774 return 0; 1775 1776 err_write_header: 1777 epf_ntb_db_mw_bar_cleanup(ntb, type); 1778 1779 err_db_mw_bar_init: 1780 epf_ntb_peer_spad_bar_clear(ntb->epc[type]); 1781 1782 err_peer_spad_bar_init: 1783 epf_ntb_config_sspad_bar_clear(ntb->epc[type]); 1784 1785 return ret; 1786 } 1787 1788 /** 1789 * epf_ntb_epc_cleanup_interface() - Cleanup NTB interface 1790 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1791 * @type: PRIMARY interface or SECONDARY interface 1792 * 1793 * Wrapper to cleanup a particular NTB interface. 1794 */ 1795 static void epf_ntb_epc_cleanup_interface(struct epf_ntb *ntb, 1796 enum pci_epc_interface_type type) 1797 { 1798 struct epf_ntb_epc *ntb_epc; 1799 1800 if (type < 0) 1801 return; 1802 1803 ntb_epc = ntb->epc[type]; 1804 cancel_delayed_work(&ntb_epc->cmd_handler); 1805 epf_ntb_db_mw_bar_cleanup(ntb, type); 1806 epf_ntb_peer_spad_bar_clear(ntb_epc); 1807 epf_ntb_config_sspad_bar_clear(ntb_epc); 1808 } 1809 1810 /** 1811 * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces 1812 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1813 * 1814 * Wrapper to cleanup all NTB interfaces. 1815 */ 1816 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb) 1817 { 1818 enum pci_epc_interface_type type; 1819 1820 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) 1821 epf_ntb_epc_cleanup_interface(ntb, type); 1822 } 1823 1824 /** 1825 * epf_ntb_epc_init() - Initialize all NTB interfaces 1826 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1827 * 1828 * Wrapper to initialize all NTB interface and start the workqueue 1829 * to check for commands from host. 1830 */ 1831 static int epf_ntb_epc_init(struct epf_ntb *ntb) 1832 { 1833 enum pci_epc_interface_type type; 1834 struct device *dev; 1835 int ret; 1836 1837 dev = &ntb->epf->dev; 1838 1839 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 1840 ret = epf_ntb_epc_init_interface(ntb, type); 1841 if (ret) { 1842 dev_err(dev, "%s intf: Failed to initialize\n", 1843 pci_epc_interface_string(type)); 1844 goto err_init_type; 1845 } 1846 } 1847 1848 return 0; 1849 1850 err_init_type: 1851 epf_ntb_epc_cleanup_interface(ntb, type - 1); 1852 1853 return ret; 1854 } 1855 1856 /** 1857 * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality 1858 * @epf: NTB endpoint function device 1859 * 1860 * Initialize both the endpoint controllers associated with NTB function device. 1861 * Invoked when a primary interface or secondary interface is bound to EPC 1862 * device. This function will succeed only when EPC is bound to both the 1863 * interfaces. 1864 */ 1865 static int epf_ntb_bind(struct pci_epf *epf) 1866 { 1867 struct epf_ntb *ntb = epf_get_drvdata(epf); 1868 struct device *dev = &epf->dev; 1869 int ret; 1870 1871 if (!epf->epc) { 1872 dev_dbg(dev, "PRIMARY EPC interface not yet bound\n"); 1873 return 0; 1874 } 1875 1876 if (!epf->sec_epc) { 1877 dev_dbg(dev, "SECONDARY EPC interface not yet bound\n"); 1878 return 0; 1879 } 1880 1881 ret = epf_ntb_epc_create(ntb); 1882 if (ret) { 1883 dev_err(dev, "Failed to create NTB EPC\n"); 1884 return ret; 1885 } 1886 1887 ret = epf_ntb_init_epc_bar(ntb); 1888 if (ret) { 1889 dev_err(dev, "Failed to create NTB EPC\n"); 1890 goto err_bar_init; 1891 } 1892 1893 ret = epf_ntb_config_spad_bar_alloc_interface(ntb); 1894 if (ret) { 1895 dev_err(dev, "Failed to allocate BAR memory\n"); 1896 goto err_bar_alloc; 1897 } 1898 1899 ret = epf_ntb_epc_init(ntb); 1900 if (ret) { 1901 dev_err(dev, "Failed to initialize EPC\n"); 1902 goto err_bar_alloc; 1903 } 1904 1905 epf_set_drvdata(epf, ntb); 1906 1907 return 0; 1908 1909 err_bar_alloc: 1910 epf_ntb_config_spad_bar_free(ntb); 1911 1912 err_bar_init: 1913 epf_ntb_epc_destroy(ntb); 1914 1915 return ret; 1916 } 1917 1918 /** 1919 * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind() 1920 * @epf: NTB endpoint function device 1921 * 1922 * Cleanup the initialization from epf_ntb_bind() 1923 */ 1924 static void epf_ntb_unbind(struct pci_epf *epf) 1925 { 1926 struct epf_ntb *ntb = epf_get_drvdata(epf); 1927 1928 epf_ntb_epc_cleanup(ntb); 1929 epf_ntb_config_spad_bar_free(ntb); 1930 epf_ntb_epc_destroy(ntb); 1931 } 1932 1933 #define EPF_NTB_R(_name) \ 1934 static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ 1935 char *page) \ 1936 { \ 1937 struct config_group *group = to_config_group(item); \ 1938 struct epf_ntb *ntb = to_epf_ntb(group); \ 1939 \ 1940 return sysfs_emit(page, "%d\n", ntb->_name); \ 1941 } 1942 1943 #define EPF_NTB_W(_name) \ 1944 static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ 1945 const char *page, size_t len) \ 1946 { \ 1947 struct config_group *group = to_config_group(item); \ 1948 struct epf_ntb *ntb = to_epf_ntb(group); \ 1949 u32 val; \ 1950 \ 1951 if (kstrtou32(page, 0, &val) < 0) \ 1952 return -EINVAL; \ 1953 \ 1954 ntb->_name = val; \ 1955 \ 1956 return len; \ 1957 } 1958 1959 #define EPF_NTB_MW_R(_name) \ 1960 static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ 1961 char *page) \ 1962 { \ 1963 struct config_group *group = to_config_group(item); \ 1964 struct epf_ntb *ntb = to_epf_ntb(group); \ 1965 int win_no; \ 1966 \ 1967 sscanf(#_name, "mw%d", &win_no); \ 1968 \ 1969 return sysfs_emit(page, "%lld\n", ntb->mws_size[win_no - 1]); \ 1970 } 1971 1972 #define EPF_NTB_MW_W(_name) \ 1973 static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ 1974 const char *page, size_t len) \ 1975 { \ 1976 struct config_group *group = to_config_group(item); \ 1977 struct epf_ntb *ntb = to_epf_ntb(group); \ 1978 struct device *dev = &ntb->epf->dev; \ 1979 int win_no; \ 1980 u64 val; \ 1981 \ 1982 if (kstrtou64(page, 0, &val) < 0) \ 1983 return -EINVAL; \ 1984 \ 1985 if (sscanf(#_name, "mw%d", &win_no) != 1) \ 1986 return -EINVAL; \ 1987 \ 1988 if (ntb->num_mws < win_no) { \ 1989 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \ 1990 return -EINVAL; \ 1991 } \ 1992 \ 1993 ntb->mws_size[win_no - 1] = val; \ 1994 \ 1995 return len; \ 1996 } 1997 1998 static ssize_t epf_ntb_num_mws_store(struct config_item *item, 1999 const char *page, size_t len) 2000 { 2001 struct config_group *group = to_config_group(item); 2002 struct epf_ntb *ntb = to_epf_ntb(group); 2003 u32 val; 2004 2005 if (kstrtou32(page, 0, &val) < 0) 2006 return -EINVAL; 2007 2008 if (val > MAX_MW) 2009 return -EINVAL; 2010 2011 ntb->num_mws = val; 2012 2013 return len; 2014 } 2015 2016 EPF_NTB_R(spad_count) 2017 EPF_NTB_W(spad_count) 2018 EPF_NTB_R(db_count) 2019 EPF_NTB_W(db_count) 2020 EPF_NTB_R(num_mws) 2021 EPF_NTB_MW_R(mw1) 2022 EPF_NTB_MW_W(mw1) 2023 EPF_NTB_MW_R(mw2) 2024 EPF_NTB_MW_W(mw2) 2025 EPF_NTB_MW_R(mw3) 2026 EPF_NTB_MW_W(mw3) 2027 EPF_NTB_MW_R(mw4) 2028 EPF_NTB_MW_W(mw4) 2029 2030 CONFIGFS_ATTR(epf_ntb_, spad_count); 2031 CONFIGFS_ATTR(epf_ntb_, db_count); 2032 CONFIGFS_ATTR(epf_ntb_, num_mws); 2033 CONFIGFS_ATTR(epf_ntb_, mw1); 2034 CONFIGFS_ATTR(epf_ntb_, mw2); 2035 CONFIGFS_ATTR(epf_ntb_, mw3); 2036 CONFIGFS_ATTR(epf_ntb_, mw4); 2037 2038 static struct configfs_attribute *epf_ntb_attrs[] = { 2039 &epf_ntb_attr_spad_count, 2040 &epf_ntb_attr_db_count, 2041 &epf_ntb_attr_num_mws, 2042 &epf_ntb_attr_mw1, 2043 &epf_ntb_attr_mw2, 2044 &epf_ntb_attr_mw3, 2045 &epf_ntb_attr_mw4, 2046 NULL, 2047 }; 2048 2049 static const struct config_item_type ntb_group_type = { 2050 .ct_attrs = epf_ntb_attrs, 2051 .ct_owner = THIS_MODULE, 2052 }; 2053 2054 /** 2055 * epf_ntb_add_cfs() - Add configfs directory specific to NTB 2056 * @epf: NTB endpoint function device 2057 * @group: A pointer to the config_group structure referencing a group of 2058 * config_items of a specific type that belong to a specific sub-system. 2059 * 2060 * Add configfs directory specific to NTB. This directory will hold 2061 * NTB specific properties like db_count, spad_count, num_mws etc., 2062 */ 2063 static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf, 2064 struct config_group *group) 2065 { 2066 struct epf_ntb *ntb = epf_get_drvdata(epf); 2067 struct config_group *ntb_group = &ntb->group; 2068 struct device *dev = &epf->dev; 2069 2070 config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type); 2071 2072 return ntb_group; 2073 } 2074 2075 /** 2076 * epf_ntb_probe() - Probe NTB function driver 2077 * @epf: NTB endpoint function device 2078 * 2079 * Probe NTB function driver when endpoint function bus detects a NTB 2080 * endpoint function. 2081 */ 2082 static int epf_ntb_probe(struct pci_epf *epf) 2083 { 2084 struct epf_ntb *ntb; 2085 struct device *dev; 2086 2087 dev = &epf->dev; 2088 2089 ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL); 2090 if (!ntb) 2091 return -ENOMEM; 2092 2093 epf->header = &epf_ntb_header; 2094 ntb->epf = epf; 2095 epf_set_drvdata(epf, ntb); 2096 2097 return 0; 2098 } 2099 2100 static struct pci_epf_ops epf_ntb_ops = { 2101 .bind = epf_ntb_bind, 2102 .unbind = epf_ntb_unbind, 2103 .add_cfs = epf_ntb_add_cfs, 2104 }; 2105 2106 static const struct pci_epf_device_id epf_ntb_ids[] = { 2107 { 2108 .name = "pci_epf_ntb", 2109 }, 2110 {}, 2111 }; 2112 2113 static struct pci_epf_driver epf_ntb_driver = { 2114 .driver.name = "pci_epf_ntb", 2115 .probe = epf_ntb_probe, 2116 .id_table = epf_ntb_ids, 2117 .ops = &epf_ntb_ops, 2118 .owner = THIS_MODULE, 2119 }; 2120 2121 static int __init epf_ntb_init(void) 2122 { 2123 int ret; 2124 2125 kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM | 2126 WQ_HIGHPRI, 0); 2127 ret = pci_epf_register_driver(&epf_ntb_driver); 2128 if (ret) { 2129 destroy_workqueue(kpcintb_workqueue); 2130 pr_err("Failed to register pci epf ntb driver --> %d\n", ret); 2131 return ret; 2132 } 2133 2134 return 0; 2135 } 2136 module_init(epf_ntb_init); 2137 2138 static void __exit epf_ntb_exit(void) 2139 { 2140 pci_epf_unregister_driver(&epf_ntb_driver); 2141 destroy_workqueue(kpcintb_workqueue); 2142 } 2143 module_exit(epf_ntb_exit); 2144 2145 MODULE_DESCRIPTION("PCI EPF NTB DRIVER"); 2146 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); 2147 MODULE_LICENSE("GPL v2"); 2148