1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Microsemi Switchtec(tm) PCIe Management Driver 4 * Copyright (c) 2017, Microsemi Corporation 5 */ 6 7 #include <linux/interrupt.h> 8 #include <linux/io-64-nonatomic-lo-hi.h> 9 #include <linux/delay.h> 10 #include <linux/kthread.h> 11 #include <linux/module.h> 12 #include <linux/ntb.h> 13 #include <linux/pci.h> 14 #include <linux/switchtec.h> 15 16 MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver"); 17 MODULE_VERSION("0.1"); 18 MODULE_LICENSE("GPL"); 19 MODULE_AUTHOR("Microsemi Corporation"); 20 21 static ulong max_mw_size = SZ_2M; 22 module_param(max_mw_size, ulong, 0644); 23 MODULE_PARM_DESC(max_mw_size, 24 "Max memory window size reported to the upper layer"); 25 26 static bool use_lut_mws; 27 module_param(use_lut_mws, bool, 0644); 28 MODULE_PARM_DESC(use_lut_mws, 29 "Enable the use of the LUT based memory windows"); 30 31 #define SWITCHTEC_NTB_MAGIC 0x45CC0001 32 #define MAX_MWS 128 33 34 struct shared_mw { 35 u32 magic; 36 u32 link_sta; 37 u32 partition_id; 38 u64 mw_sizes[MAX_MWS]; 39 u32 spad[128]; 40 }; 41 42 #define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry) 43 #define LUT_SIZE SZ_64K 44 45 struct switchtec_ntb { 46 struct ntb_dev ntb; 47 struct switchtec_dev *stdev; 48 49 int self_partition; 50 int peer_partition; 51 52 int doorbell_irq; 53 int message_irq; 54 55 struct ntb_info_regs __iomem *mmio_ntb; 56 struct ntb_ctrl_regs __iomem *mmio_ctrl; 57 struct ntb_dbmsg_regs __iomem *mmio_dbmsg; 58 struct ntb_ctrl_regs __iomem *mmio_self_ctrl; 59 struct ntb_ctrl_regs __iomem *mmio_peer_ctrl; 60 struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg; 61 struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg; 62 63 void __iomem *mmio_xlink_win; 64 65 struct shared_mw *self_shared; 66 struct shared_mw __iomem *peer_shared; 67 dma_addr_t self_shared_dma; 68 69 u64 db_mask; 70 u64 db_valid_mask; 71 int db_shift; 72 int db_peer_shift; 73 74 /* synchronize rmw access of db_mask and hw reg */ 75 spinlock_t db_mask_lock; 76 77 int nr_direct_mw; 78 int nr_lut_mw; 79 int nr_rsvd_luts; 80 int direct_mw_to_bar[MAX_DIRECT_MW]; 81 82 int peer_nr_direct_mw; 83 int peer_nr_lut_mw; 84 int peer_direct_mw_to_bar[MAX_DIRECT_MW]; 85 86 bool link_is_up; 87 enum ntb_speed link_speed; 88 enum ntb_width link_width; 89 struct work_struct link_reinit_work; 90 }; 91 92 static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb) 93 { 94 return container_of(ntb, struct switchtec_ntb, ntb); 95 } 96 97 static int switchtec_ntb_part_op(struct switchtec_ntb *sndev, 98 struct ntb_ctrl_regs __iomem *ctl, 99 u32 op, int wait_status) 100 { 101 static const char * const op_text[] = { 102 [NTB_CTRL_PART_OP_LOCK] = "lock", 103 [NTB_CTRL_PART_OP_CFG] = "configure", 104 [NTB_CTRL_PART_OP_RESET] = "reset", 105 }; 106 107 int i; 108 u32 ps; 109 int status; 110 111 switch (op) { 112 case NTB_CTRL_PART_OP_LOCK: 113 status = NTB_CTRL_PART_STATUS_LOCKING; 114 break; 115 case NTB_CTRL_PART_OP_CFG: 116 status = NTB_CTRL_PART_STATUS_CONFIGURING; 117 break; 118 case NTB_CTRL_PART_OP_RESET: 119 status = NTB_CTRL_PART_STATUS_RESETTING; 120 break; 121 default: 122 return -EINVAL; 123 } 124 125 iowrite32(op, &ctl->partition_op); 126 127 for (i = 0; i < 1000; i++) { 128 if (msleep_interruptible(50) != 0) { 129 iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op); 130 return -EINTR; 131 } 132 133 ps = ioread32(&ctl->partition_status) & 0xFFFF; 134 135 if (ps != status) 136 break; 137 } 138 139 if (ps == wait_status) 140 return 0; 141 142 if (ps == status) { 143 dev_err(&sndev->stdev->dev, 144 "Timed out while performing %s (%d). (%08x)\n", 145 op_text[op], op, 146 ioread32(&ctl->partition_status)); 147 148 return -ETIMEDOUT; 149 } 150 151 return -EIO; 152 } 153 154 static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx, 155 u32 val) 156 { 157 if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg)) 158 return -EINVAL; 159 160 iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg); 161 162 return 0; 163 } 164 165 static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx) 166 { 167 struct switchtec_ntb *sndev = ntb_sndev(ntb); 168 int nr_direct_mw = sndev->peer_nr_direct_mw; 169 int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts; 170 171 if (pidx != NTB_DEF_PEER_IDX) 172 return -EINVAL; 173 174 if (!use_lut_mws) 175 nr_lut_mw = 0; 176 177 return nr_direct_mw + nr_lut_mw; 178 } 179 180 static int lut_index(struct switchtec_ntb *sndev, int mw_idx) 181 { 182 return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts; 183 } 184 185 static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx) 186 { 187 return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts; 188 } 189 190 static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, 191 int widx, resource_size_t *addr_align, 192 resource_size_t *size_align, 193 resource_size_t *size_max) 194 { 195 struct switchtec_ntb *sndev = ntb_sndev(ntb); 196 int lut; 197 resource_size_t size; 198 199 if (pidx != NTB_DEF_PEER_IDX) 200 return -EINVAL; 201 202 lut = widx >= sndev->peer_nr_direct_mw; 203 size = ioread64(&sndev->peer_shared->mw_sizes[widx]); 204 205 if (size == 0) 206 return -EINVAL; 207 208 if (addr_align) 209 *addr_align = lut ? size : SZ_4K; 210 211 if (size_align) 212 *size_align = lut ? size : SZ_4K; 213 214 if (size_max) 215 *size_max = size; 216 217 return 0; 218 } 219 220 static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx) 221 { 222 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 223 int bar = sndev->peer_direct_mw_to_bar[idx]; 224 u32 ctl_val; 225 226 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 227 ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN; 228 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 229 iowrite32(0, &ctl->bar_entry[bar].win_size); 230 iowrite32(0, &ctl->bar_ext_entry[bar].win_size); 231 iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr); 232 } 233 234 static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx) 235 { 236 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 237 238 iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]); 239 } 240 241 static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx, 242 dma_addr_t addr, resource_size_t size) 243 { 244 int xlate_pos = ilog2(size); 245 int bar = sndev->peer_direct_mw_to_bar[idx]; 246 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 247 u32 ctl_val; 248 249 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 250 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; 251 252 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 253 iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000), 254 &ctl->bar_entry[bar].win_size); 255 iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size); 256 iowrite64(sndev->self_partition | addr, 257 &ctl->bar_entry[bar].xlate_addr); 258 } 259 260 static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx, 261 dma_addr_t addr, resource_size_t size) 262 { 263 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 264 265 iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr), 266 &ctl->lut_entry[peer_lut_index(sndev, idx)]); 267 } 268 269 static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx, 270 dma_addr_t addr, resource_size_t size) 271 { 272 struct switchtec_ntb *sndev = ntb_sndev(ntb); 273 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 274 int xlate_pos = ilog2(size); 275 int nr_direct_mw = sndev->peer_nr_direct_mw; 276 int rc; 277 278 if (pidx != NTB_DEF_PEER_IDX) 279 return -EINVAL; 280 281 dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n", 282 widx, pidx, &addr, &size); 283 284 if (widx >= switchtec_ntb_mw_count(ntb, pidx)) 285 return -EINVAL; 286 287 if (xlate_pos < 12) 288 return -EINVAL; 289 290 if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) { 291 /* 292 * In certain circumstances we can get a buffer that is 293 * not aligned to its size. (Most of the time 294 * dma_alloc_coherent ensures this). This can happen when 295 * using large buffers allocated by the CMA 296 * (see CMA_CONFIG_ALIGNMENT) 297 */ 298 dev_err(&sndev->stdev->dev, 299 "ERROR: Memory window address is not aligned to it's size!\n"); 300 return -EINVAL; 301 } 302 303 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 304 NTB_CTRL_PART_STATUS_LOCKED); 305 if (rc) 306 return rc; 307 308 if (addr == 0 || size == 0) { 309 if (widx < nr_direct_mw) 310 switchtec_ntb_mw_clr_direct(sndev, widx); 311 else 312 switchtec_ntb_mw_clr_lut(sndev, widx); 313 } else { 314 if (widx < nr_direct_mw) 315 switchtec_ntb_mw_set_direct(sndev, widx, addr, size); 316 else 317 switchtec_ntb_mw_set_lut(sndev, widx, addr, size); 318 } 319 320 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 321 NTB_CTRL_PART_STATUS_NORMAL); 322 323 if (rc == -EIO) { 324 dev_err(&sndev->stdev->dev, 325 "Hardware reported an error configuring mw %d: %08x\n", 326 widx, ioread32(&ctl->bar_error)); 327 328 if (widx < nr_direct_mw) 329 switchtec_ntb_mw_clr_direct(sndev, widx); 330 else 331 switchtec_ntb_mw_clr_lut(sndev, widx); 332 333 switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 334 NTB_CTRL_PART_STATUS_NORMAL); 335 } 336 337 return rc; 338 } 339 340 static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb) 341 { 342 struct switchtec_ntb *sndev = ntb_sndev(ntb); 343 int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts; 344 345 return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0); 346 } 347 348 static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev, 349 int idx, phys_addr_t *base, 350 resource_size_t *size) 351 { 352 int bar = sndev->direct_mw_to_bar[idx]; 353 size_t offset = 0; 354 355 if (bar < 0) 356 return -EINVAL; 357 358 if (idx == 0) { 359 /* 360 * This is the direct BAR shared with the LUTs 361 * which means the actual window will be offset 362 * by the size of all the LUT entries. 363 */ 364 365 offset = LUT_SIZE * sndev->nr_lut_mw; 366 } 367 368 if (base) 369 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset; 370 371 if (size) { 372 *size = pci_resource_len(sndev->ntb.pdev, bar) - offset; 373 if (offset && *size > offset) 374 *size = offset; 375 376 if (*size > max_mw_size) 377 *size = max_mw_size; 378 } 379 380 return 0; 381 } 382 383 static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev, 384 int idx, phys_addr_t *base, 385 resource_size_t *size) 386 { 387 int bar = sndev->direct_mw_to_bar[0]; 388 int offset; 389 390 offset = LUT_SIZE * lut_index(sndev, idx); 391 392 if (base) 393 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset; 394 395 if (size) 396 *size = LUT_SIZE; 397 398 return 0; 399 } 400 401 static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, 402 phys_addr_t *base, 403 resource_size_t *size) 404 { 405 struct switchtec_ntb *sndev = ntb_sndev(ntb); 406 407 if (idx < sndev->nr_direct_mw) 408 return switchtec_ntb_direct_get_addr(sndev, idx, base, size); 409 else if (idx < switchtec_ntb_peer_mw_count(ntb)) 410 return switchtec_ntb_lut_get_addr(sndev, idx, base, size); 411 else 412 return -EINVAL; 413 } 414 415 static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev, 416 int partition, 417 enum ntb_speed *speed, 418 enum ntb_width *width) 419 { 420 struct switchtec_dev *stdev = sndev->stdev; 421 422 u32 pff = ioread32(&stdev->mmio_part_cfg[partition].vep_pff_inst_id); 423 u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]); 424 425 if (speed) 426 *speed = (linksta >> 16) & 0xF; 427 428 if (width) 429 *width = (linksta >> 20) & 0x3F; 430 } 431 432 static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev) 433 { 434 enum ntb_speed self_speed, peer_speed; 435 enum ntb_width self_width, peer_width; 436 437 if (!sndev->link_is_up) { 438 sndev->link_speed = NTB_SPEED_NONE; 439 sndev->link_width = NTB_WIDTH_NONE; 440 return; 441 } 442 443 switchtec_ntb_part_link_speed(sndev, sndev->self_partition, 444 &self_speed, &self_width); 445 switchtec_ntb_part_link_speed(sndev, sndev->peer_partition, 446 &peer_speed, &peer_width); 447 448 sndev->link_speed = min(self_speed, peer_speed); 449 sndev->link_width = min(self_width, peer_width); 450 } 451 452 static int crosslink_is_enabled(struct switchtec_ntb *sndev) 453 { 454 struct ntb_info_regs __iomem *inf = sndev->mmio_ntb; 455 456 return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled); 457 } 458 459 static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev) 460 { 461 int i; 462 u32 msg_map = 0; 463 464 if (!crosslink_is_enabled(sndev)) 465 return; 466 467 for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) { 468 int m = i | sndev->self_partition << 2; 469 470 msg_map |= m << i * 8; 471 } 472 473 iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map); 474 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, 475 &sndev->mmio_peer_dbmsg->odb_mask); 476 } 477 478 enum switchtec_msg { 479 LINK_MESSAGE = 0, 480 MSG_LINK_UP = 1, 481 MSG_LINK_DOWN = 2, 482 MSG_CHECK_LINK = 3, 483 MSG_LINK_FORCE_DOWN = 4, 484 }; 485 486 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev); 487 488 static void link_reinit_work(struct work_struct *work) 489 { 490 struct switchtec_ntb *sndev; 491 492 sndev = container_of(work, struct switchtec_ntb, link_reinit_work); 493 494 switchtec_ntb_reinit_peer(sndev); 495 } 496 497 static void switchtec_ntb_check_link(struct switchtec_ntb *sndev, 498 enum switchtec_msg msg) 499 { 500 int link_sta; 501 int old = sndev->link_is_up; 502 503 if (msg == MSG_LINK_FORCE_DOWN) { 504 schedule_work(&sndev->link_reinit_work); 505 506 if (sndev->link_is_up) { 507 sndev->link_is_up = 0; 508 ntb_link_event(&sndev->ntb); 509 dev_info(&sndev->stdev->dev, "ntb link forced down\n"); 510 } 511 512 return; 513 } 514 515 link_sta = sndev->self_shared->link_sta; 516 if (link_sta) { 517 u64 peer = ioread64(&sndev->peer_shared->magic); 518 519 if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC) 520 link_sta = peer >> 32; 521 else 522 link_sta = 0; 523 } 524 525 sndev->link_is_up = link_sta; 526 switchtec_ntb_set_link_speed(sndev); 527 528 if (link_sta != old) { 529 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK); 530 ntb_link_event(&sndev->ntb); 531 dev_info(&sndev->stdev->dev, "ntb link %s\n", 532 link_sta ? "up" : "down"); 533 534 if (link_sta) 535 crosslink_init_dbmsgs(sndev); 536 } 537 } 538 539 static void switchtec_ntb_link_notification(struct switchtec_dev *stdev) 540 { 541 struct switchtec_ntb *sndev = stdev->sndev; 542 543 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); 544 } 545 546 static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb, 547 enum ntb_speed *speed, 548 enum ntb_width *width) 549 { 550 struct switchtec_ntb *sndev = ntb_sndev(ntb); 551 552 if (speed) 553 *speed = sndev->link_speed; 554 if (width) 555 *width = sndev->link_width; 556 557 return sndev->link_is_up; 558 } 559 560 static int switchtec_ntb_link_enable(struct ntb_dev *ntb, 561 enum ntb_speed max_speed, 562 enum ntb_width max_width) 563 { 564 struct switchtec_ntb *sndev = ntb_sndev(ntb); 565 566 dev_dbg(&sndev->stdev->dev, "enabling link\n"); 567 568 sndev->self_shared->link_sta = 1; 569 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP); 570 571 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); 572 573 return 0; 574 } 575 576 static int switchtec_ntb_link_disable(struct ntb_dev *ntb) 577 { 578 struct switchtec_ntb *sndev = ntb_sndev(ntb); 579 580 dev_dbg(&sndev->stdev->dev, "disabling link\n"); 581 582 sndev->self_shared->link_sta = 0; 583 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN); 584 585 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); 586 587 return 0; 588 } 589 590 static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb) 591 { 592 struct switchtec_ntb *sndev = ntb_sndev(ntb); 593 594 return sndev->db_valid_mask; 595 } 596 597 static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb) 598 { 599 return 1; 600 } 601 602 static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 603 { 604 struct switchtec_ntb *sndev = ntb_sndev(ntb); 605 606 if (db_vector < 0 || db_vector > 1) 607 return 0; 608 609 return sndev->db_valid_mask; 610 } 611 612 static u64 switchtec_ntb_db_read(struct ntb_dev *ntb) 613 { 614 u64 ret; 615 struct switchtec_ntb *sndev = ntb_sndev(ntb); 616 617 ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift; 618 619 return ret & sndev->db_valid_mask; 620 } 621 622 static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 623 { 624 struct switchtec_ntb *sndev = ntb_sndev(ntb); 625 626 iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb); 627 628 return 0; 629 } 630 631 static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 632 { 633 unsigned long irqflags; 634 struct switchtec_ntb *sndev = ntb_sndev(ntb); 635 636 if (db_bits & ~sndev->db_valid_mask) 637 return -EINVAL; 638 639 spin_lock_irqsave(&sndev->db_mask_lock, irqflags); 640 641 sndev->db_mask |= db_bits << sndev->db_shift; 642 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 643 644 spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags); 645 646 return 0; 647 } 648 649 static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 650 { 651 unsigned long irqflags; 652 struct switchtec_ntb *sndev = ntb_sndev(ntb); 653 654 if (db_bits & ~sndev->db_valid_mask) 655 return -EINVAL; 656 657 spin_lock_irqsave(&sndev->db_mask_lock, irqflags); 658 659 sndev->db_mask &= ~(db_bits << sndev->db_shift); 660 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 661 662 spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags); 663 664 return 0; 665 } 666 667 static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb) 668 { 669 struct switchtec_ntb *sndev = ntb_sndev(ntb); 670 671 return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask; 672 } 673 674 static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb, 675 phys_addr_t *db_addr, 676 resource_size_t *db_size, 677 u64 *db_data, 678 int db_bit) 679 { 680 struct switchtec_ntb *sndev = ntb_sndev(ntb); 681 unsigned long offset; 682 683 if (unlikely(db_bit >= BITS_PER_LONG_LONG)) 684 return -EINVAL; 685 686 offset = (unsigned long)sndev->mmio_peer_dbmsg->odb - 687 (unsigned long)sndev->stdev->mmio; 688 689 offset += sndev->db_shift / 8; 690 691 if (db_addr) 692 *db_addr = pci_resource_start(ntb->pdev, 0) + offset; 693 if (db_size) 694 *db_size = sizeof(u32); 695 if (db_data) 696 *db_data = BIT_ULL(db_bit) << sndev->db_peer_shift; 697 698 return 0; 699 } 700 701 static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 702 { 703 struct switchtec_ntb *sndev = ntb_sndev(ntb); 704 705 iowrite64(db_bits << sndev->db_peer_shift, 706 &sndev->mmio_peer_dbmsg->odb); 707 708 return 0; 709 } 710 711 static int switchtec_ntb_spad_count(struct ntb_dev *ntb) 712 { 713 struct switchtec_ntb *sndev = ntb_sndev(ntb); 714 715 return ARRAY_SIZE(sndev->self_shared->spad); 716 } 717 718 static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx) 719 { 720 struct switchtec_ntb *sndev = ntb_sndev(ntb); 721 722 if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad)) 723 return 0; 724 725 if (!sndev->self_shared) 726 return 0; 727 728 return sndev->self_shared->spad[idx]; 729 } 730 731 static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val) 732 { 733 struct switchtec_ntb *sndev = ntb_sndev(ntb); 734 735 if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad)) 736 return -EINVAL; 737 738 if (!sndev->self_shared) 739 return -EIO; 740 741 sndev->self_shared->spad[idx] = val; 742 743 return 0; 744 } 745 746 static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, 747 int sidx) 748 { 749 struct switchtec_ntb *sndev = ntb_sndev(ntb); 750 751 if (pidx != NTB_DEF_PEER_IDX) 752 return -EINVAL; 753 754 if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad)) 755 return 0; 756 757 if (!sndev->peer_shared) 758 return 0; 759 760 return ioread32(&sndev->peer_shared->spad[sidx]); 761 } 762 763 static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, 764 int sidx, u32 val) 765 { 766 struct switchtec_ntb *sndev = ntb_sndev(ntb); 767 768 if (pidx != NTB_DEF_PEER_IDX) 769 return -EINVAL; 770 771 if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad)) 772 return -EINVAL; 773 774 if (!sndev->peer_shared) 775 return -EIO; 776 777 iowrite32(val, &sndev->peer_shared->spad[sidx]); 778 779 return 0; 780 } 781 782 static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, 783 int sidx, phys_addr_t *spad_addr) 784 { 785 struct switchtec_ntb *sndev = ntb_sndev(ntb); 786 unsigned long offset; 787 788 if (pidx != NTB_DEF_PEER_IDX) 789 return -EINVAL; 790 791 offset = (unsigned long)&sndev->peer_shared->spad[sidx] - 792 (unsigned long)sndev->stdev->mmio; 793 794 if (spad_addr) 795 *spad_addr = pci_resource_start(ntb->pdev, 0) + offset; 796 797 return 0; 798 } 799 800 static const struct ntb_dev_ops switchtec_ntb_ops = { 801 .mw_count = switchtec_ntb_mw_count, 802 .mw_get_align = switchtec_ntb_mw_get_align, 803 .mw_set_trans = switchtec_ntb_mw_set_trans, 804 .peer_mw_count = switchtec_ntb_peer_mw_count, 805 .peer_mw_get_addr = switchtec_ntb_peer_mw_get_addr, 806 .link_is_up = switchtec_ntb_link_is_up, 807 .link_enable = switchtec_ntb_link_enable, 808 .link_disable = switchtec_ntb_link_disable, 809 .db_valid_mask = switchtec_ntb_db_valid_mask, 810 .db_vector_count = switchtec_ntb_db_vector_count, 811 .db_vector_mask = switchtec_ntb_db_vector_mask, 812 .db_read = switchtec_ntb_db_read, 813 .db_clear = switchtec_ntb_db_clear, 814 .db_set_mask = switchtec_ntb_db_set_mask, 815 .db_clear_mask = switchtec_ntb_db_clear_mask, 816 .db_read_mask = switchtec_ntb_db_read_mask, 817 .peer_db_addr = switchtec_ntb_peer_db_addr, 818 .peer_db_set = switchtec_ntb_peer_db_set, 819 .spad_count = switchtec_ntb_spad_count, 820 .spad_read = switchtec_ntb_spad_read, 821 .spad_write = switchtec_ntb_spad_write, 822 .peer_spad_read = switchtec_ntb_peer_spad_read, 823 .peer_spad_write = switchtec_ntb_peer_spad_write, 824 .peer_spad_addr = switchtec_ntb_peer_spad_addr, 825 }; 826 827 static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev) 828 { 829 u64 tpart_vec; 830 int self; 831 u64 part_map; 832 int bit; 833 834 sndev->ntb.pdev = sndev->stdev->pdev; 835 sndev->ntb.topo = NTB_TOPO_SWITCH; 836 sndev->ntb.ops = &switchtec_ntb_ops; 837 838 INIT_WORK(&sndev->link_reinit_work, link_reinit_work); 839 840 sndev->self_partition = sndev->stdev->partition; 841 842 sndev->mmio_ntb = sndev->stdev->mmio_ntb; 843 844 self = sndev->self_partition; 845 tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high); 846 tpart_vec <<= 32; 847 tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low); 848 849 part_map = ioread64(&sndev->mmio_ntb->ep_map); 850 part_map &= ~(1 << sndev->self_partition); 851 852 if (!ffs(tpart_vec)) { 853 if (sndev->stdev->partition_count != 2) { 854 dev_err(&sndev->stdev->dev, 855 "ntb target partition not defined\n"); 856 return -ENODEV; 857 } 858 859 bit = ffs(part_map); 860 if (!bit) { 861 dev_err(&sndev->stdev->dev, 862 "peer partition is not NT partition\n"); 863 return -ENODEV; 864 } 865 866 sndev->peer_partition = bit - 1; 867 } else { 868 if (ffs(tpart_vec) != fls(tpart_vec)) { 869 dev_err(&sndev->stdev->dev, 870 "ntb driver only supports 1 pair of 1-1 ntb mapping\n"); 871 return -ENODEV; 872 } 873 874 sndev->peer_partition = ffs(tpart_vec) - 1; 875 if (!(part_map & (1 << sndev->peer_partition))) { 876 dev_err(&sndev->stdev->dev, 877 "ntb target partition is not NT partition\n"); 878 return -ENODEV; 879 } 880 } 881 882 dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n", 883 sndev->self_partition, sndev->stdev->partition_count); 884 885 sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb + 886 SWITCHTEC_NTB_REG_CTRL_OFFSET; 887 sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb + 888 SWITCHTEC_NTB_REG_DBMSG_OFFSET; 889 890 sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition]; 891 sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition]; 892 sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition]; 893 sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg; 894 895 return 0; 896 } 897 898 static int config_rsvd_lut_win(struct switchtec_ntb *sndev, 899 struct ntb_ctrl_regs __iomem *ctl, 900 int lut_idx, int partition, u64 addr) 901 { 902 int peer_bar = sndev->peer_direct_mw_to_bar[0]; 903 u32 ctl_val; 904 int rc; 905 906 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 907 NTB_CTRL_PART_STATUS_LOCKED); 908 if (rc) 909 return rc; 910 911 ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl); 912 ctl_val &= 0xFF; 913 ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN; 914 ctl_val |= ilog2(LUT_SIZE) << 8; 915 ctl_val |= (sndev->nr_lut_mw - 1) << 14; 916 iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl); 917 918 iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr), 919 &ctl->lut_entry[lut_idx]); 920 921 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 922 NTB_CTRL_PART_STATUS_NORMAL); 923 if (rc) { 924 u32 bar_error, lut_error; 925 926 bar_error = ioread32(&ctl->bar_error); 927 lut_error = ioread32(&ctl->lut_error); 928 dev_err(&sndev->stdev->dev, 929 "Error setting up reserved lut window: %08x / %08x\n", 930 bar_error, lut_error); 931 return rc; 932 } 933 934 return 0; 935 } 936 937 static int config_req_id_table(struct switchtec_ntb *sndev, 938 struct ntb_ctrl_regs __iomem *mmio_ctrl, 939 int *req_ids, int count) 940 { 941 int i, rc = 0; 942 u32 error; 943 u32 proxy_id; 944 945 if (ioread32(&mmio_ctrl->req_id_table_size) < count) { 946 dev_err(&sndev->stdev->dev, 947 "Not enough requester IDs available.\n"); 948 return -EFAULT; 949 } 950 951 rc = switchtec_ntb_part_op(sndev, mmio_ctrl, 952 NTB_CTRL_PART_OP_LOCK, 953 NTB_CTRL_PART_STATUS_LOCKED); 954 if (rc) 955 return rc; 956 957 iowrite32(NTB_PART_CTRL_ID_PROT_DIS, 958 &mmio_ctrl->partition_ctrl); 959 960 for (i = 0; i < count; i++) { 961 iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN, 962 &mmio_ctrl->req_id_table[i]); 963 964 proxy_id = ioread32(&mmio_ctrl->req_id_table[i]); 965 dev_dbg(&sndev->stdev->dev, 966 "Requester ID %02X:%02X.%X -> BB:%02X.%X\n", 967 req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F, 968 req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F, 969 (proxy_id >> 1) & 0x7); 970 } 971 972 rc = switchtec_ntb_part_op(sndev, mmio_ctrl, 973 NTB_CTRL_PART_OP_CFG, 974 NTB_CTRL_PART_STATUS_NORMAL); 975 976 if (rc == -EIO) { 977 error = ioread32(&mmio_ctrl->req_id_error); 978 dev_err(&sndev->stdev->dev, 979 "Error setting up the requester ID table: %08x\n", 980 error); 981 } 982 983 return 0; 984 } 985 986 static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx, 987 u64 *mw_addrs, int mw_count) 988 { 989 int rc, i; 990 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl; 991 u64 addr; 992 size_t size, offset; 993 int bar; 994 int xlate_pos; 995 u32 ctl_val; 996 997 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 998 NTB_CTRL_PART_STATUS_LOCKED); 999 if (rc) 1000 return rc; 1001 1002 for (i = 0; i < sndev->nr_lut_mw; i++) { 1003 if (i == ntb_lut_idx) 1004 continue; 1005 1006 addr = mw_addrs[0] + LUT_SIZE * i; 1007 1008 iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) | 1009 addr), 1010 &ctl->lut_entry[i]); 1011 } 1012 1013 sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count); 1014 1015 for (i = 0; i < sndev->nr_direct_mw; i++) { 1016 bar = sndev->direct_mw_to_bar[i]; 1017 offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0; 1018 addr = mw_addrs[i] + offset; 1019 size = pci_resource_len(sndev->ntb.pdev, bar) - offset; 1020 xlate_pos = ilog2(size); 1021 1022 if (offset && size > offset) 1023 size = offset; 1024 1025 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 1026 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; 1027 1028 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 1029 iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000), 1030 &ctl->bar_entry[bar].win_size); 1031 iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size); 1032 iowrite64(sndev->peer_partition | addr, 1033 &ctl->bar_entry[bar].xlate_addr); 1034 } 1035 1036 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 1037 NTB_CTRL_PART_STATUS_NORMAL); 1038 if (rc) { 1039 u32 bar_error, lut_error; 1040 1041 bar_error = ioread32(&ctl->bar_error); 1042 lut_error = ioread32(&ctl->lut_error); 1043 dev_err(&sndev->stdev->dev, 1044 "Error setting up cross link windows: %08x / %08x\n", 1045 bar_error, lut_error); 1046 return rc; 1047 } 1048 1049 return 0; 1050 } 1051 1052 static int crosslink_setup_req_ids(struct switchtec_ntb *sndev, 1053 struct ntb_ctrl_regs __iomem *mmio_ctrl) 1054 { 1055 int req_ids[16]; 1056 int i; 1057 u32 proxy_id; 1058 1059 for (i = 0; i < ARRAY_SIZE(req_ids); i++) { 1060 proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]); 1061 1062 if (!(proxy_id & NTB_CTRL_REQ_ID_EN)) 1063 break; 1064 1065 req_ids[i] = ((proxy_id >> 1) & 0xFF); 1066 } 1067 1068 return config_req_id_table(sndev, mmio_ctrl, req_ids, i); 1069 } 1070 1071 /* 1072 * In crosslink configuration there is a virtual partition in the 1073 * middle of the two switches. The BARs in this partition have to be 1074 * enumerated and assigned addresses. 1075 */ 1076 static int crosslink_enum_partition(struct switchtec_ntb *sndev, 1077 u64 *bar_addrs) 1078 { 1079 struct part_cfg_regs __iomem *part_cfg = 1080 &sndev->stdev->mmio_part_cfg_all[sndev->peer_partition]; 1081 u32 pff = ioread32(&part_cfg->vep_pff_inst_id); 1082 struct pff_csr_regs __iomem *mmio_pff = 1083 &sndev->stdev->mmio_pff_csr[pff]; 1084 const u64 bar_space = 0x1000000000LL; 1085 u64 bar_addr; 1086 int bar_cnt = 0; 1087 int i; 1088 1089 iowrite16(0x6, &mmio_pff->pcicmd); 1090 1091 for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) { 1092 iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]); 1093 bar_addr = ioread64(&mmio_pff->pci_bar64[i]); 1094 bar_addr &= ~0xf; 1095 1096 dev_dbg(&sndev->stdev->dev, 1097 "Crosslink BAR%d addr: %llx\n", 1098 i*2, bar_addr); 1099 1100 if (bar_addr != bar_space * i) 1101 continue; 1102 1103 bar_addrs[bar_cnt++] = bar_addr; 1104 } 1105 1106 return bar_cnt; 1107 } 1108 1109 static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev) 1110 { 1111 int rc; 1112 int bar = sndev->direct_mw_to_bar[0]; 1113 const int ntb_lut_idx = 1; 1114 u64 bar_addrs[6]; 1115 u64 addr; 1116 int offset; 1117 int bar_cnt; 1118 1119 if (!crosslink_is_enabled(sndev)) 1120 return 0; 1121 1122 dev_info(&sndev->stdev->dev, "Using crosslink configuration\n"); 1123 sndev->ntb.topo = NTB_TOPO_CROSSLINK; 1124 1125 bar_cnt = crosslink_enum_partition(sndev, bar_addrs); 1126 if (bar_cnt < sndev->nr_direct_mw + 1) { 1127 dev_err(&sndev->stdev->dev, 1128 "Error enumerating crosslink partition\n"); 1129 return -EINVAL; 1130 } 1131 1132 addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET + 1133 SWITCHTEC_NTB_REG_DBMSG_OFFSET + 1134 sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition); 1135 1136 offset = addr & (LUT_SIZE - 1); 1137 addr -= offset; 1138 1139 rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx, 1140 sndev->peer_partition, addr); 1141 if (rc) 1142 return rc; 1143 1144 rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1], 1145 bar_cnt - 1); 1146 if (rc) 1147 return rc; 1148 1149 rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl); 1150 if (rc) 1151 return rc; 1152 1153 sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar, 1154 LUT_SIZE, LUT_SIZE); 1155 if (!sndev->mmio_xlink_win) { 1156 rc = -ENOMEM; 1157 return rc; 1158 } 1159 1160 sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset; 1161 sndev->nr_rsvd_luts++; 1162 1163 crosslink_init_dbmsgs(sndev); 1164 1165 return 0; 1166 } 1167 1168 static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev) 1169 { 1170 if (sndev->mmio_xlink_win) 1171 pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win); 1172 } 1173 1174 static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl) 1175 { 1176 int i; 1177 int cnt = 0; 1178 1179 for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) { 1180 u32 r = ioread32(&ctrl->bar_entry[i].ctl); 1181 1182 if (r & NTB_CTRL_BAR_VALID) 1183 map[cnt++] = i; 1184 } 1185 1186 return cnt; 1187 } 1188 1189 static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) 1190 { 1191 sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar, 1192 sndev->mmio_self_ctrl); 1193 1194 sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries); 1195 sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); 1196 1197 dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n", 1198 sndev->nr_direct_mw, sndev->nr_lut_mw); 1199 1200 sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar, 1201 sndev->mmio_peer_ctrl); 1202 1203 sndev->peer_nr_lut_mw = 1204 ioread16(&sndev->mmio_peer_ctrl->lut_table_entries); 1205 sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); 1206 1207 dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n", 1208 sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw); 1209 1210 } 1211 1212 /* 1213 * There are 64 doorbells in the switch hardware but this is 1214 * shared among all partitions. So we must split them in half 1215 * (32 for each partition). However, the message interrupts are 1216 * also shared with the top 4 doorbells so we just limit this to 1217 * 28 doorbells per partition. 1218 * 1219 * In crosslink mode, each side has it's own dbmsg register so 1220 * they can each use all 60 of the available doorbells. 1221 */ 1222 static void switchtec_ntb_init_db(struct switchtec_ntb *sndev) 1223 { 1224 sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL; 1225 1226 if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) { 1227 sndev->db_shift = 0; 1228 sndev->db_peer_shift = 0; 1229 sndev->db_valid_mask = sndev->db_mask; 1230 } else if (sndev->self_partition < sndev->peer_partition) { 1231 sndev->db_shift = 0; 1232 sndev->db_peer_shift = 32; 1233 sndev->db_valid_mask = 0x0FFFFFFF; 1234 } else { 1235 sndev->db_shift = 32; 1236 sndev->db_peer_shift = 0; 1237 sndev->db_valid_mask = 0x0FFFFFFF; 1238 } 1239 1240 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 1241 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, 1242 &sndev->mmio_peer_dbmsg->odb_mask); 1243 1244 dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n", 1245 sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask); 1246 } 1247 1248 static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev) 1249 { 1250 int i; 1251 u32 msg_map = 0; 1252 1253 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { 1254 int m = i | sndev->peer_partition << 2; 1255 1256 msg_map |= m << i * 8; 1257 } 1258 1259 iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map); 1260 1261 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) 1262 iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK, 1263 &sndev->mmio_self_dbmsg->imsg[i]); 1264 } 1265 1266 static int 1267 switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev) 1268 { 1269 int req_ids[2]; 1270 1271 /* 1272 * Root Complex Requester ID (which is 0:00.0) 1273 */ 1274 req_ids[0] = 0; 1275 1276 /* 1277 * Host Bridge Requester ID (as read from the mmap address) 1278 */ 1279 req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id); 1280 1281 return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids, 1282 ARRAY_SIZE(req_ids)); 1283 } 1284 1285 static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev) 1286 { 1287 int i; 1288 1289 memset(sndev->self_shared, 0, LUT_SIZE); 1290 sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC; 1291 sndev->self_shared->partition_id = sndev->stdev->partition; 1292 1293 for (i = 0; i < sndev->nr_direct_mw; i++) { 1294 int bar = sndev->direct_mw_to_bar[i]; 1295 resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar); 1296 1297 if (i == 0) 1298 sz = min_t(resource_size_t, sz, 1299 LUT_SIZE * sndev->nr_lut_mw); 1300 1301 sndev->self_shared->mw_sizes[i] = sz; 1302 } 1303 1304 for (i = 0; i < sndev->nr_lut_mw; i++) { 1305 int idx = sndev->nr_direct_mw + i; 1306 1307 sndev->self_shared->mw_sizes[idx] = LUT_SIZE; 1308 } 1309 } 1310 1311 static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev) 1312 { 1313 int self_bar = sndev->direct_mw_to_bar[0]; 1314 int rc; 1315 1316 sndev->nr_rsvd_luts++; 1317 sndev->self_shared = dma_alloc_coherent(&sndev->stdev->pdev->dev, 1318 LUT_SIZE, 1319 &sndev->self_shared_dma, 1320 GFP_KERNEL); 1321 if (!sndev->self_shared) { 1322 dev_err(&sndev->stdev->dev, 1323 "unable to allocate memory for shared mw\n"); 1324 return -ENOMEM; 1325 } 1326 1327 switchtec_ntb_init_shared(sndev); 1328 1329 rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0, 1330 sndev->self_partition, 1331 sndev->self_shared_dma); 1332 if (rc) 1333 goto unalloc_and_exit; 1334 1335 sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE); 1336 if (!sndev->peer_shared) { 1337 rc = -ENOMEM; 1338 goto unalloc_and_exit; 1339 } 1340 1341 dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n"); 1342 return 0; 1343 1344 unalloc_and_exit: 1345 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 1346 sndev->self_shared, sndev->self_shared_dma); 1347 1348 return rc; 1349 } 1350 1351 static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev) 1352 { 1353 if (sndev->peer_shared) 1354 pci_iounmap(sndev->stdev->pdev, sndev->peer_shared); 1355 1356 if (sndev->self_shared) 1357 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 1358 sndev->self_shared, 1359 sndev->self_shared_dma); 1360 sndev->nr_rsvd_luts--; 1361 } 1362 1363 static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev) 1364 { 1365 struct switchtec_ntb *sndev = dev; 1366 1367 dev_dbg(&sndev->stdev->dev, "doorbell\n"); 1368 1369 ntb_db_event(&sndev->ntb, 0); 1370 1371 return IRQ_HANDLED; 1372 } 1373 1374 static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev) 1375 { 1376 int i; 1377 struct switchtec_ntb *sndev = dev; 1378 1379 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { 1380 u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]); 1381 1382 if (msg & NTB_DBMSG_IMSG_STATUS) { 1383 dev_dbg(&sndev->stdev->dev, "message: %d %08x\n", 1384 i, (u32)msg); 1385 iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status); 1386 1387 if (i == LINK_MESSAGE) 1388 switchtec_ntb_check_link(sndev, msg); 1389 } 1390 } 1391 1392 return IRQ_HANDLED; 1393 } 1394 1395 static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev) 1396 { 1397 int i; 1398 int rc; 1399 int doorbell_irq = 0; 1400 int message_irq = 0; 1401 int event_irq; 1402 int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map); 1403 1404 event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number); 1405 1406 while (doorbell_irq == event_irq) 1407 doorbell_irq++; 1408 while (message_irq == doorbell_irq || 1409 message_irq == event_irq) 1410 message_irq++; 1411 1412 dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n", 1413 event_irq, doorbell_irq, message_irq); 1414 1415 for (i = 0; i < idb_vecs - 4; i++) 1416 iowrite8(doorbell_irq, 1417 &sndev->mmio_self_dbmsg->idb_vec_map[i]); 1418 1419 for (; i < idb_vecs; i++) 1420 iowrite8(message_irq, 1421 &sndev->mmio_self_dbmsg->idb_vec_map[i]); 1422 1423 sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq); 1424 sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq); 1425 1426 rc = request_irq(sndev->doorbell_irq, 1427 switchtec_ntb_doorbell_isr, 0, 1428 "switchtec_ntb_doorbell", sndev); 1429 if (rc) 1430 return rc; 1431 1432 rc = request_irq(sndev->message_irq, 1433 switchtec_ntb_message_isr, 0, 1434 "switchtec_ntb_message", sndev); 1435 if (rc) { 1436 free_irq(sndev->doorbell_irq, sndev); 1437 return rc; 1438 } 1439 1440 return 0; 1441 } 1442 1443 static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev) 1444 { 1445 free_irq(sndev->doorbell_irq, sndev); 1446 free_irq(sndev->message_irq, sndev); 1447 } 1448 1449 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev) 1450 { 1451 dev_info(&sndev->stdev->dev, "peer reinitialized\n"); 1452 switchtec_ntb_deinit_shared_mw(sndev); 1453 switchtec_ntb_init_mw(sndev); 1454 return switchtec_ntb_init_shared_mw(sndev); 1455 } 1456 1457 static int switchtec_ntb_add(struct device *dev, 1458 struct class_interface *class_intf) 1459 { 1460 struct switchtec_dev *stdev = to_stdev(dev); 1461 struct switchtec_ntb *sndev; 1462 int rc; 1463 1464 stdev->sndev = NULL; 1465 1466 if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8)) 1467 return -ENODEV; 1468 1469 sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev)); 1470 if (!sndev) 1471 return -ENOMEM; 1472 1473 sndev->stdev = stdev; 1474 rc = switchtec_ntb_init_sndev(sndev); 1475 if (rc) 1476 goto free_and_exit; 1477 1478 switchtec_ntb_init_mw(sndev); 1479 1480 rc = switchtec_ntb_init_req_id_table(sndev); 1481 if (rc) 1482 goto free_and_exit; 1483 1484 rc = switchtec_ntb_init_crosslink(sndev); 1485 if (rc) 1486 goto free_and_exit; 1487 1488 switchtec_ntb_init_db(sndev); 1489 switchtec_ntb_init_msgs(sndev); 1490 1491 rc = switchtec_ntb_init_shared_mw(sndev); 1492 if (rc) 1493 goto deinit_crosslink; 1494 1495 rc = switchtec_ntb_init_db_msg_irq(sndev); 1496 if (rc) 1497 goto deinit_shared_and_exit; 1498 1499 /* 1500 * If this host crashed, the other host may think the link is 1501 * still up. Tell them to force it down (it will go back up 1502 * once we register the ntb device). 1503 */ 1504 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN); 1505 1506 rc = ntb_register_device(&sndev->ntb); 1507 if (rc) 1508 goto deinit_and_exit; 1509 1510 stdev->sndev = sndev; 1511 stdev->link_notifier = switchtec_ntb_link_notification; 1512 dev_info(dev, "NTB device registered\n"); 1513 1514 return 0; 1515 1516 deinit_and_exit: 1517 switchtec_ntb_deinit_db_msg_irq(sndev); 1518 deinit_shared_and_exit: 1519 switchtec_ntb_deinit_shared_mw(sndev); 1520 deinit_crosslink: 1521 switchtec_ntb_deinit_crosslink(sndev); 1522 free_and_exit: 1523 kfree(sndev); 1524 dev_err(dev, "failed to register ntb device: %d\n", rc); 1525 return rc; 1526 } 1527 1528 static void switchtec_ntb_remove(struct device *dev, 1529 struct class_interface *class_intf) 1530 { 1531 struct switchtec_dev *stdev = to_stdev(dev); 1532 struct switchtec_ntb *sndev = stdev->sndev; 1533 1534 if (!sndev) 1535 return; 1536 1537 stdev->link_notifier = NULL; 1538 stdev->sndev = NULL; 1539 ntb_unregister_device(&sndev->ntb); 1540 switchtec_ntb_deinit_db_msg_irq(sndev); 1541 switchtec_ntb_deinit_shared_mw(sndev); 1542 switchtec_ntb_deinit_crosslink(sndev); 1543 kfree(sndev); 1544 dev_info(dev, "ntb device unregistered\n"); 1545 } 1546 1547 static struct class_interface switchtec_interface = { 1548 .add_dev = switchtec_ntb_add, 1549 .remove_dev = switchtec_ntb_remove, 1550 }; 1551 1552 static int __init switchtec_ntb_init(void) 1553 { 1554 switchtec_interface.class = switchtec_class; 1555 return class_interface_register(&switchtec_interface); 1556 } 1557 module_init(switchtec_ntb_init); 1558 1559 static void __exit switchtec_ntb_exit(void) 1560 { 1561 class_interface_unregister(&switchtec_interface); 1562 } 1563 module_exit(switchtec_ntb_exit); 1564