1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/platform_device.h> 35 #include <linux/vmalloc.h> 36 #include <rdma/ib_umem.h> 37 #include "hns_roce_device.h" 38 #include "hns_roce_cmd.h" 39 #include "hns_roce_hem.h" 40 41 static u32 hw_index_to_key(unsigned long ind) 42 { 43 return (u32)(ind >> 24) | (ind << 8); 44 } 45 46 unsigned long key_to_hw_index(u32 key) 47 { 48 return (key << 24) | (key >> 8); 49 } 50 EXPORT_SYMBOL_GPL(key_to_hw_index); 51 52 static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev, 53 struct hns_roce_cmd_mailbox *mailbox, 54 unsigned long mpt_index) 55 { 56 return hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, mpt_index, 0, 57 HNS_ROCE_CMD_SW2HW_MPT, 58 HNS_ROCE_CMD_TIMEOUT_MSECS); 59 } 60 61 int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev, 62 struct hns_roce_cmd_mailbox *mailbox, 63 unsigned long mpt_index) 64 { 65 return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox->dma : 0, 66 mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT, 67 HNS_ROCE_CMD_TIMEOUT_MSECS); 68 } 69 EXPORT_SYMBOL_GPL(hns_roce_hw2sw_mpt); 70 71 static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order, 72 unsigned long *seg) 73 { 74 int o; 75 u32 m; 76 77 spin_lock(&buddy->lock); 78 79 for (o = order; o <= buddy->max_order; ++o) { 80 if (buddy->num_free[o]) { 81 m = 1 << (buddy->max_order - o); 82 *seg = find_first_bit(buddy->bits[o], m); 83 if (*seg < m) 84 goto found; 85 } 86 } 87 spin_unlock(&buddy->lock); 88 return -1; 89 90 found: 91 clear_bit(*seg, buddy->bits[o]); 92 --buddy->num_free[o]; 93 94 while (o > order) { 95 --o; 96 *seg <<= 1; 97 set_bit(*seg ^ 1, buddy->bits[o]); 98 ++buddy->num_free[o]; 99 } 100 101 spin_unlock(&buddy->lock); 102 103 *seg <<= order; 104 return 0; 105 } 106 107 static void hns_roce_buddy_free(struct hns_roce_buddy *buddy, unsigned long seg, 108 int order) 109 { 110 seg >>= order; 111 112 spin_lock(&buddy->lock); 113 114 while (test_bit(seg ^ 1, buddy->bits[order])) { 115 clear_bit(seg ^ 1, buddy->bits[order]); 116 --buddy->num_free[order]; 117 seg >>= 1; 118 ++order; 119 } 120 121 set_bit(seg, buddy->bits[order]); 122 ++buddy->num_free[order]; 123 124 spin_unlock(&buddy->lock); 125 } 126 127 static int hns_roce_buddy_init(struct hns_roce_buddy *buddy, int max_order) 128 { 129 int i, s; 130 131 buddy->max_order = max_order; 132 spin_lock_init(&buddy->lock); 133 buddy->bits = kcalloc(buddy->max_order + 1, 134 sizeof(*buddy->bits), 135 GFP_KERNEL); 136 buddy->num_free = kcalloc(buddy->max_order + 1, 137 sizeof(*buddy->num_free), 138 GFP_KERNEL); 139 if (!buddy->bits || !buddy->num_free) 140 goto err_out; 141 142 for (i = 0; i <= buddy->max_order; ++i) { 143 s = BITS_TO_LONGS(1 << (buddy->max_order - i)); 144 buddy->bits[i] = kcalloc(s, sizeof(long), GFP_KERNEL | 145 __GFP_NOWARN); 146 if (!buddy->bits[i]) { 147 buddy->bits[i] = vzalloc(array_size(s, sizeof(long))); 148 if (!buddy->bits[i]) 149 goto err_out_free; 150 } 151 } 152 153 set_bit(0, buddy->bits[buddy->max_order]); 154 buddy->num_free[buddy->max_order] = 1; 155 156 return 0; 157 158 err_out_free: 159 for (i = 0; i <= buddy->max_order; ++i) 160 kvfree(buddy->bits[i]); 161 162 err_out: 163 kfree(buddy->bits); 164 kfree(buddy->num_free); 165 return -ENOMEM; 166 } 167 168 static void hns_roce_buddy_cleanup(struct hns_roce_buddy *buddy) 169 { 170 int i; 171 172 for (i = 0; i <= buddy->max_order; ++i) 173 kvfree(buddy->bits[i]); 174 175 kfree(buddy->bits); 176 kfree(buddy->num_free); 177 } 178 179 static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order, 180 unsigned long *seg, u32 mtt_type) 181 { 182 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 183 struct hns_roce_hem_table *table; 184 struct hns_roce_buddy *buddy; 185 int ret; 186 187 if (mtt_type == MTT_TYPE_WQE) { 188 buddy = &mr_table->mtt_buddy; 189 table = &mr_table->mtt_table; 190 } else { 191 buddy = &mr_table->mtt_cqe_buddy; 192 table = &mr_table->mtt_cqe_table; 193 } 194 195 ret = hns_roce_buddy_alloc(buddy, order, seg); 196 if (ret == -1) 197 return -1; 198 199 if (hns_roce_table_get_range(hr_dev, table, *seg, 200 *seg + (1 << order) - 1)) { 201 hns_roce_buddy_free(buddy, *seg, order); 202 return -1; 203 } 204 205 return 0; 206 } 207 208 int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift, 209 struct hns_roce_mtt *mtt) 210 { 211 int ret; 212 int i; 213 214 /* Page num is zero, correspond to DMA memory register */ 215 if (!npages) { 216 mtt->order = -1; 217 mtt->page_shift = HNS_ROCE_HEM_PAGE_SHIFT; 218 return 0; 219 } 220 221 /* Note: if page_shift is zero, FAST memory register */ 222 mtt->page_shift = page_shift; 223 224 /* Compute MTT entry necessary */ 225 for (mtt->order = 0, i = HNS_ROCE_MTT_ENTRY_PER_SEG; i < npages; 226 i <<= 1) 227 ++mtt->order; 228 229 /* Allocate MTT entry */ 230 ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg, 231 mtt->mtt_type); 232 if (ret == -1) 233 return -ENOMEM; 234 235 return 0; 236 } 237 238 void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt) 239 { 240 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 241 242 if (mtt->order < 0) 243 return; 244 245 if (mtt->mtt_type == MTT_TYPE_WQE) { 246 hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg, 247 mtt->order); 248 hns_roce_table_put_range(hr_dev, &mr_table->mtt_table, 249 mtt->first_seg, 250 mtt->first_seg + (1 << mtt->order) - 1); 251 } else { 252 hns_roce_buddy_free(&mr_table->mtt_cqe_buddy, mtt->first_seg, 253 mtt->order); 254 hns_roce_table_put_range(hr_dev, &mr_table->mtt_cqe_table, 255 mtt->first_seg, 256 mtt->first_seg + (1 << mtt->order) - 1); 257 } 258 } 259 EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup); 260 261 static void hns_roce_loop_free(struct hns_roce_dev *hr_dev, 262 struct hns_roce_mr *mr, int err_loop_index, 263 int loop_i, int loop_j) 264 { 265 struct device *dev = hr_dev->dev; 266 u32 mhop_num; 267 u32 pbl_bt_sz; 268 u64 bt_idx; 269 int i, j; 270 271 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); 272 mhop_num = hr_dev->caps.pbl_hop_num; 273 274 i = loop_i; 275 if (mhop_num == 3 && err_loop_index == 2) { 276 for (; i >= 0; i--) { 277 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], 278 mr->pbl_l1_dma_addr[i]); 279 280 for (j = 0; j < pbl_bt_sz / 8; j++) { 281 if (i == loop_i && j >= loop_j) 282 break; 283 284 bt_idx = i * pbl_bt_sz / 8 + j; 285 dma_free_coherent(dev, pbl_bt_sz, 286 mr->pbl_bt_l2[bt_idx], 287 mr->pbl_l2_dma_addr[bt_idx]); 288 } 289 } 290 } else if (mhop_num == 3 && err_loop_index == 1) { 291 for (i -= 1; i >= 0; i--) { 292 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], 293 mr->pbl_l1_dma_addr[i]); 294 295 for (j = 0; j < pbl_bt_sz / 8; j++) { 296 bt_idx = i * pbl_bt_sz / 8 + j; 297 dma_free_coherent(dev, pbl_bt_sz, 298 mr->pbl_bt_l2[bt_idx], 299 mr->pbl_l2_dma_addr[bt_idx]); 300 } 301 } 302 } else if (mhop_num == 2 && err_loop_index == 1) { 303 for (i -= 1; i >= 0; i--) 304 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], 305 mr->pbl_l1_dma_addr[i]); 306 } else { 307 dev_warn(dev, "not support: mhop_num=%d, err_loop_index=%d.", 308 mhop_num, err_loop_index); 309 return; 310 } 311 312 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0, mr->pbl_l0_dma_addr); 313 mr->pbl_bt_l0 = NULL; 314 mr->pbl_l0_dma_addr = 0; 315 } 316 317 /* PBL multi hop addressing */ 318 static int hns_roce_mhop_alloc(struct hns_roce_dev *hr_dev, int npages, 319 struct hns_roce_mr *mr) 320 { 321 struct device *dev = hr_dev->dev; 322 int mr_alloc_done = 0; 323 int npages_allocated; 324 int i = 0, j = 0; 325 u32 pbl_bt_sz; 326 u32 mhop_num; 327 u64 pbl_last_bt_num; 328 u64 pbl_bt_cnt = 0; 329 u64 bt_idx; 330 u64 size; 331 332 mhop_num = hr_dev->caps.pbl_hop_num; 333 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); 334 pbl_last_bt_num = (npages + pbl_bt_sz / 8 - 1) / (pbl_bt_sz / 8); 335 336 if (mhop_num == HNS_ROCE_HOP_NUM_0) 337 return 0; 338 339 /* hop_num = 1 */ 340 if (mhop_num == 1) { 341 if (npages > pbl_bt_sz / 8) { 342 dev_err(dev, "npages %d is larger than buf_pg_sz!", 343 npages); 344 return -EINVAL; 345 } 346 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, 347 &(mr->pbl_dma_addr), 348 GFP_KERNEL); 349 if (!mr->pbl_buf) 350 return -ENOMEM; 351 352 mr->pbl_size = npages; 353 mr->pbl_ba = mr->pbl_dma_addr; 354 mr->pbl_hop_num = hr_dev->caps.pbl_hop_num; 355 mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz; 356 mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz; 357 return 0; 358 } 359 360 mr->pbl_l1_dma_addr = kcalloc(pbl_bt_sz / 8, 361 sizeof(*mr->pbl_l1_dma_addr), 362 GFP_KERNEL); 363 if (!mr->pbl_l1_dma_addr) 364 return -ENOMEM; 365 366 mr->pbl_bt_l1 = kcalloc(pbl_bt_sz / 8, sizeof(*mr->pbl_bt_l1), 367 GFP_KERNEL); 368 if (!mr->pbl_bt_l1) 369 goto err_kcalloc_bt_l1; 370 371 if (mhop_num == 3) { 372 mr->pbl_l2_dma_addr = kcalloc(pbl_last_bt_num, 373 sizeof(*mr->pbl_l2_dma_addr), 374 GFP_KERNEL); 375 if (!mr->pbl_l2_dma_addr) 376 goto err_kcalloc_l2_dma; 377 378 mr->pbl_bt_l2 = kcalloc(pbl_last_bt_num, 379 sizeof(*mr->pbl_bt_l2), 380 GFP_KERNEL); 381 if (!mr->pbl_bt_l2) 382 goto err_kcalloc_bt_l2; 383 } 384 385 /* alloc L0 BT */ 386 mr->pbl_bt_l0 = dma_alloc_coherent(dev, pbl_bt_sz, 387 &(mr->pbl_l0_dma_addr), 388 GFP_KERNEL); 389 if (!mr->pbl_bt_l0) 390 goto err_dma_alloc_l0; 391 392 if (mhop_num == 2) { 393 /* alloc L1 BT */ 394 for (i = 0; i < pbl_bt_sz / 8; i++) { 395 if (pbl_bt_cnt + 1 < pbl_last_bt_num) { 396 size = pbl_bt_sz; 397 } else { 398 npages_allocated = i * (pbl_bt_sz / 8); 399 size = (npages - npages_allocated) * 8; 400 } 401 mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, size, 402 &(mr->pbl_l1_dma_addr[i]), 403 GFP_KERNEL); 404 if (!mr->pbl_bt_l1[i]) { 405 hns_roce_loop_free(hr_dev, mr, 1, i, 0); 406 goto err_dma_alloc_l0; 407 } 408 409 *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i]; 410 411 pbl_bt_cnt++; 412 if (pbl_bt_cnt >= pbl_last_bt_num) 413 break; 414 } 415 } else if (mhop_num == 3) { 416 /* alloc L1, L2 BT */ 417 for (i = 0; i < pbl_bt_sz / 8; i++) { 418 mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, pbl_bt_sz, 419 &(mr->pbl_l1_dma_addr[i]), 420 GFP_KERNEL); 421 if (!mr->pbl_bt_l1[i]) { 422 hns_roce_loop_free(hr_dev, mr, 1, i, 0); 423 goto err_dma_alloc_l0; 424 } 425 426 *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i]; 427 428 for (j = 0; j < pbl_bt_sz / 8; j++) { 429 bt_idx = i * pbl_bt_sz / 8 + j; 430 431 if (pbl_bt_cnt + 1 < pbl_last_bt_num) { 432 size = pbl_bt_sz; 433 } else { 434 npages_allocated = bt_idx * 435 (pbl_bt_sz / 8); 436 size = (npages - npages_allocated) * 8; 437 } 438 mr->pbl_bt_l2[bt_idx] = dma_alloc_coherent( 439 dev, size, 440 &(mr->pbl_l2_dma_addr[bt_idx]), 441 GFP_KERNEL); 442 if (!mr->pbl_bt_l2[bt_idx]) { 443 hns_roce_loop_free(hr_dev, mr, 2, i, j); 444 goto err_dma_alloc_l0; 445 } 446 447 *(mr->pbl_bt_l1[i] + j) = 448 mr->pbl_l2_dma_addr[bt_idx]; 449 450 pbl_bt_cnt++; 451 if (pbl_bt_cnt >= pbl_last_bt_num) { 452 mr_alloc_done = 1; 453 break; 454 } 455 } 456 457 if (mr_alloc_done) 458 break; 459 } 460 } 461 462 mr->l0_chunk_last_num = i + 1; 463 if (mhop_num == 3) 464 mr->l1_chunk_last_num = j + 1; 465 466 mr->pbl_size = npages; 467 mr->pbl_ba = mr->pbl_l0_dma_addr; 468 mr->pbl_hop_num = hr_dev->caps.pbl_hop_num; 469 mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz; 470 mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz; 471 472 return 0; 473 474 err_dma_alloc_l0: 475 kfree(mr->pbl_bt_l2); 476 mr->pbl_bt_l2 = NULL; 477 478 err_kcalloc_bt_l2: 479 kfree(mr->pbl_l2_dma_addr); 480 mr->pbl_l2_dma_addr = NULL; 481 482 err_kcalloc_l2_dma: 483 kfree(mr->pbl_bt_l1); 484 mr->pbl_bt_l1 = NULL; 485 486 err_kcalloc_bt_l1: 487 kfree(mr->pbl_l1_dma_addr); 488 mr->pbl_l1_dma_addr = NULL; 489 490 return -ENOMEM; 491 } 492 493 static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova, 494 u64 size, u32 access, int npages, 495 struct hns_roce_mr *mr) 496 { 497 struct device *dev = hr_dev->dev; 498 unsigned long index = 0; 499 int ret = 0; 500 501 /* Allocate a key for mr from mr_table */ 502 ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index); 503 if (ret == -1) 504 return -ENOMEM; 505 506 mr->iova = iova; /* MR va starting addr */ 507 mr->size = size; /* MR addr range */ 508 mr->pd = pd; /* MR num */ 509 mr->access = access; /* MR access permit */ 510 mr->enabled = 0; /* MR active status */ 511 mr->key = hw_index_to_key(index); /* MR key */ 512 513 if (size == ~0ull) { 514 mr->type = MR_TYPE_DMA; 515 mr->pbl_buf = NULL; 516 mr->pbl_dma_addr = 0; 517 /* PBL multi-hop addressing parameters */ 518 mr->pbl_bt_l2 = NULL; 519 mr->pbl_bt_l1 = NULL; 520 mr->pbl_bt_l0 = NULL; 521 mr->pbl_l2_dma_addr = NULL; 522 mr->pbl_l1_dma_addr = NULL; 523 mr->pbl_l0_dma_addr = 0; 524 } else { 525 mr->type = MR_TYPE_MR; 526 if (!hr_dev->caps.pbl_hop_num) { 527 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, 528 &(mr->pbl_dma_addr), 529 GFP_KERNEL); 530 if (!mr->pbl_buf) 531 return -ENOMEM; 532 } else { 533 ret = hns_roce_mhop_alloc(hr_dev, npages, mr); 534 } 535 } 536 537 return ret; 538 } 539 540 static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev, 541 struct hns_roce_mr *mr) 542 { 543 struct device *dev = hr_dev->dev; 544 int npages_allocated; 545 int npages; 546 int i, j; 547 u32 pbl_bt_sz; 548 u32 mhop_num; 549 u64 bt_idx; 550 551 npages = ib_umem_page_count(mr->umem); 552 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); 553 mhop_num = hr_dev->caps.pbl_hop_num; 554 555 if (mhop_num == HNS_ROCE_HOP_NUM_0) 556 return; 557 558 /* hop_num = 1 */ 559 if (mhop_num == 1) { 560 dma_free_coherent(dev, (unsigned int)(npages * 8), 561 mr->pbl_buf, mr->pbl_dma_addr); 562 return; 563 } 564 565 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0, 566 mr->pbl_l0_dma_addr); 567 568 if (mhop_num == 2) { 569 for (i = 0; i < mr->l0_chunk_last_num; i++) { 570 if (i == mr->l0_chunk_last_num - 1) { 571 npages_allocated = i * (pbl_bt_sz / 8); 572 573 dma_free_coherent(dev, 574 (npages - npages_allocated) * 8, 575 mr->pbl_bt_l1[i], 576 mr->pbl_l1_dma_addr[i]); 577 578 break; 579 } 580 581 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], 582 mr->pbl_l1_dma_addr[i]); 583 } 584 } else if (mhop_num == 3) { 585 for (i = 0; i < mr->l0_chunk_last_num; i++) { 586 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], 587 mr->pbl_l1_dma_addr[i]); 588 589 for (j = 0; j < pbl_bt_sz / 8; j++) { 590 bt_idx = i * (pbl_bt_sz / 8) + j; 591 592 if ((i == mr->l0_chunk_last_num - 1) 593 && j == mr->l1_chunk_last_num - 1) { 594 npages_allocated = bt_idx * 595 (pbl_bt_sz / 8); 596 597 dma_free_coherent(dev, 598 (npages - npages_allocated) * 8, 599 mr->pbl_bt_l2[bt_idx], 600 mr->pbl_l2_dma_addr[bt_idx]); 601 602 break; 603 } 604 605 dma_free_coherent(dev, pbl_bt_sz, 606 mr->pbl_bt_l2[bt_idx], 607 mr->pbl_l2_dma_addr[bt_idx]); 608 } 609 } 610 } 611 612 kfree(mr->pbl_bt_l1); 613 kfree(mr->pbl_l1_dma_addr); 614 mr->pbl_bt_l1 = NULL; 615 mr->pbl_l1_dma_addr = NULL; 616 if (mhop_num == 3) { 617 kfree(mr->pbl_bt_l2); 618 kfree(mr->pbl_l2_dma_addr); 619 mr->pbl_bt_l2 = NULL; 620 mr->pbl_l2_dma_addr = NULL; 621 } 622 } 623 624 static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, 625 struct hns_roce_mr *mr) 626 { 627 struct device *dev = hr_dev->dev; 628 int npages = 0; 629 int ret; 630 631 if (mr->enabled) { 632 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key) 633 & (hr_dev->caps.num_mtpts - 1)); 634 if (ret) 635 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret); 636 } 637 638 if (mr->size != ~0ULL) { 639 npages = ib_umem_page_count(mr->umem); 640 641 if (!hr_dev->caps.pbl_hop_num) 642 dma_free_coherent(dev, (unsigned int)(npages * 8), 643 mr->pbl_buf, mr->pbl_dma_addr); 644 else 645 hns_roce_mhop_free(hr_dev, mr); 646 } 647 648 if (mr->enabled) 649 hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table, 650 key_to_hw_index(mr->key)); 651 652 hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, 653 key_to_hw_index(mr->key), BITMAP_NO_RR); 654 } 655 656 static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, 657 struct hns_roce_mr *mr) 658 { 659 int ret; 660 unsigned long mtpt_idx = key_to_hw_index(mr->key); 661 struct device *dev = hr_dev->dev; 662 struct hns_roce_cmd_mailbox *mailbox; 663 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 664 665 /* Prepare HEM entry memory */ 666 ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx); 667 if (ret) 668 return ret; 669 670 /* Allocate mailbox memory */ 671 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 672 if (IS_ERR(mailbox)) { 673 ret = PTR_ERR(mailbox); 674 goto err_table; 675 } 676 677 ret = hr_dev->hw->write_mtpt(mailbox->buf, mr, mtpt_idx); 678 if (ret) { 679 dev_err(dev, "Write mtpt fail!\n"); 680 goto err_page; 681 } 682 683 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, 684 mtpt_idx & (hr_dev->caps.num_mtpts - 1)); 685 if (ret) { 686 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret); 687 goto err_page; 688 } 689 690 mr->enabled = 1; 691 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 692 693 return 0; 694 695 err_page: 696 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 697 698 err_table: 699 hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx); 700 return ret; 701 } 702 703 static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev, 704 struct hns_roce_mtt *mtt, u32 start_index, 705 u32 npages, u64 *page_list) 706 { 707 struct hns_roce_hem_table *table; 708 dma_addr_t dma_handle; 709 __le64 *mtts; 710 u32 s = start_index * sizeof(u64); 711 u32 bt_page_size; 712 u32 i; 713 714 if (mtt->mtt_type == MTT_TYPE_WQE) 715 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT); 716 else 717 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT); 718 719 /* All MTTs must fit in the same page */ 720 if (start_index / (bt_page_size / sizeof(u64)) != 721 (start_index + npages - 1) / (bt_page_size / sizeof(u64))) 722 return -EINVAL; 723 724 if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1)) 725 return -EINVAL; 726 727 if (mtt->mtt_type == MTT_TYPE_WQE) 728 table = &hr_dev->mr_table.mtt_table; 729 else 730 table = &hr_dev->mr_table.mtt_cqe_table; 731 732 mtts = hns_roce_table_find(hr_dev, table, 733 mtt->first_seg + s / hr_dev->caps.mtt_entry_sz, 734 &dma_handle); 735 if (!mtts) 736 return -ENOMEM; 737 738 /* Save page addr, low 12 bits : 0 */ 739 for (i = 0; i < npages; ++i) { 740 if (!hr_dev->caps.mtt_hop_num) 741 mtts[i] = cpu_to_le64(page_list[i] >> PAGE_ADDR_SHIFT); 742 else 743 mtts[i] = cpu_to_le64(page_list[i]); 744 } 745 746 return 0; 747 } 748 749 static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev, 750 struct hns_roce_mtt *mtt, u32 start_index, 751 u32 npages, u64 *page_list) 752 { 753 int chunk; 754 int ret; 755 u32 bt_page_size; 756 757 if (mtt->order < 0) 758 return -EINVAL; 759 760 if (mtt->mtt_type == MTT_TYPE_WQE) 761 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT); 762 else 763 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT); 764 765 while (npages > 0) { 766 chunk = min_t(int, bt_page_size / sizeof(u64), npages); 767 768 ret = hns_roce_write_mtt_chunk(hr_dev, mtt, start_index, chunk, 769 page_list); 770 if (ret) 771 return ret; 772 773 npages -= chunk; 774 start_index += chunk; 775 page_list += chunk; 776 } 777 778 return 0; 779 } 780 781 int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev, 782 struct hns_roce_mtt *mtt, struct hns_roce_buf *buf) 783 { 784 u64 *page_list; 785 int ret; 786 u32 i; 787 788 page_list = kmalloc_array(buf->npages, sizeof(*page_list), GFP_KERNEL); 789 if (!page_list) 790 return -ENOMEM; 791 792 for (i = 0; i < buf->npages; ++i) { 793 if (buf->nbufs == 1) 794 page_list[i] = buf->direct.map + (i << buf->page_shift); 795 else 796 page_list[i] = buf->page_list[i].map; 797 798 } 799 ret = hns_roce_write_mtt(hr_dev, mtt, 0, buf->npages, page_list); 800 801 kfree(page_list); 802 803 return ret; 804 } 805 806 int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev) 807 { 808 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 809 int ret; 810 811 ret = hns_roce_bitmap_init(&mr_table->mtpt_bitmap, 812 hr_dev->caps.num_mtpts, 813 hr_dev->caps.num_mtpts - 1, 814 hr_dev->caps.reserved_mrws, 0); 815 if (ret) 816 return ret; 817 818 ret = hns_roce_buddy_init(&mr_table->mtt_buddy, 819 ilog2(hr_dev->caps.num_mtt_segs)); 820 if (ret) 821 goto err_buddy; 822 823 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) { 824 ret = hns_roce_buddy_init(&mr_table->mtt_cqe_buddy, 825 ilog2(hr_dev->caps.num_cqe_segs)); 826 if (ret) 827 goto err_buddy_cqe; 828 } 829 return 0; 830 831 err_buddy_cqe: 832 hns_roce_buddy_cleanup(&mr_table->mtt_buddy); 833 834 err_buddy: 835 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap); 836 return ret; 837 } 838 839 void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev) 840 { 841 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 842 843 hns_roce_buddy_cleanup(&mr_table->mtt_buddy); 844 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) 845 hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy); 846 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap); 847 } 848 849 struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc) 850 { 851 struct hns_roce_mr *mr; 852 int ret; 853 854 mr = kmalloc(sizeof(*mr), GFP_KERNEL); 855 if (mr == NULL) 856 return ERR_PTR(-ENOMEM); 857 858 /* Allocate memory region key */ 859 ret = hns_roce_mr_alloc(to_hr_dev(pd->device), to_hr_pd(pd)->pdn, 0, 860 ~0ULL, acc, 0, mr); 861 if (ret) 862 goto err_free; 863 864 ret = hns_roce_mr_enable(to_hr_dev(pd->device), mr); 865 if (ret) 866 goto err_mr; 867 868 mr->ibmr.rkey = mr->ibmr.lkey = mr->key; 869 mr->umem = NULL; 870 871 return &mr->ibmr; 872 873 err_mr: 874 hns_roce_mr_free(to_hr_dev(pd->device), mr); 875 876 err_free: 877 kfree(mr); 878 return ERR_PTR(ret); 879 } 880 881 int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, 882 struct hns_roce_mtt *mtt, struct ib_umem *umem) 883 { 884 struct device *dev = hr_dev->dev; 885 struct scatterlist *sg; 886 unsigned int order; 887 int i, k, entry; 888 int npage = 0; 889 int ret = 0; 890 int len; 891 u64 page_addr; 892 u64 *pages; 893 u32 bt_page_size; 894 u32 n; 895 896 order = mtt->mtt_type == MTT_TYPE_WQE ? hr_dev->caps.mtt_ba_pg_sz : 897 hr_dev->caps.cqe_ba_pg_sz; 898 bt_page_size = 1 << (order + PAGE_SHIFT); 899 900 pages = (u64 *) __get_free_pages(GFP_KERNEL, order); 901 if (!pages) 902 return -ENOMEM; 903 904 i = n = 0; 905 906 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 907 len = sg_dma_len(sg) >> PAGE_SHIFT; 908 for (k = 0; k < len; ++k) { 909 page_addr = 910 sg_dma_address(sg) + (k << umem->page_shift); 911 if (!(npage % (1 << (mtt->page_shift - PAGE_SHIFT)))) { 912 if (page_addr & ((1 << mtt->page_shift) - 1)) { 913 dev_err(dev, "page_addr 0x%llx is not page_shift %d alignment!\n", 914 page_addr, mtt->page_shift); 915 ret = -EINVAL; 916 goto out; 917 } 918 pages[i++] = page_addr; 919 } 920 npage++; 921 if (i == bt_page_size / sizeof(u64)) { 922 ret = hns_roce_write_mtt(hr_dev, mtt, n, i, 923 pages); 924 if (ret) 925 goto out; 926 n += i; 927 i = 0; 928 } 929 } 930 } 931 932 if (i) 933 ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages); 934 935 out: 936 free_pages((unsigned long) pages, order); 937 return ret; 938 } 939 940 static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev, 941 struct hns_roce_mr *mr, 942 struct ib_umem *umem) 943 { 944 struct scatterlist *sg; 945 int i = 0, j = 0, k; 946 int entry; 947 int len; 948 u64 page_addr; 949 u32 pbl_bt_sz; 950 951 if (hr_dev->caps.pbl_hop_num == HNS_ROCE_HOP_NUM_0) 952 return 0; 953 954 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); 955 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 956 len = sg_dma_len(sg) >> PAGE_SHIFT; 957 for (k = 0; k < len; ++k) { 958 page_addr = sg_dma_address(sg) + 959 (k << umem->page_shift); 960 961 if (!hr_dev->caps.pbl_hop_num) { 962 mr->pbl_buf[i++] = page_addr >> 12; 963 } else if (hr_dev->caps.pbl_hop_num == 1) { 964 mr->pbl_buf[i++] = page_addr; 965 } else { 966 if (hr_dev->caps.pbl_hop_num == 2) 967 mr->pbl_bt_l1[i][j] = page_addr; 968 else if (hr_dev->caps.pbl_hop_num == 3) 969 mr->pbl_bt_l2[i][j] = page_addr; 970 971 j++; 972 if (j >= (pbl_bt_sz / 8)) { 973 i++; 974 j = 0; 975 } 976 } 977 } 978 } 979 980 /* Memory barrier */ 981 mb(); 982 983 return 0; 984 } 985 986 struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 987 u64 virt_addr, int access_flags, 988 struct ib_udata *udata) 989 { 990 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); 991 struct device *dev = hr_dev->dev; 992 struct hns_roce_mr *mr; 993 int bt_size; 994 int ret; 995 int n; 996 int i; 997 998 mr = kmalloc(sizeof(*mr), GFP_KERNEL); 999 if (!mr) 1000 return ERR_PTR(-ENOMEM); 1001 1002 mr->umem = ib_umem_get(pd->uobject->context, start, length, 1003 access_flags, 0); 1004 if (IS_ERR(mr->umem)) { 1005 ret = PTR_ERR(mr->umem); 1006 goto err_free; 1007 } 1008 1009 n = ib_umem_page_count(mr->umem); 1010 1011 if (!hr_dev->caps.pbl_hop_num) { 1012 if (n > HNS_ROCE_MAX_MTPT_PBL_NUM) { 1013 dev_err(dev, 1014 " MR len %lld err. MR is limited to 4G at most!\n", 1015 length); 1016 ret = -EINVAL; 1017 goto err_umem; 1018 } 1019 } else { 1020 int pbl_size = 1; 1021 1022 bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) / 8; 1023 for (i = 0; i < hr_dev->caps.pbl_hop_num; i++) 1024 pbl_size *= bt_size; 1025 if (n > pbl_size) { 1026 dev_err(dev, 1027 " MR len %lld err. MR page num is limited to %d!\n", 1028 length, pbl_size); 1029 ret = -EINVAL; 1030 goto err_umem; 1031 } 1032 } 1033 1034 ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, virt_addr, length, 1035 access_flags, n, mr); 1036 if (ret) 1037 goto err_umem; 1038 1039 ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem); 1040 if (ret) 1041 goto err_mr; 1042 1043 ret = hns_roce_mr_enable(hr_dev, mr); 1044 if (ret) 1045 goto err_mr; 1046 1047 mr->ibmr.rkey = mr->ibmr.lkey = mr->key; 1048 1049 return &mr->ibmr; 1050 1051 err_mr: 1052 hns_roce_mr_free(hr_dev, mr); 1053 1054 err_umem: 1055 ib_umem_release(mr->umem); 1056 1057 err_free: 1058 kfree(mr); 1059 return ERR_PTR(ret); 1060 } 1061 1062 int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length, 1063 u64 virt_addr, int mr_access_flags, struct ib_pd *pd, 1064 struct ib_udata *udata) 1065 { 1066 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); 1067 struct hns_roce_mr *mr = to_hr_mr(ibmr); 1068 struct hns_roce_cmd_mailbox *mailbox; 1069 struct device *dev = hr_dev->dev; 1070 unsigned long mtpt_idx; 1071 u32 pdn = 0; 1072 int npages; 1073 int ret; 1074 1075 if (!mr->enabled) 1076 return -EINVAL; 1077 1078 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 1079 if (IS_ERR(mailbox)) 1080 return PTR_ERR(mailbox); 1081 1082 mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1); 1083 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, mtpt_idx, 0, 1084 HNS_ROCE_CMD_QUERY_MPT, 1085 HNS_ROCE_CMD_TIMEOUT_MSECS); 1086 if (ret) 1087 goto free_cmd_mbox; 1088 1089 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, mtpt_idx); 1090 if (ret) 1091 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret); 1092 1093 mr->enabled = 0; 1094 1095 if (flags & IB_MR_REREG_PD) 1096 pdn = to_hr_pd(pd)->pdn; 1097 1098 if (flags & IB_MR_REREG_TRANS) { 1099 if (mr->size != ~0ULL) { 1100 npages = ib_umem_page_count(mr->umem); 1101 1102 if (hr_dev->caps.pbl_hop_num) 1103 hns_roce_mhop_free(hr_dev, mr); 1104 else 1105 dma_free_coherent(dev, npages * 8, mr->pbl_buf, 1106 mr->pbl_dma_addr); 1107 } 1108 ib_umem_release(mr->umem); 1109 1110 mr->umem = ib_umem_get(ibmr->uobject->context, start, length, 1111 mr_access_flags, 0); 1112 if (IS_ERR(mr->umem)) { 1113 ret = PTR_ERR(mr->umem); 1114 mr->umem = NULL; 1115 goto free_cmd_mbox; 1116 } 1117 npages = ib_umem_page_count(mr->umem); 1118 1119 if (hr_dev->caps.pbl_hop_num) { 1120 ret = hns_roce_mhop_alloc(hr_dev, npages, mr); 1121 if (ret) 1122 goto release_umem; 1123 } else { 1124 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, 1125 &(mr->pbl_dma_addr), 1126 GFP_KERNEL); 1127 if (!mr->pbl_buf) { 1128 ret = -ENOMEM; 1129 goto release_umem; 1130 } 1131 } 1132 } 1133 1134 ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn, 1135 mr_access_flags, virt_addr, 1136 length, mailbox->buf); 1137 if (ret) { 1138 if (flags & IB_MR_REREG_TRANS) 1139 goto release_umem; 1140 else 1141 goto free_cmd_mbox; 1142 } 1143 1144 if (flags & IB_MR_REREG_TRANS) { 1145 ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem); 1146 if (ret) { 1147 if (mr->size != ~0ULL) { 1148 npages = ib_umem_page_count(mr->umem); 1149 1150 if (hr_dev->caps.pbl_hop_num) 1151 hns_roce_mhop_free(hr_dev, mr); 1152 else 1153 dma_free_coherent(dev, npages * 8, 1154 mr->pbl_buf, 1155 mr->pbl_dma_addr); 1156 } 1157 1158 goto release_umem; 1159 } 1160 } 1161 1162 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, mtpt_idx); 1163 if (ret) { 1164 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret); 1165 goto release_umem; 1166 } 1167 1168 mr->enabled = 1; 1169 if (flags & IB_MR_REREG_ACCESS) 1170 mr->access = mr_access_flags; 1171 1172 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 1173 1174 return 0; 1175 1176 release_umem: 1177 ib_umem_release(mr->umem); 1178 1179 free_cmd_mbox: 1180 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 1181 1182 return ret; 1183 } 1184 1185 int hns_roce_dereg_mr(struct ib_mr *ibmr) 1186 { 1187 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); 1188 struct hns_roce_mr *mr = to_hr_mr(ibmr); 1189 int ret = 0; 1190 1191 if (hr_dev->hw->dereg_mr) { 1192 ret = hr_dev->hw->dereg_mr(hr_dev, mr); 1193 } else { 1194 hns_roce_mr_free(hr_dev, mr); 1195 1196 if (mr->umem) 1197 ib_umem_release(mr->umem); 1198 1199 kfree(mr); 1200 } 1201 1202 return ret; 1203 } 1204