1 /* 2 * Common eBPF ELF object loading operations. 3 * 4 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 5 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 6 * Copyright (C) 2015 Huawei Inc. 7 */ 8 9 #include <stdlib.h> 10 #include <stdio.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <string.h> 14 #include <unistd.h> 15 #include <fcntl.h> 16 #include <errno.h> 17 #include <asm/unistd.h> 18 #include <linux/kernel.h> 19 #include <linux/bpf.h> 20 #include <linux/list.h> 21 #include <libelf.h> 22 #include <gelf.h> 23 24 #include "libbpf.h" 25 #include "bpf.h" 26 27 #define __printf(a, b) __attribute__((format(printf, a, b))) 28 29 __printf(1, 2) 30 static int __base_pr(const char *format, ...) 31 { 32 va_list args; 33 int err; 34 35 va_start(args, format); 36 err = vfprintf(stderr, format, args); 37 va_end(args); 38 return err; 39 } 40 41 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; 42 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; 43 static __printf(1, 2) libbpf_print_fn_t __pr_debug; 44 45 #define __pr(func, fmt, ...) \ 46 do { \ 47 if ((func)) \ 48 (func)("libbpf: " fmt, ##__VA_ARGS__); \ 49 } while (0) 50 51 #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) 52 #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) 53 #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) 54 55 void libbpf_set_print(libbpf_print_fn_t warn, 56 libbpf_print_fn_t info, 57 libbpf_print_fn_t debug) 58 { 59 __pr_warning = warn; 60 __pr_info = info; 61 __pr_debug = debug; 62 } 63 64 #define STRERR_BUFSIZE 128 65 66 #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START) 67 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c) 68 #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START) 69 70 static const char *libbpf_strerror_table[NR_ERRNO] = { 71 [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf", 72 [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid", 73 [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost", 74 [ERRCODE_OFFSET(ENDIAN)] = "Endian missmatch", 75 [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf", 76 [ERRCODE_OFFSET(RELOC)] = "Relocation failed", 77 [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading", 78 [ERRCODE_OFFSET(PROG2BIG)] = "Program too big", 79 [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version", 80 }; 81 82 int libbpf_strerror(int err, char *buf, size_t size) 83 { 84 if (!buf || !size) 85 return -1; 86 87 err = err > 0 ? err : -err; 88 89 if (err < __LIBBPF_ERRNO__START) { 90 int ret; 91 92 ret = strerror_r(err, buf, size); 93 buf[size - 1] = '\0'; 94 return ret; 95 } 96 97 if (err < __LIBBPF_ERRNO__END) { 98 const char *msg; 99 100 msg = libbpf_strerror_table[ERRNO_OFFSET(err)]; 101 snprintf(buf, size, "%s", msg); 102 buf[size - 1] = '\0'; 103 return 0; 104 } 105 106 snprintf(buf, size, "Unknown libbpf error %d", err); 107 buf[size - 1] = '\0'; 108 return -1; 109 } 110 111 #define CHECK_ERR(action, err, out) do { \ 112 err = action; \ 113 if (err) \ 114 goto out; \ 115 } while(0) 116 117 118 /* Copied from tools/perf/util/util.h */ 119 #ifndef zfree 120 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 121 #endif 122 123 #ifndef zclose 124 # define zclose(fd) ({ \ 125 int ___err = 0; \ 126 if ((fd) >= 0) \ 127 ___err = close((fd)); \ 128 fd = -1; \ 129 ___err; }) 130 #endif 131 132 #ifdef HAVE_LIBELF_MMAP_SUPPORT 133 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 134 #else 135 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 136 #endif 137 138 /* 139 * bpf_prog should be a better name but it has been used in 140 * linux/filter.h. 141 */ 142 struct bpf_program { 143 /* Index in elf obj file, for relocation use. */ 144 int idx; 145 char *section_name; 146 struct bpf_insn *insns; 147 size_t insns_cnt; 148 149 struct { 150 int insn_idx; 151 int map_idx; 152 } *reloc_desc; 153 int nr_reloc; 154 155 int fd; 156 157 struct bpf_object *obj; 158 void *priv; 159 bpf_program_clear_priv_t clear_priv; 160 }; 161 162 static LIST_HEAD(bpf_objects_list); 163 164 struct bpf_object { 165 char license[64]; 166 u32 kern_version; 167 void *maps_buf; 168 size_t maps_buf_sz; 169 170 struct bpf_program *programs; 171 size_t nr_programs; 172 int *map_fds; 173 /* 174 * This field is required because maps_buf will be freed and 175 * maps_buf_sz will be set to 0 after loaded. 176 */ 177 size_t nr_map_fds; 178 bool loaded; 179 180 /* 181 * Information when doing elf related work. Only valid if fd 182 * is valid. 183 */ 184 struct { 185 int fd; 186 void *obj_buf; 187 size_t obj_buf_sz; 188 Elf *elf; 189 GElf_Ehdr ehdr; 190 Elf_Data *symbols; 191 struct { 192 GElf_Shdr shdr; 193 Elf_Data *data; 194 } *reloc; 195 int nr_reloc; 196 } efile; 197 /* 198 * All loaded bpf_object is linked in a list, which is 199 * hidden to caller. bpf_objects__<func> handlers deal with 200 * all objects. 201 */ 202 struct list_head list; 203 char path[]; 204 }; 205 #define obj_elf_valid(o) ((o)->efile.elf) 206 207 static void bpf_program__unload(struct bpf_program *prog) 208 { 209 if (!prog) 210 return; 211 212 zclose(prog->fd); 213 } 214 215 static void bpf_program__exit(struct bpf_program *prog) 216 { 217 if (!prog) 218 return; 219 220 if (prog->clear_priv) 221 prog->clear_priv(prog, prog->priv); 222 223 prog->priv = NULL; 224 prog->clear_priv = NULL; 225 226 bpf_program__unload(prog); 227 zfree(&prog->section_name); 228 zfree(&prog->insns); 229 zfree(&prog->reloc_desc); 230 231 prog->nr_reloc = 0; 232 prog->insns_cnt = 0; 233 prog->idx = -1; 234 } 235 236 static int 237 bpf_program__init(void *data, size_t size, char *name, int idx, 238 struct bpf_program *prog) 239 { 240 if (size < sizeof(struct bpf_insn)) { 241 pr_warning("corrupted section '%s'\n", name); 242 return -EINVAL; 243 } 244 245 bzero(prog, sizeof(*prog)); 246 247 prog->section_name = strdup(name); 248 if (!prog->section_name) { 249 pr_warning("failed to alloc name for prog %s\n", 250 name); 251 goto errout; 252 } 253 254 prog->insns = malloc(size); 255 if (!prog->insns) { 256 pr_warning("failed to alloc insns for %s\n", name); 257 goto errout; 258 } 259 prog->insns_cnt = size / sizeof(struct bpf_insn); 260 memcpy(prog->insns, data, 261 prog->insns_cnt * sizeof(struct bpf_insn)); 262 prog->idx = idx; 263 prog->fd = -1; 264 265 return 0; 266 errout: 267 bpf_program__exit(prog); 268 return -ENOMEM; 269 } 270 271 static int 272 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 273 char *name, int idx) 274 { 275 struct bpf_program prog, *progs; 276 int nr_progs, err; 277 278 err = bpf_program__init(data, size, name, idx, &prog); 279 if (err) 280 return err; 281 282 progs = obj->programs; 283 nr_progs = obj->nr_programs; 284 285 progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); 286 if (!progs) { 287 /* 288 * In this case the original obj->programs 289 * is still valid, so don't need special treat for 290 * bpf_close_object(). 291 */ 292 pr_warning("failed to alloc a new program '%s'\n", 293 name); 294 bpf_program__exit(&prog); 295 return -ENOMEM; 296 } 297 298 pr_debug("found program %s\n", prog.section_name); 299 obj->programs = progs; 300 obj->nr_programs = nr_progs + 1; 301 prog.obj = obj; 302 progs[nr_progs] = prog; 303 return 0; 304 } 305 306 static struct bpf_object *bpf_object__new(const char *path, 307 void *obj_buf, 308 size_t obj_buf_sz) 309 { 310 struct bpf_object *obj; 311 312 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 313 if (!obj) { 314 pr_warning("alloc memory failed for %s\n", path); 315 return ERR_PTR(-ENOMEM); 316 } 317 318 strcpy(obj->path, path); 319 obj->efile.fd = -1; 320 321 /* 322 * Caller of this function should also calls 323 * bpf_object__elf_finish() after data collection to return 324 * obj_buf to user. If not, we should duplicate the buffer to 325 * avoid user freeing them before elf finish. 326 */ 327 obj->efile.obj_buf = obj_buf; 328 obj->efile.obj_buf_sz = obj_buf_sz; 329 330 obj->loaded = false; 331 332 INIT_LIST_HEAD(&obj->list); 333 list_add(&obj->list, &bpf_objects_list); 334 return obj; 335 } 336 337 static void bpf_object__elf_finish(struct bpf_object *obj) 338 { 339 if (!obj_elf_valid(obj)) 340 return; 341 342 if (obj->efile.elf) { 343 elf_end(obj->efile.elf); 344 obj->efile.elf = NULL; 345 } 346 obj->efile.symbols = NULL; 347 348 zfree(&obj->efile.reloc); 349 obj->efile.nr_reloc = 0; 350 zclose(obj->efile.fd); 351 obj->efile.obj_buf = NULL; 352 obj->efile.obj_buf_sz = 0; 353 } 354 355 static int bpf_object__elf_init(struct bpf_object *obj) 356 { 357 int err = 0; 358 GElf_Ehdr *ep; 359 360 if (obj_elf_valid(obj)) { 361 pr_warning("elf init: internal error\n"); 362 return -LIBBPF_ERRNO__LIBELF; 363 } 364 365 if (obj->efile.obj_buf_sz > 0) { 366 /* 367 * obj_buf should have been validated by 368 * bpf_object__open_buffer(). 369 */ 370 obj->efile.elf = elf_memory(obj->efile.obj_buf, 371 obj->efile.obj_buf_sz); 372 } else { 373 obj->efile.fd = open(obj->path, O_RDONLY); 374 if (obj->efile.fd < 0) { 375 pr_warning("failed to open %s: %s\n", obj->path, 376 strerror(errno)); 377 return -errno; 378 } 379 380 obj->efile.elf = elf_begin(obj->efile.fd, 381 LIBBPF_ELF_C_READ_MMAP, 382 NULL); 383 } 384 385 if (!obj->efile.elf) { 386 pr_warning("failed to open %s as ELF file\n", 387 obj->path); 388 err = -LIBBPF_ERRNO__LIBELF; 389 goto errout; 390 } 391 392 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 393 pr_warning("failed to get EHDR from %s\n", 394 obj->path); 395 err = -LIBBPF_ERRNO__FORMAT; 396 goto errout; 397 } 398 ep = &obj->efile.ehdr; 399 400 if ((ep->e_type != ET_REL) || (ep->e_machine != 0)) { 401 pr_warning("%s is not an eBPF object file\n", 402 obj->path); 403 err = -LIBBPF_ERRNO__FORMAT; 404 goto errout; 405 } 406 407 return 0; 408 errout: 409 bpf_object__elf_finish(obj); 410 return err; 411 } 412 413 static int 414 bpf_object__check_endianness(struct bpf_object *obj) 415 { 416 static unsigned int const endian = 1; 417 418 switch (obj->efile.ehdr.e_ident[EI_DATA]) { 419 case ELFDATA2LSB: 420 /* We are big endian, BPF obj is little endian. */ 421 if (*(unsigned char const *)&endian != 1) 422 goto mismatch; 423 break; 424 425 case ELFDATA2MSB: 426 /* We are little endian, BPF obj is big endian. */ 427 if (*(unsigned char const *)&endian != 0) 428 goto mismatch; 429 break; 430 default: 431 return -LIBBPF_ERRNO__ENDIAN; 432 } 433 434 return 0; 435 436 mismatch: 437 pr_warning("Error: endianness mismatch.\n"); 438 return -LIBBPF_ERRNO__ENDIAN; 439 } 440 441 static int 442 bpf_object__init_license(struct bpf_object *obj, 443 void *data, size_t size) 444 { 445 memcpy(obj->license, data, 446 min(size, sizeof(obj->license) - 1)); 447 pr_debug("license of %s is %s\n", obj->path, obj->license); 448 return 0; 449 } 450 451 static int 452 bpf_object__init_kversion(struct bpf_object *obj, 453 void *data, size_t size) 454 { 455 u32 kver; 456 457 if (size != sizeof(kver)) { 458 pr_warning("invalid kver section in %s\n", obj->path); 459 return -LIBBPF_ERRNO__FORMAT; 460 } 461 memcpy(&kver, data, sizeof(kver)); 462 obj->kern_version = kver; 463 pr_debug("kernel version of %s is %x\n", obj->path, 464 obj->kern_version); 465 return 0; 466 } 467 468 static int 469 bpf_object__init_maps(struct bpf_object *obj, void *data, 470 size_t size) 471 { 472 if (size == 0) { 473 pr_debug("%s doesn't need map definition\n", 474 obj->path); 475 return 0; 476 } 477 478 obj->maps_buf = malloc(size); 479 if (!obj->maps_buf) { 480 pr_warning("malloc maps failed: %s\n", obj->path); 481 return -ENOMEM; 482 } 483 484 obj->maps_buf_sz = size; 485 memcpy(obj->maps_buf, data, size); 486 pr_debug("maps in %s: %ld bytes\n", obj->path, (long)size); 487 return 0; 488 } 489 490 static int bpf_object__elf_collect(struct bpf_object *obj) 491 { 492 Elf *elf = obj->efile.elf; 493 GElf_Ehdr *ep = &obj->efile.ehdr; 494 Elf_Scn *scn = NULL; 495 int idx = 0, err = 0; 496 497 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 498 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 499 pr_warning("failed to get e_shstrndx from %s\n", 500 obj->path); 501 return -LIBBPF_ERRNO__FORMAT; 502 } 503 504 while ((scn = elf_nextscn(elf, scn)) != NULL) { 505 char *name; 506 GElf_Shdr sh; 507 Elf_Data *data; 508 509 idx++; 510 if (gelf_getshdr(scn, &sh) != &sh) { 511 pr_warning("failed to get section header from %s\n", 512 obj->path); 513 err = -LIBBPF_ERRNO__FORMAT; 514 goto out; 515 } 516 517 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 518 if (!name) { 519 pr_warning("failed to get section name from %s\n", 520 obj->path); 521 err = -LIBBPF_ERRNO__FORMAT; 522 goto out; 523 } 524 525 data = elf_getdata(scn, 0); 526 if (!data) { 527 pr_warning("failed to get section data from %s(%s)\n", 528 name, obj->path); 529 err = -LIBBPF_ERRNO__FORMAT; 530 goto out; 531 } 532 pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n", 533 name, (unsigned long)data->d_size, 534 (int)sh.sh_link, (unsigned long)sh.sh_flags, 535 (int)sh.sh_type); 536 537 if (strcmp(name, "license") == 0) 538 err = bpf_object__init_license(obj, 539 data->d_buf, 540 data->d_size); 541 else if (strcmp(name, "version") == 0) 542 err = bpf_object__init_kversion(obj, 543 data->d_buf, 544 data->d_size); 545 else if (strcmp(name, "maps") == 0) 546 err = bpf_object__init_maps(obj, data->d_buf, 547 data->d_size); 548 else if (sh.sh_type == SHT_SYMTAB) { 549 if (obj->efile.symbols) { 550 pr_warning("bpf: multiple SYMTAB in %s\n", 551 obj->path); 552 err = -LIBBPF_ERRNO__FORMAT; 553 } else 554 obj->efile.symbols = data; 555 } else if ((sh.sh_type == SHT_PROGBITS) && 556 (sh.sh_flags & SHF_EXECINSTR) && 557 (data->d_size > 0)) { 558 err = bpf_object__add_program(obj, data->d_buf, 559 data->d_size, name, idx); 560 if (err) { 561 char errmsg[STRERR_BUFSIZE]; 562 563 strerror_r(-err, errmsg, sizeof(errmsg)); 564 pr_warning("failed to alloc program %s (%s): %s", 565 name, obj->path, errmsg); 566 } 567 } else if (sh.sh_type == SHT_REL) { 568 void *reloc = obj->efile.reloc; 569 int nr_reloc = obj->efile.nr_reloc + 1; 570 571 reloc = realloc(reloc, 572 sizeof(*obj->efile.reloc) * nr_reloc); 573 if (!reloc) { 574 pr_warning("realloc failed\n"); 575 err = -ENOMEM; 576 } else { 577 int n = nr_reloc - 1; 578 579 obj->efile.reloc = reloc; 580 obj->efile.nr_reloc = nr_reloc; 581 582 obj->efile.reloc[n].shdr = sh; 583 obj->efile.reloc[n].data = data; 584 } 585 } 586 if (err) 587 goto out; 588 } 589 out: 590 return err; 591 } 592 593 static struct bpf_program * 594 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 595 { 596 struct bpf_program *prog; 597 size_t i; 598 599 for (i = 0; i < obj->nr_programs; i++) { 600 prog = &obj->programs[i]; 601 if (prog->idx == idx) 602 return prog; 603 } 604 return NULL; 605 } 606 607 static int 608 bpf_program__collect_reloc(struct bpf_program *prog, 609 size_t nr_maps, GElf_Shdr *shdr, 610 Elf_Data *data, Elf_Data *symbols) 611 { 612 int i, nrels; 613 614 pr_debug("collecting relocating info for: '%s'\n", 615 prog->section_name); 616 nrels = shdr->sh_size / shdr->sh_entsize; 617 618 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 619 if (!prog->reloc_desc) { 620 pr_warning("failed to alloc memory in relocation\n"); 621 return -ENOMEM; 622 } 623 prog->nr_reloc = nrels; 624 625 for (i = 0; i < nrels; i++) { 626 GElf_Sym sym; 627 GElf_Rel rel; 628 unsigned int insn_idx; 629 struct bpf_insn *insns = prog->insns; 630 size_t map_idx; 631 632 if (!gelf_getrel(data, i, &rel)) { 633 pr_warning("relocation: failed to get %d reloc\n", i); 634 return -LIBBPF_ERRNO__FORMAT; 635 } 636 637 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 638 pr_debug("relocation: insn_idx=%u\n", insn_idx); 639 640 if (!gelf_getsym(symbols, 641 GELF_R_SYM(rel.r_info), 642 &sym)) { 643 pr_warning("relocation: symbol %"PRIx64" not found\n", 644 GELF_R_SYM(rel.r_info)); 645 return -LIBBPF_ERRNO__FORMAT; 646 } 647 648 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 649 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 650 insn_idx, insns[insn_idx].code); 651 return -LIBBPF_ERRNO__RELOC; 652 } 653 654 map_idx = sym.st_value / sizeof(struct bpf_map_def); 655 if (map_idx >= nr_maps) { 656 pr_warning("bpf relocation: map_idx %d large than %d\n", 657 (int)map_idx, (int)nr_maps - 1); 658 return -LIBBPF_ERRNO__RELOC; 659 } 660 661 prog->reloc_desc[i].insn_idx = insn_idx; 662 prog->reloc_desc[i].map_idx = map_idx; 663 } 664 return 0; 665 } 666 667 static int 668 bpf_object__create_maps(struct bpf_object *obj) 669 { 670 unsigned int i; 671 size_t nr_maps; 672 int *pfd; 673 674 nr_maps = obj->maps_buf_sz / sizeof(struct bpf_map_def); 675 if (!obj->maps_buf || !nr_maps) { 676 pr_debug("don't need create maps for %s\n", 677 obj->path); 678 return 0; 679 } 680 681 obj->map_fds = malloc(sizeof(int) * nr_maps); 682 if (!obj->map_fds) { 683 pr_warning("realloc perf_bpf_map_fds failed\n"); 684 return -ENOMEM; 685 } 686 obj->nr_map_fds = nr_maps; 687 688 /* fill all fd with -1 */ 689 memset(obj->map_fds, -1, sizeof(int) * nr_maps); 690 691 pfd = obj->map_fds; 692 for (i = 0; i < nr_maps; i++) { 693 struct bpf_map_def def; 694 695 def = *(struct bpf_map_def *)(obj->maps_buf + 696 i * sizeof(struct bpf_map_def)); 697 698 *pfd = bpf_create_map(def.type, 699 def.key_size, 700 def.value_size, 701 def.max_entries); 702 if (*pfd < 0) { 703 size_t j; 704 int err = *pfd; 705 706 pr_warning("failed to create map: %s\n", 707 strerror(errno)); 708 for (j = 0; j < i; j++) 709 zclose(obj->map_fds[j]); 710 obj->nr_map_fds = 0; 711 zfree(&obj->map_fds); 712 return err; 713 } 714 pr_debug("create map: fd=%d\n", *pfd); 715 pfd++; 716 } 717 718 zfree(&obj->maps_buf); 719 obj->maps_buf_sz = 0; 720 return 0; 721 } 722 723 static int 724 bpf_program__relocate(struct bpf_program *prog, int *map_fds) 725 { 726 int i; 727 728 if (!prog || !prog->reloc_desc) 729 return 0; 730 731 for (i = 0; i < prog->nr_reloc; i++) { 732 int insn_idx, map_idx; 733 struct bpf_insn *insns = prog->insns; 734 735 insn_idx = prog->reloc_desc[i].insn_idx; 736 map_idx = prog->reloc_desc[i].map_idx; 737 738 if (insn_idx >= (int)prog->insns_cnt) { 739 pr_warning("relocation out of range: '%s'\n", 740 prog->section_name); 741 return -LIBBPF_ERRNO__RELOC; 742 } 743 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 744 insns[insn_idx].imm = map_fds[map_idx]; 745 } 746 747 zfree(&prog->reloc_desc); 748 prog->nr_reloc = 0; 749 return 0; 750 } 751 752 753 static int 754 bpf_object__relocate(struct bpf_object *obj) 755 { 756 struct bpf_program *prog; 757 size_t i; 758 int err; 759 760 for (i = 0; i < obj->nr_programs; i++) { 761 prog = &obj->programs[i]; 762 763 err = bpf_program__relocate(prog, obj->map_fds); 764 if (err) { 765 pr_warning("failed to relocate '%s'\n", 766 prog->section_name); 767 return err; 768 } 769 } 770 return 0; 771 } 772 773 static int bpf_object__collect_reloc(struct bpf_object *obj) 774 { 775 int i, err; 776 777 if (!obj_elf_valid(obj)) { 778 pr_warning("Internal error: elf object is closed\n"); 779 return -LIBBPF_ERRNO__INTERNAL; 780 } 781 782 for (i = 0; i < obj->efile.nr_reloc; i++) { 783 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 784 Elf_Data *data = obj->efile.reloc[i].data; 785 int idx = shdr->sh_info; 786 struct bpf_program *prog; 787 size_t nr_maps = obj->maps_buf_sz / 788 sizeof(struct bpf_map_def); 789 790 if (shdr->sh_type != SHT_REL) { 791 pr_warning("internal error at %d\n", __LINE__); 792 return -LIBBPF_ERRNO__INTERNAL; 793 } 794 795 prog = bpf_object__find_prog_by_idx(obj, idx); 796 if (!prog) { 797 pr_warning("relocation failed: no %d section\n", 798 idx); 799 return -LIBBPF_ERRNO__RELOC; 800 } 801 802 err = bpf_program__collect_reloc(prog, nr_maps, 803 shdr, data, 804 obj->efile.symbols); 805 if (err) 806 return err; 807 } 808 return 0; 809 } 810 811 static int 812 load_program(struct bpf_insn *insns, int insns_cnt, 813 char *license, u32 kern_version, int *pfd) 814 { 815 int ret; 816 char *log_buf; 817 818 if (!insns || !insns_cnt) 819 return -EINVAL; 820 821 log_buf = malloc(BPF_LOG_BUF_SIZE); 822 if (!log_buf) 823 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 824 825 ret = bpf_load_program(BPF_PROG_TYPE_KPROBE, insns, 826 insns_cnt, license, kern_version, 827 log_buf, BPF_LOG_BUF_SIZE); 828 829 if (ret >= 0) { 830 *pfd = ret; 831 ret = 0; 832 goto out; 833 } 834 835 ret = -LIBBPF_ERRNO__LOAD; 836 pr_warning("load bpf program failed: %s\n", strerror(errno)); 837 838 if (log_buf && log_buf[0] != '\0') { 839 ret = -LIBBPF_ERRNO__VERIFY; 840 pr_warning("-- BEGIN DUMP LOG ---\n"); 841 pr_warning("\n%s\n", log_buf); 842 pr_warning("-- END LOG --\n"); 843 } else { 844 if (insns_cnt >= BPF_MAXINSNS) { 845 pr_warning("Program too large (%d insns), at most %d insns\n", 846 insns_cnt, BPF_MAXINSNS); 847 ret = -LIBBPF_ERRNO__PROG2BIG; 848 } else if (log_buf) { 849 pr_warning("log buffer is empty\n"); 850 ret = -LIBBPF_ERRNO__KVER; 851 } 852 } 853 854 out: 855 free(log_buf); 856 return ret; 857 } 858 859 static int 860 bpf_program__load(struct bpf_program *prog, 861 char *license, u32 kern_version) 862 { 863 int err, fd; 864 865 err = load_program(prog->insns, prog->insns_cnt, 866 license, kern_version, &fd); 867 if (!err) 868 prog->fd = fd; 869 870 if (err) 871 pr_warning("failed to load program '%s'\n", 872 prog->section_name); 873 zfree(&prog->insns); 874 prog->insns_cnt = 0; 875 return err; 876 } 877 878 static int 879 bpf_object__load_progs(struct bpf_object *obj) 880 { 881 size_t i; 882 int err; 883 884 for (i = 0; i < obj->nr_programs; i++) { 885 err = bpf_program__load(&obj->programs[i], 886 obj->license, 887 obj->kern_version); 888 if (err) 889 return err; 890 } 891 return 0; 892 } 893 894 static int bpf_object__validate(struct bpf_object *obj) 895 { 896 if (obj->kern_version == 0) { 897 pr_warning("%s doesn't provide kernel version\n", 898 obj->path); 899 return -LIBBPF_ERRNO__KVERSION; 900 } 901 return 0; 902 } 903 904 static struct bpf_object * 905 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) 906 { 907 struct bpf_object *obj; 908 int err; 909 910 if (elf_version(EV_CURRENT) == EV_NONE) { 911 pr_warning("failed to init libelf for %s\n", path); 912 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 913 } 914 915 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 916 if (IS_ERR(obj)) 917 return obj; 918 919 CHECK_ERR(bpf_object__elf_init(obj), err, out); 920 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 921 CHECK_ERR(bpf_object__elf_collect(obj), err, out); 922 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 923 CHECK_ERR(bpf_object__validate(obj), err, out); 924 925 bpf_object__elf_finish(obj); 926 return obj; 927 out: 928 bpf_object__close(obj); 929 return ERR_PTR(err); 930 } 931 932 struct bpf_object *bpf_object__open(const char *path) 933 { 934 /* param validation */ 935 if (!path) 936 return NULL; 937 938 pr_debug("loading %s\n", path); 939 940 return __bpf_object__open(path, NULL, 0); 941 } 942 943 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 944 size_t obj_buf_sz, 945 const char *name) 946 { 947 char tmp_name[64]; 948 949 /* param validation */ 950 if (!obj_buf || obj_buf_sz <= 0) 951 return NULL; 952 953 if (!name) { 954 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 955 (unsigned long)obj_buf, 956 (unsigned long)obj_buf_sz); 957 tmp_name[sizeof(tmp_name) - 1] = '\0'; 958 name = tmp_name; 959 } 960 pr_debug("loading object '%s' from buffer\n", 961 name); 962 963 return __bpf_object__open(name, obj_buf, obj_buf_sz); 964 } 965 966 int bpf_object__unload(struct bpf_object *obj) 967 { 968 size_t i; 969 970 if (!obj) 971 return -EINVAL; 972 973 for (i = 0; i < obj->nr_map_fds; i++) 974 zclose(obj->map_fds[i]); 975 zfree(&obj->map_fds); 976 obj->nr_map_fds = 0; 977 978 for (i = 0; i < obj->nr_programs; i++) 979 bpf_program__unload(&obj->programs[i]); 980 981 return 0; 982 } 983 984 int bpf_object__load(struct bpf_object *obj) 985 { 986 int err; 987 988 if (!obj) 989 return -EINVAL; 990 991 if (obj->loaded) { 992 pr_warning("object should not be loaded twice\n"); 993 return -EINVAL; 994 } 995 996 obj->loaded = true; 997 998 CHECK_ERR(bpf_object__create_maps(obj), err, out); 999 CHECK_ERR(bpf_object__relocate(obj), err, out); 1000 CHECK_ERR(bpf_object__load_progs(obj), err, out); 1001 1002 return 0; 1003 out: 1004 bpf_object__unload(obj); 1005 pr_warning("failed to load object '%s'\n", obj->path); 1006 return err; 1007 } 1008 1009 void bpf_object__close(struct bpf_object *obj) 1010 { 1011 size_t i; 1012 1013 if (!obj) 1014 return; 1015 1016 bpf_object__elf_finish(obj); 1017 bpf_object__unload(obj); 1018 1019 zfree(&obj->maps_buf); 1020 1021 if (obj->programs && obj->nr_programs) { 1022 for (i = 0; i < obj->nr_programs; i++) 1023 bpf_program__exit(&obj->programs[i]); 1024 } 1025 zfree(&obj->programs); 1026 1027 list_del(&obj->list); 1028 free(obj); 1029 } 1030 1031 struct bpf_object * 1032 bpf_object__next(struct bpf_object *prev) 1033 { 1034 struct bpf_object *next; 1035 1036 if (!prev) 1037 next = list_first_entry(&bpf_objects_list, 1038 struct bpf_object, 1039 list); 1040 else 1041 next = list_next_entry(prev, list); 1042 1043 /* Empty list is noticed here so don't need checking on entry. */ 1044 if (&next->list == &bpf_objects_list) 1045 return NULL; 1046 1047 return next; 1048 } 1049 1050 const char * 1051 bpf_object__get_name(struct bpf_object *obj) 1052 { 1053 if (!obj) 1054 return ERR_PTR(-EINVAL); 1055 return obj->path; 1056 } 1057 1058 unsigned int 1059 bpf_object__get_kversion(struct bpf_object *obj) 1060 { 1061 if (!obj) 1062 return 0; 1063 return obj->kern_version; 1064 } 1065 1066 struct bpf_program * 1067 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 1068 { 1069 size_t idx; 1070 1071 if (!obj->programs) 1072 return NULL; 1073 /* First handler */ 1074 if (prev == NULL) 1075 return &obj->programs[0]; 1076 1077 if (prev->obj != obj) { 1078 pr_warning("error: program handler doesn't match object\n"); 1079 return NULL; 1080 } 1081 1082 idx = (prev - obj->programs) + 1; 1083 if (idx >= obj->nr_programs) 1084 return NULL; 1085 return &obj->programs[idx]; 1086 } 1087 1088 int bpf_program__set_private(struct bpf_program *prog, 1089 void *priv, 1090 bpf_program_clear_priv_t clear_priv) 1091 { 1092 if (prog->priv && prog->clear_priv) 1093 prog->clear_priv(prog, prog->priv); 1094 1095 prog->priv = priv; 1096 prog->clear_priv = clear_priv; 1097 return 0; 1098 } 1099 1100 int bpf_program__get_private(struct bpf_program *prog, void **ppriv) 1101 { 1102 *ppriv = prog->priv; 1103 return 0; 1104 } 1105 1106 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 1107 { 1108 const char *title; 1109 1110 title = prog->section_name; 1111 if (needs_copy) { 1112 title = strdup(title); 1113 if (!title) { 1114 pr_warning("failed to strdup program title\n"); 1115 return ERR_PTR(-ENOMEM); 1116 } 1117 } 1118 1119 return title; 1120 } 1121 1122 int bpf_program__fd(struct bpf_program *prog) 1123 { 1124 return prog->fd; 1125 } 1126