1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 */ 4 #ifndef _LINUX_BPF_H 5 #define _LINUX_BPF_H 1 6 7 #include <uapi/linux/bpf.h> 8 #include <uapi/linux/filter.h> 9 10 #include <linux/workqueue.h> 11 #include <linux/file.h> 12 #include <linux/percpu.h> 13 #include <linux/err.h> 14 #include <linux/rbtree_latch.h> 15 #include <linux/numa.h> 16 #include <linux/mm_types.h> 17 #include <linux/wait.h> 18 #include <linux/refcount.h> 19 #include <linux/mutex.h> 20 #include <linux/module.h> 21 #include <linux/kallsyms.h> 22 #include <linux/capability.h> 23 #include <linux/sched/mm.h> 24 #include <linux/slab.h> 25 #include <linux/percpu-refcount.h> 26 #include <linux/stddef.h> 27 #include <linux/bpfptr.h> 28 #include <linux/btf.h> 29 #include <linux/rcupdate_trace.h> 30 #include <linux/static_call.h> 31 32 struct bpf_verifier_env; 33 struct bpf_verifier_log; 34 struct perf_event; 35 struct bpf_prog; 36 struct bpf_prog_aux; 37 struct bpf_map; 38 struct sock; 39 struct seq_file; 40 struct btf; 41 struct btf_type; 42 struct exception_table_entry; 43 struct seq_operations; 44 struct bpf_iter_aux_info; 45 struct bpf_local_storage; 46 struct bpf_local_storage_map; 47 struct kobject; 48 struct mem_cgroup; 49 struct module; 50 struct bpf_func_state; 51 struct ftrace_ops; 52 struct cgroup; 53 54 extern struct idr btf_idr; 55 extern spinlock_t btf_idr_lock; 56 extern struct kobject *btf_kobj; 57 extern struct bpf_mem_alloc bpf_global_ma; 58 extern bool bpf_global_ma_set; 59 60 typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64); 61 typedef int (*bpf_iter_init_seq_priv_t)(void *private_data, 62 struct bpf_iter_aux_info *aux); 63 typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data); 64 typedef unsigned int (*bpf_func_t)(const void *, 65 const struct bpf_insn *); 66 struct bpf_iter_seq_info { 67 const struct seq_operations *seq_ops; 68 bpf_iter_init_seq_priv_t init_seq_private; 69 bpf_iter_fini_seq_priv_t fini_seq_private; 70 u32 seq_priv_size; 71 }; 72 73 /* map is generic key/value storage optionally accessible by eBPF programs */ 74 struct bpf_map_ops { 75 /* funcs callable from userspace (via syscall) */ 76 int (*map_alloc_check)(union bpf_attr *attr); 77 struct bpf_map *(*map_alloc)(union bpf_attr *attr); 78 void (*map_release)(struct bpf_map *map, struct file *map_file); 79 void (*map_free)(struct bpf_map *map); 80 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); 81 void (*map_release_uref)(struct bpf_map *map); 82 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); 83 int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr, 84 union bpf_attr __user *uattr); 85 int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key, 86 void *value, u64 flags); 87 int (*map_lookup_and_delete_batch)(struct bpf_map *map, 88 const union bpf_attr *attr, 89 union bpf_attr __user *uattr); 90 int (*map_update_batch)(struct bpf_map *map, struct file *map_file, 91 const union bpf_attr *attr, 92 union bpf_attr __user *uattr); 93 int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr, 94 union bpf_attr __user *uattr); 95 96 /* funcs callable from userspace and from eBPF programs */ 97 void *(*map_lookup_elem)(struct bpf_map *map, void *key); 98 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); 99 int (*map_delete_elem)(struct bpf_map *map, void *key); 100 int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); 101 int (*map_pop_elem)(struct bpf_map *map, void *value); 102 int (*map_peek_elem)(struct bpf_map *map, void *value); 103 void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu); 104 105 /* funcs called by prog_array and perf_event_array map */ 106 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, 107 int fd); 108 void (*map_fd_put_ptr)(void *ptr); 109 int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); 110 u32 (*map_fd_sys_lookup_elem)(void *ptr); 111 void (*map_seq_show_elem)(struct bpf_map *map, void *key, 112 struct seq_file *m); 113 int (*map_check_btf)(const struct bpf_map *map, 114 const struct btf *btf, 115 const struct btf_type *key_type, 116 const struct btf_type *value_type); 117 118 /* Prog poke tracking helpers. */ 119 int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux); 120 void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux); 121 void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old, 122 struct bpf_prog *new); 123 124 /* Direct value access helpers. */ 125 int (*map_direct_value_addr)(const struct bpf_map *map, 126 u64 *imm, u32 off); 127 int (*map_direct_value_meta)(const struct bpf_map *map, 128 u64 imm, u32 *off); 129 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); 130 __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, 131 struct poll_table_struct *pts); 132 133 /* Functions called by bpf_local_storage maps */ 134 int (*map_local_storage_charge)(struct bpf_local_storage_map *smap, 135 void *owner, u32 size); 136 void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap, 137 void *owner, u32 size); 138 struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner); 139 140 /* Misc helpers.*/ 141 int (*map_redirect)(struct bpf_map *map, u64 key, u64 flags); 142 143 /* map_meta_equal must be implemented for maps that can be 144 * used as an inner map. It is a runtime check to ensure 145 * an inner map can be inserted to an outer map. 146 * 147 * Some properties of the inner map has been used during the 148 * verification time. When inserting an inner map at the runtime, 149 * map_meta_equal has to ensure the inserting map has the same 150 * properties that the verifier has used earlier. 151 */ 152 bool (*map_meta_equal)(const struct bpf_map *meta0, 153 const struct bpf_map *meta1); 154 155 156 int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env, 157 struct bpf_func_state *caller, 158 struct bpf_func_state *callee); 159 int (*map_for_each_callback)(struct bpf_map *map, 160 bpf_callback_t callback_fn, 161 void *callback_ctx, u64 flags); 162 163 /* BTF id of struct allocated by map_alloc */ 164 int *map_btf_id; 165 166 /* bpf_iter info used to open a seq_file */ 167 const struct bpf_iter_seq_info *iter_seq_info; 168 }; 169 170 enum { 171 /* Support at most 10 fields in a BTF type */ 172 BTF_FIELDS_MAX = 10, 173 }; 174 175 enum btf_field_type { 176 BPF_SPIN_LOCK = (1 << 0), 177 BPF_TIMER = (1 << 1), 178 BPF_KPTR_UNREF = (1 << 2), 179 BPF_KPTR_REF = (1 << 3), 180 BPF_KPTR = BPF_KPTR_UNREF | BPF_KPTR_REF, 181 BPF_LIST_HEAD = (1 << 4), 182 BPF_LIST_NODE = (1 << 5), 183 }; 184 185 struct btf_field_kptr { 186 struct btf *btf; 187 struct module *module; 188 btf_dtor_kfunc_t dtor; 189 u32 btf_id; 190 }; 191 192 struct btf_field_list_head { 193 struct btf *btf; 194 u32 value_btf_id; 195 u32 node_offset; 196 struct btf_record *value_rec; 197 }; 198 199 struct btf_field { 200 u32 offset; 201 enum btf_field_type type; 202 union { 203 struct btf_field_kptr kptr; 204 struct btf_field_list_head list_head; 205 }; 206 }; 207 208 struct btf_record { 209 u32 cnt; 210 u32 field_mask; 211 int spin_lock_off; 212 int timer_off; 213 struct btf_field fields[]; 214 }; 215 216 struct btf_field_offs { 217 u32 cnt; 218 u32 field_off[BTF_FIELDS_MAX]; 219 u8 field_sz[BTF_FIELDS_MAX]; 220 }; 221 222 struct bpf_map { 223 /* The first two cachelines with read-mostly members of which some 224 * are also accessed in fast-path (e.g. ops, max_entries). 225 */ 226 const struct bpf_map_ops *ops ____cacheline_aligned; 227 struct bpf_map *inner_map_meta; 228 #ifdef CONFIG_SECURITY 229 void *security; 230 #endif 231 enum bpf_map_type map_type; 232 u32 key_size; 233 u32 value_size; 234 u32 max_entries; 235 u64 map_extra; /* any per-map-type extra fields */ 236 u32 map_flags; 237 u32 id; 238 struct btf_record *record; 239 int numa_node; 240 u32 btf_key_type_id; 241 u32 btf_value_type_id; 242 u32 btf_vmlinux_value_type_id; 243 struct btf *btf; 244 #ifdef CONFIG_MEMCG_KMEM 245 struct obj_cgroup *objcg; 246 #endif 247 char name[BPF_OBJ_NAME_LEN]; 248 struct btf_field_offs *field_offs; 249 /* The 3rd and 4th cacheline with misc members to avoid false sharing 250 * particularly with refcounting. 251 */ 252 atomic64_t refcnt ____cacheline_aligned; 253 atomic64_t usercnt; 254 struct work_struct work; 255 struct mutex freeze_mutex; 256 atomic64_t writecnt; 257 /* 'Ownership' of program-containing map is claimed by the first program 258 * that is going to use this map or by the first program which FD is 259 * stored in the map to make sure that all callers and callees have the 260 * same prog type, JITed flag and xdp_has_frags flag. 261 */ 262 struct { 263 spinlock_t lock; 264 enum bpf_prog_type type; 265 bool jited; 266 bool xdp_has_frags; 267 } owner; 268 bool bypass_spec_v1; 269 bool frozen; /* write-once; write-protected by freeze_mutex */ 270 }; 271 272 static inline const char *btf_field_type_name(enum btf_field_type type) 273 { 274 switch (type) { 275 case BPF_SPIN_LOCK: 276 return "bpf_spin_lock"; 277 case BPF_TIMER: 278 return "bpf_timer"; 279 case BPF_KPTR_UNREF: 280 case BPF_KPTR_REF: 281 return "kptr"; 282 case BPF_LIST_HEAD: 283 return "bpf_list_head"; 284 case BPF_LIST_NODE: 285 return "bpf_list_node"; 286 default: 287 WARN_ON_ONCE(1); 288 return "unknown"; 289 } 290 } 291 292 static inline u32 btf_field_type_size(enum btf_field_type type) 293 { 294 switch (type) { 295 case BPF_SPIN_LOCK: 296 return sizeof(struct bpf_spin_lock); 297 case BPF_TIMER: 298 return sizeof(struct bpf_timer); 299 case BPF_KPTR_UNREF: 300 case BPF_KPTR_REF: 301 return sizeof(u64); 302 case BPF_LIST_HEAD: 303 return sizeof(struct bpf_list_head); 304 case BPF_LIST_NODE: 305 return sizeof(struct bpf_list_node); 306 default: 307 WARN_ON_ONCE(1); 308 return 0; 309 } 310 } 311 312 static inline u32 btf_field_type_align(enum btf_field_type type) 313 { 314 switch (type) { 315 case BPF_SPIN_LOCK: 316 return __alignof__(struct bpf_spin_lock); 317 case BPF_TIMER: 318 return __alignof__(struct bpf_timer); 319 case BPF_KPTR_UNREF: 320 case BPF_KPTR_REF: 321 return __alignof__(u64); 322 case BPF_LIST_HEAD: 323 return __alignof__(struct bpf_list_head); 324 case BPF_LIST_NODE: 325 return __alignof__(struct bpf_list_node); 326 default: 327 WARN_ON_ONCE(1); 328 return 0; 329 } 330 } 331 332 static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type) 333 { 334 if (IS_ERR_OR_NULL(rec)) 335 return false; 336 return rec->field_mask & type; 337 } 338 339 static inline void bpf_obj_init(const struct btf_field_offs *foffs, void *obj) 340 { 341 int i; 342 343 if (!foffs) 344 return; 345 for (i = 0; i < foffs->cnt; i++) 346 memset(obj + foffs->field_off[i], 0, foffs->field_sz[i]); 347 } 348 349 static inline void check_and_init_map_value(struct bpf_map *map, void *dst) 350 { 351 bpf_obj_init(map->field_offs, dst); 352 } 353 354 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and 355 * forced to use 'long' read/writes to try to atomically copy long counters. 356 * Best-effort only. No barriers here, since it _will_ race with concurrent 357 * updates from BPF programs. Called from bpf syscall and mostly used with 358 * size 8 or 16 bytes, so ask compiler to inline it. 359 */ 360 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) 361 { 362 const long *lsrc = src; 363 long *ldst = dst; 364 365 size /= sizeof(long); 366 while (size--) 367 *ldst++ = *lsrc++; 368 } 369 370 /* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */ 371 static inline void bpf_obj_memcpy(struct btf_field_offs *foffs, 372 void *dst, void *src, u32 size, 373 bool long_memcpy) 374 { 375 u32 curr_off = 0; 376 int i; 377 378 if (likely(!foffs)) { 379 if (long_memcpy) 380 bpf_long_memcpy(dst, src, round_up(size, 8)); 381 else 382 memcpy(dst, src, size); 383 return; 384 } 385 386 for (i = 0; i < foffs->cnt; i++) { 387 u32 next_off = foffs->field_off[i]; 388 u32 sz = next_off - curr_off; 389 390 memcpy(dst + curr_off, src + curr_off, sz); 391 curr_off += foffs->field_sz[i] + sz; 392 } 393 memcpy(dst + curr_off, src + curr_off, size - curr_off); 394 } 395 396 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) 397 { 398 bpf_obj_memcpy(map->field_offs, dst, src, map->value_size, false); 399 } 400 401 static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) 402 { 403 bpf_obj_memcpy(map->field_offs, dst, src, map->value_size, true); 404 } 405 406 static inline void bpf_obj_memzero(struct btf_field_offs *foffs, void *dst, u32 size) 407 { 408 u32 curr_off = 0; 409 int i; 410 411 if (likely(!foffs)) { 412 memset(dst, 0, size); 413 return; 414 } 415 416 for (i = 0; i < foffs->cnt; i++) { 417 u32 next_off = foffs->field_off[i]; 418 u32 sz = next_off - curr_off; 419 420 memset(dst + curr_off, 0, sz); 421 curr_off += foffs->field_sz[i] + sz; 422 } 423 memset(dst + curr_off, 0, size - curr_off); 424 } 425 426 static inline void zero_map_value(struct bpf_map *map, void *dst) 427 { 428 bpf_obj_memzero(map->field_offs, dst, map->value_size); 429 } 430 431 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, 432 bool lock_src); 433 void bpf_timer_cancel_and_free(void *timer); 434 void bpf_list_head_free(const struct btf_field *field, void *list_head, 435 struct bpf_spin_lock *spin_lock); 436 437 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size); 438 439 struct bpf_offload_dev; 440 struct bpf_offloaded_map; 441 442 struct bpf_map_dev_ops { 443 int (*map_get_next_key)(struct bpf_offloaded_map *map, 444 void *key, void *next_key); 445 int (*map_lookup_elem)(struct bpf_offloaded_map *map, 446 void *key, void *value); 447 int (*map_update_elem)(struct bpf_offloaded_map *map, 448 void *key, void *value, u64 flags); 449 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); 450 }; 451 452 struct bpf_offloaded_map { 453 struct bpf_map map; 454 struct net_device *netdev; 455 const struct bpf_map_dev_ops *dev_ops; 456 void *dev_priv; 457 struct list_head offloads; 458 }; 459 460 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) 461 { 462 return container_of(map, struct bpf_offloaded_map, map); 463 } 464 465 static inline bool bpf_map_offload_neutral(const struct bpf_map *map) 466 { 467 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 468 } 469 470 static inline bool bpf_map_support_seq_show(const struct bpf_map *map) 471 { 472 return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) && 473 map->ops->map_seq_show_elem; 474 } 475 476 int map_check_no_btf(const struct bpf_map *map, 477 const struct btf *btf, 478 const struct btf_type *key_type, 479 const struct btf_type *value_type); 480 481 bool bpf_map_meta_equal(const struct bpf_map *meta0, 482 const struct bpf_map *meta1); 483 484 extern const struct bpf_map_ops bpf_map_offload_ops; 485 486 /* bpf_type_flag contains a set of flags that are applicable to the values of 487 * arg_type, ret_type and reg_type. For example, a pointer value may be null, 488 * or a memory is read-only. We classify types into two categories: base types 489 * and extended types. Extended types are base types combined with a type flag. 490 * 491 * Currently there are no more than 32 base types in arg_type, ret_type and 492 * reg_types. 493 */ 494 #define BPF_BASE_TYPE_BITS 8 495 496 enum bpf_type_flag { 497 /* PTR may be NULL. */ 498 PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS), 499 500 /* MEM is read-only. When applied on bpf_arg, it indicates the arg is 501 * compatible with both mutable and immutable memory. 502 */ 503 MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS), 504 505 /* MEM points to BPF ring buffer reservation. */ 506 MEM_RINGBUF = BIT(2 + BPF_BASE_TYPE_BITS), 507 508 /* MEM is in user address space. */ 509 MEM_USER = BIT(3 + BPF_BASE_TYPE_BITS), 510 511 /* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged 512 * with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In 513 * order to drop this tag, it must be passed into bpf_per_cpu_ptr() 514 * or bpf_this_cpu_ptr(), which will return the pointer corresponding 515 * to the specified cpu. 516 */ 517 MEM_PERCPU = BIT(4 + BPF_BASE_TYPE_BITS), 518 519 /* Indicates that the argument will be released. */ 520 OBJ_RELEASE = BIT(5 + BPF_BASE_TYPE_BITS), 521 522 /* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark 523 * unreferenced and referenced kptr loaded from map value using a load 524 * instruction, so that they can only be dereferenced but not escape the 525 * BPF program into the kernel (i.e. cannot be passed as arguments to 526 * kfunc or bpf helpers). 527 */ 528 PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS), 529 530 MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS), 531 532 /* DYNPTR points to memory local to the bpf program. */ 533 DYNPTR_TYPE_LOCAL = BIT(8 + BPF_BASE_TYPE_BITS), 534 535 /* DYNPTR points to a kernel-produced ringbuf record. */ 536 DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS), 537 538 /* Size is known at compile time. */ 539 MEM_FIXED_SIZE = BIT(10 + BPF_BASE_TYPE_BITS), 540 541 /* MEM is of an allocated object of type in program BTF. This is used to 542 * tag PTR_TO_BTF_ID allocated using bpf_obj_new. 543 */ 544 MEM_ALLOC = BIT(11 + BPF_BASE_TYPE_BITS), 545 546 /* PTR was passed from the kernel in a trusted context, and may be 547 * passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions. 548 * Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above. 549 * PTR_UNTRUSTED refers to a kptr that was read directly from a map 550 * without invoking bpf_kptr_xchg(). What we really need to know is 551 * whether a pointer is safe to pass to a kfunc or BPF helper function. 552 * While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF 553 * helpers, they do not cover all possible instances of unsafe 554 * pointers. For example, a pointer that was obtained from walking a 555 * struct will _not_ get the PTR_UNTRUSTED type modifier, despite the 556 * fact that it may be NULL, invalid, etc. This is due to backwards 557 * compatibility requirements, as this was the behavior that was first 558 * introduced when kptrs were added. The behavior is now considered 559 * deprecated, and PTR_UNTRUSTED will eventually be removed. 560 * 561 * PTR_TRUSTED, on the other hand, is a pointer that the kernel 562 * guarantees to be valid and safe to pass to kfuncs and BPF helpers. 563 * For example, pointers passed to tracepoint arguments are considered 564 * PTR_TRUSTED, as are pointers that are passed to struct_ops 565 * callbacks. As alluded to above, pointers that are obtained from 566 * walking PTR_TRUSTED pointers are _not_ trusted. For example, if a 567 * struct task_struct *task is PTR_TRUSTED, then accessing 568 * task->last_wakee will lose the PTR_TRUSTED modifier when it's stored 569 * in a BPF register. Similarly, pointers passed to certain programs 570 * types such as kretprobes are not guaranteed to be valid, as they may 571 * for example contain an object that was recently freed. 572 */ 573 PTR_TRUSTED = BIT(12 + BPF_BASE_TYPE_BITS), 574 575 /* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */ 576 MEM_RCU = BIT(13 + BPF_BASE_TYPE_BITS), 577 578 __BPF_TYPE_FLAG_MAX, 579 __BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1, 580 }; 581 582 #define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF) 583 584 /* Max number of base types. */ 585 #define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS) 586 587 /* Max number of all types. */ 588 #define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1)) 589 590 /* function argument constraints */ 591 enum bpf_arg_type { 592 ARG_DONTCARE = 0, /* unused argument in helper function */ 593 594 /* the following constraints used to prototype 595 * bpf_map_lookup/update/delete_elem() functions 596 */ 597 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ 598 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ 599 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ 600 601 /* Used to prototype bpf_memcmp() and other functions that access data 602 * on eBPF program stack 603 */ 604 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ 605 606 ARG_CONST_SIZE, /* number of bytes accessed from memory */ 607 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ 608 609 ARG_PTR_TO_CTX, /* pointer to context */ 610 ARG_ANYTHING, /* any (initialized) argument is ok */ 611 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ 612 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ 613 ARG_PTR_TO_INT, /* pointer to int */ 614 ARG_PTR_TO_LONG, /* pointer to long */ 615 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ 616 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */ 617 ARG_PTR_TO_RINGBUF_MEM, /* pointer to dynamically reserved ringbuf memory */ 618 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */ 619 ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */ 620 ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */ 621 ARG_PTR_TO_FUNC, /* pointer to a bpf program function */ 622 ARG_PTR_TO_STACK, /* pointer to stack */ 623 ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */ 624 ARG_PTR_TO_TIMER, /* pointer to bpf_timer */ 625 ARG_PTR_TO_KPTR, /* pointer to referenced kptr */ 626 ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */ 627 __BPF_ARG_TYPE_MAX, 628 629 /* Extended arg_types. */ 630 ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE, 631 ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MEM, 632 ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_CTX, 633 ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET, 634 ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_STACK, 635 ARG_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_BTF_ID, 636 /* pointer to memory does not need to be initialized, helper function must fill 637 * all bytes or clear them in error case. 638 */ 639 ARG_PTR_TO_UNINIT_MEM = MEM_UNINIT | ARG_PTR_TO_MEM, 640 /* Pointer to valid memory of size known at compile time. */ 641 ARG_PTR_TO_FIXED_SIZE_MEM = MEM_FIXED_SIZE | ARG_PTR_TO_MEM, 642 643 /* This must be the last entry. Its purpose is to ensure the enum is 644 * wide enough to hold the higher bits reserved for bpf_type_flag. 645 */ 646 __BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT, 647 }; 648 static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); 649 650 /* type of values returned from helper functions */ 651 enum bpf_return_type { 652 RET_INTEGER, /* function returns integer */ 653 RET_VOID, /* function doesn't return anything */ 654 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ 655 RET_PTR_TO_SOCKET, /* returns a pointer to a socket */ 656 RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */ 657 RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */ 658 RET_PTR_TO_MEM, /* returns a pointer to memory */ 659 RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */ 660 RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */ 661 __BPF_RET_TYPE_MAX, 662 663 /* Extended ret_types. */ 664 RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE, 665 RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET, 666 RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK, 667 RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON, 668 RET_PTR_TO_RINGBUF_MEM_OR_NULL = PTR_MAYBE_NULL | MEM_RINGBUF | RET_PTR_TO_MEM, 669 RET_PTR_TO_DYNPTR_MEM_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MEM, 670 RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID, 671 RET_PTR_TO_BTF_ID_TRUSTED = PTR_TRUSTED | RET_PTR_TO_BTF_ID, 672 673 /* This must be the last entry. Its purpose is to ensure the enum is 674 * wide enough to hold the higher bits reserved for bpf_type_flag. 675 */ 676 __BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT, 677 }; 678 static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); 679 680 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs 681 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL 682 * instructions after verifying 683 */ 684 struct bpf_func_proto { 685 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 686 bool gpl_only; 687 bool pkt_access; 688 bool might_sleep; 689 enum bpf_return_type ret_type; 690 union { 691 struct { 692 enum bpf_arg_type arg1_type; 693 enum bpf_arg_type arg2_type; 694 enum bpf_arg_type arg3_type; 695 enum bpf_arg_type arg4_type; 696 enum bpf_arg_type arg5_type; 697 }; 698 enum bpf_arg_type arg_type[5]; 699 }; 700 union { 701 struct { 702 u32 *arg1_btf_id; 703 u32 *arg2_btf_id; 704 u32 *arg3_btf_id; 705 u32 *arg4_btf_id; 706 u32 *arg5_btf_id; 707 }; 708 u32 *arg_btf_id[5]; 709 struct { 710 size_t arg1_size; 711 size_t arg2_size; 712 size_t arg3_size; 713 size_t arg4_size; 714 size_t arg5_size; 715 }; 716 size_t arg_size[5]; 717 }; 718 int *ret_btf_id; /* return value btf_id */ 719 bool (*allowed)(const struct bpf_prog *prog); 720 }; 721 722 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is 723 * the first argument to eBPF programs. 724 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *' 725 */ 726 struct bpf_context; 727 728 enum bpf_access_type { 729 BPF_READ = 1, 730 BPF_WRITE = 2 731 }; 732 733 /* types of values stored in eBPF registers */ 734 /* Pointer types represent: 735 * pointer 736 * pointer + imm 737 * pointer + (u16) var 738 * pointer + (u16) var + imm 739 * if (range > 0) then [ptr, ptr + range - off) is safe to access 740 * if (id > 0) means that some 'var' was added 741 * if (off > 0) means that 'imm' was added 742 */ 743 enum bpf_reg_type { 744 NOT_INIT = 0, /* nothing was written into register */ 745 SCALAR_VALUE, /* reg doesn't contain a valid pointer */ 746 PTR_TO_CTX, /* reg points to bpf_context */ 747 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ 748 PTR_TO_MAP_VALUE, /* reg points to map element value */ 749 PTR_TO_MAP_KEY, /* reg points to a map element key */ 750 PTR_TO_STACK, /* reg == frame_pointer + offset */ 751 PTR_TO_PACKET_META, /* skb->data - meta_len */ 752 PTR_TO_PACKET, /* reg points to skb->data */ 753 PTR_TO_PACKET_END, /* skb->data + headlen */ 754 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ 755 PTR_TO_SOCKET, /* reg points to struct bpf_sock */ 756 PTR_TO_SOCK_COMMON, /* reg points to sock_common */ 757 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ 758 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ 759 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ 760 /* PTR_TO_BTF_ID points to a kernel struct that does not need 761 * to be null checked by the BPF program. This does not imply the 762 * pointer is _not_ null and in practice this can easily be a null 763 * pointer when reading pointer chains. The assumption is program 764 * context will handle null pointer dereference typically via fault 765 * handling. The verifier must keep this in mind and can make no 766 * assumptions about null or non-null when doing branch analysis. 767 * Further, when passed into helpers the helpers can not, without 768 * additional context, assume the value is non-null. 769 */ 770 PTR_TO_BTF_ID, 771 /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not 772 * been checked for null. Used primarily to inform the verifier 773 * an explicit null check is required for this struct. 774 */ 775 PTR_TO_MEM, /* reg points to valid memory region */ 776 PTR_TO_BUF, /* reg points to a read/write buffer */ 777 PTR_TO_FUNC, /* reg points to a bpf program function */ 778 CONST_PTR_TO_DYNPTR, /* reg points to a const struct bpf_dynptr */ 779 __BPF_REG_TYPE_MAX, 780 781 /* Extended reg_types. */ 782 PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MAP_VALUE, 783 PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET, 784 PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON, 785 PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK, 786 PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID, 787 788 /* This must be the last entry. Its purpose is to ensure the enum is 789 * wide enough to hold the higher bits reserved for bpf_type_flag. 790 */ 791 __BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT, 792 }; 793 static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); 794 795 /* The information passed from prog-specific *_is_valid_access 796 * back to the verifier. 797 */ 798 struct bpf_insn_access_aux { 799 enum bpf_reg_type reg_type; 800 union { 801 int ctx_field_size; 802 struct { 803 struct btf *btf; 804 u32 btf_id; 805 }; 806 }; 807 struct bpf_verifier_log *log; /* for verbose logs */ 808 }; 809 810 static inline void 811 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) 812 { 813 aux->ctx_field_size = size; 814 } 815 816 static inline bool bpf_pseudo_func(const struct bpf_insn *insn) 817 { 818 return insn->code == (BPF_LD | BPF_IMM | BPF_DW) && 819 insn->src_reg == BPF_PSEUDO_FUNC; 820 } 821 822 struct bpf_prog_ops { 823 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, 824 union bpf_attr __user *uattr); 825 }; 826 827 struct bpf_reg_state; 828 struct bpf_verifier_ops { 829 /* return eBPF function prototype for verification */ 830 const struct bpf_func_proto * 831 (*get_func_proto)(enum bpf_func_id func_id, 832 const struct bpf_prog *prog); 833 834 /* return true if 'size' wide access at offset 'off' within bpf_context 835 * with 'type' (read or write) is allowed 836 */ 837 bool (*is_valid_access)(int off, int size, enum bpf_access_type type, 838 const struct bpf_prog *prog, 839 struct bpf_insn_access_aux *info); 840 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, 841 const struct bpf_prog *prog); 842 int (*gen_ld_abs)(const struct bpf_insn *orig, 843 struct bpf_insn *insn_buf); 844 u32 (*convert_ctx_access)(enum bpf_access_type type, 845 const struct bpf_insn *src, 846 struct bpf_insn *dst, 847 struct bpf_prog *prog, u32 *target_size); 848 int (*btf_struct_access)(struct bpf_verifier_log *log, 849 const struct bpf_reg_state *reg, 850 int off, int size, enum bpf_access_type atype, 851 u32 *next_btf_id, enum bpf_type_flag *flag); 852 }; 853 854 struct bpf_prog_offload_ops { 855 /* verifier basic callbacks */ 856 int (*insn_hook)(struct bpf_verifier_env *env, 857 int insn_idx, int prev_insn_idx); 858 int (*finalize)(struct bpf_verifier_env *env); 859 /* verifier optimization callbacks (called after .finalize) */ 860 int (*replace_insn)(struct bpf_verifier_env *env, u32 off, 861 struct bpf_insn *insn); 862 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); 863 /* program management callbacks */ 864 int (*prepare)(struct bpf_prog *prog); 865 int (*translate)(struct bpf_prog *prog); 866 void (*destroy)(struct bpf_prog *prog); 867 }; 868 869 struct bpf_prog_offload { 870 struct bpf_prog *prog; 871 struct net_device *netdev; 872 struct bpf_offload_dev *offdev; 873 void *dev_priv; 874 struct list_head offloads; 875 bool dev_state; 876 bool opt_failed; 877 void *jited_image; 878 u32 jited_len; 879 }; 880 881 enum bpf_cgroup_storage_type { 882 BPF_CGROUP_STORAGE_SHARED, 883 BPF_CGROUP_STORAGE_PERCPU, 884 __BPF_CGROUP_STORAGE_MAX 885 }; 886 887 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX 888 889 /* The longest tracepoint has 12 args. 890 * See include/trace/bpf_probe.h 891 */ 892 #define MAX_BPF_FUNC_ARGS 12 893 894 /* The maximum number of arguments passed through registers 895 * a single function may have. 896 */ 897 #define MAX_BPF_FUNC_REG_ARGS 5 898 899 /* The argument is a structure. */ 900 #define BTF_FMODEL_STRUCT_ARG BIT(0) 901 902 struct btf_func_model { 903 u8 ret_size; 904 u8 nr_args; 905 u8 arg_size[MAX_BPF_FUNC_ARGS]; 906 u8 arg_flags[MAX_BPF_FUNC_ARGS]; 907 }; 908 909 /* Restore arguments before returning from trampoline to let original function 910 * continue executing. This flag is used for fentry progs when there are no 911 * fexit progs. 912 */ 913 #define BPF_TRAMP_F_RESTORE_REGS BIT(0) 914 /* Call original function after fentry progs, but before fexit progs. 915 * Makes sense for fentry/fexit, normal calls and indirect calls. 916 */ 917 #define BPF_TRAMP_F_CALL_ORIG BIT(1) 918 /* Skip current frame and return to parent. Makes sense for fentry/fexit 919 * programs only. Should not be used with normal calls and indirect calls. 920 */ 921 #define BPF_TRAMP_F_SKIP_FRAME BIT(2) 922 /* Store IP address of the caller on the trampoline stack, 923 * so it's available for trampoline's programs. 924 */ 925 #define BPF_TRAMP_F_IP_ARG BIT(3) 926 /* Return the return value of fentry prog. Only used by bpf_struct_ops. */ 927 #define BPF_TRAMP_F_RET_FENTRY_RET BIT(4) 928 929 /* Get original function from stack instead of from provided direct address. 930 * Makes sense for trampolines with fexit or fmod_ret programs. 931 */ 932 #define BPF_TRAMP_F_ORIG_STACK BIT(5) 933 934 /* This trampoline is on a function with another ftrace_ops with IPMODIFY, 935 * e.g., a live patch. This flag is set and cleared by ftrace call backs, 936 */ 937 #define BPF_TRAMP_F_SHARE_IPMODIFY BIT(6) 938 939 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 940 * bytes on x86. 941 */ 942 #define BPF_MAX_TRAMP_LINKS 38 943 944 struct bpf_tramp_links { 945 struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS]; 946 int nr_links; 947 }; 948 949 struct bpf_tramp_run_ctx; 950 951 /* Different use cases for BPF trampoline: 952 * 1. replace nop at the function entry (kprobe equivalent) 953 * flags = BPF_TRAMP_F_RESTORE_REGS 954 * fentry = a set of programs to run before returning from trampoline 955 * 956 * 2. replace nop at the function entry (kprobe + kretprobe equivalent) 957 * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME 958 * orig_call = fentry_ip + MCOUNT_INSN_SIZE 959 * fentry = a set of program to run before calling original function 960 * fexit = a set of program to run after original function 961 * 962 * 3. replace direct call instruction anywhere in the function body 963 * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid) 964 * With flags = 0 965 * fentry = a set of programs to run before returning from trampoline 966 * With flags = BPF_TRAMP_F_CALL_ORIG 967 * orig_call = original callback addr or direct function addr 968 * fentry = a set of program to run before calling original function 969 * fexit = a set of program to run after original function 970 */ 971 struct bpf_tramp_image; 972 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, 973 const struct btf_func_model *m, u32 flags, 974 struct bpf_tramp_links *tlinks, 975 void *orig_call); 976 u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, 977 struct bpf_tramp_run_ctx *run_ctx); 978 void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start, 979 struct bpf_tramp_run_ctx *run_ctx); 980 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr); 981 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr); 982 typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog, 983 struct bpf_tramp_run_ctx *run_ctx); 984 typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start, 985 struct bpf_tramp_run_ctx *run_ctx); 986 bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog); 987 bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog); 988 989 struct bpf_ksym { 990 unsigned long start; 991 unsigned long end; 992 char name[KSYM_NAME_LEN]; 993 struct list_head lnode; 994 struct latch_tree_node tnode; 995 bool prog; 996 }; 997 998 enum bpf_tramp_prog_type { 999 BPF_TRAMP_FENTRY, 1000 BPF_TRAMP_FEXIT, 1001 BPF_TRAMP_MODIFY_RETURN, 1002 BPF_TRAMP_MAX, 1003 BPF_TRAMP_REPLACE, /* more than MAX */ 1004 }; 1005 1006 struct bpf_tramp_image { 1007 void *image; 1008 struct bpf_ksym ksym; 1009 struct percpu_ref pcref; 1010 void *ip_after_call; 1011 void *ip_epilogue; 1012 union { 1013 struct rcu_head rcu; 1014 struct work_struct work; 1015 }; 1016 }; 1017 1018 struct bpf_trampoline { 1019 /* hlist for trampoline_table */ 1020 struct hlist_node hlist; 1021 struct ftrace_ops *fops; 1022 /* serializes access to fields of this trampoline */ 1023 struct mutex mutex; 1024 refcount_t refcnt; 1025 u32 flags; 1026 u64 key; 1027 struct { 1028 struct btf_func_model model; 1029 void *addr; 1030 bool ftrace_managed; 1031 } func; 1032 /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF 1033 * program by replacing one of its functions. func.addr is the address 1034 * of the function it replaced. 1035 */ 1036 struct bpf_prog *extension_prog; 1037 /* list of BPF programs using this trampoline */ 1038 struct hlist_head progs_hlist[BPF_TRAMP_MAX]; 1039 /* Number of attached programs. A counter per kind. */ 1040 int progs_cnt[BPF_TRAMP_MAX]; 1041 /* Executable image of trampoline */ 1042 struct bpf_tramp_image *cur_image; 1043 u64 selector; 1044 struct module *mod; 1045 }; 1046 1047 struct bpf_attach_target_info { 1048 struct btf_func_model fmodel; 1049 long tgt_addr; 1050 const char *tgt_name; 1051 const struct btf_type *tgt_type; 1052 }; 1053 1054 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */ 1055 1056 struct bpf_dispatcher_prog { 1057 struct bpf_prog *prog; 1058 refcount_t users; 1059 }; 1060 1061 struct bpf_dispatcher { 1062 /* dispatcher mutex */ 1063 struct mutex mutex; 1064 void *func; 1065 struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX]; 1066 int num_progs; 1067 void *image; 1068 void *rw_image; 1069 u32 image_off; 1070 struct bpf_ksym ksym; 1071 #ifdef CONFIG_HAVE_STATIC_CALL 1072 struct static_call_key *sc_key; 1073 void *sc_tramp; 1074 #endif 1075 }; 1076 1077 static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( 1078 const void *ctx, 1079 const struct bpf_insn *insnsi, 1080 bpf_func_t bpf_func) 1081 { 1082 return bpf_func(ctx, insnsi); 1083 } 1084 1085 #ifdef CONFIG_BPF_JIT 1086 int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr); 1087 int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr); 1088 struct bpf_trampoline *bpf_trampoline_get(u64 key, 1089 struct bpf_attach_target_info *tgt_info); 1090 void bpf_trampoline_put(struct bpf_trampoline *tr); 1091 int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs); 1092 1093 /* 1094 * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn 1095 * indirection with a direct call to the bpf program. If the architecture does 1096 * not have STATIC_CALL, avoid a double-indirection. 1097 */ 1098 #ifdef CONFIG_HAVE_STATIC_CALL 1099 1100 #define __BPF_DISPATCHER_SC_INIT(_name) \ 1101 .sc_key = &STATIC_CALL_KEY(_name), \ 1102 .sc_tramp = STATIC_CALL_TRAMP_ADDR(_name), 1103 1104 #define __BPF_DISPATCHER_SC(name) \ 1105 DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func) 1106 1107 #define __BPF_DISPATCHER_CALL(name) \ 1108 static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func) 1109 1110 #define __BPF_DISPATCHER_UPDATE(_d, _new) \ 1111 __static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new)) 1112 1113 #else 1114 #define __BPF_DISPATCHER_SC_INIT(name) 1115 #define __BPF_DISPATCHER_SC(name) 1116 #define __BPF_DISPATCHER_CALL(name) bpf_func(ctx, insnsi) 1117 #define __BPF_DISPATCHER_UPDATE(_d, _new) 1118 #endif 1119 1120 #define BPF_DISPATCHER_INIT(_name) { \ 1121 .mutex = __MUTEX_INITIALIZER(_name.mutex), \ 1122 .func = &_name##_func, \ 1123 .progs = {}, \ 1124 .num_progs = 0, \ 1125 .image = NULL, \ 1126 .image_off = 0, \ 1127 .ksym = { \ 1128 .name = #_name, \ 1129 .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \ 1130 }, \ 1131 __BPF_DISPATCHER_SC_INIT(_name##_call) \ 1132 } 1133 1134 #define DEFINE_BPF_DISPATCHER(name) \ 1135 __BPF_DISPATCHER_SC(name); \ 1136 noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \ 1137 const void *ctx, \ 1138 const struct bpf_insn *insnsi, \ 1139 bpf_func_t bpf_func) \ 1140 { \ 1141 return __BPF_DISPATCHER_CALL(name); \ 1142 } \ 1143 EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \ 1144 struct bpf_dispatcher bpf_dispatcher_##name = \ 1145 BPF_DISPATCHER_INIT(bpf_dispatcher_##name); 1146 1147 #define DECLARE_BPF_DISPATCHER(name) \ 1148 unsigned int bpf_dispatcher_##name##_func( \ 1149 const void *ctx, \ 1150 const struct bpf_insn *insnsi, \ 1151 bpf_func_t bpf_func); \ 1152 extern struct bpf_dispatcher bpf_dispatcher_##name; 1153 1154 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func 1155 #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name) 1156 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, 1157 struct bpf_prog *to); 1158 /* Called only from JIT-enabled code, so there's no need for stubs. */ 1159 void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); 1160 void bpf_image_ksym_del(struct bpf_ksym *ksym); 1161 void bpf_ksym_add(struct bpf_ksym *ksym); 1162 void bpf_ksym_del(struct bpf_ksym *ksym); 1163 int bpf_jit_charge_modmem(u32 size); 1164 void bpf_jit_uncharge_modmem(u32 size); 1165 bool bpf_prog_has_trampoline(const struct bpf_prog *prog); 1166 #else 1167 static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link, 1168 struct bpf_trampoline *tr) 1169 { 1170 return -ENOTSUPP; 1171 } 1172 static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, 1173 struct bpf_trampoline *tr) 1174 { 1175 return -ENOTSUPP; 1176 } 1177 static inline struct bpf_trampoline *bpf_trampoline_get(u64 key, 1178 struct bpf_attach_target_info *tgt_info) 1179 { 1180 return ERR_PTR(-EOPNOTSUPP); 1181 } 1182 static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {} 1183 #define DEFINE_BPF_DISPATCHER(name) 1184 #define DECLARE_BPF_DISPATCHER(name) 1185 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func 1186 #define BPF_DISPATCHER_PTR(name) NULL 1187 static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, 1188 struct bpf_prog *from, 1189 struct bpf_prog *to) {} 1190 static inline bool is_bpf_image_address(unsigned long address) 1191 { 1192 return false; 1193 } 1194 static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog) 1195 { 1196 return false; 1197 } 1198 #endif 1199 1200 struct bpf_func_info_aux { 1201 u16 linkage; 1202 bool unreliable; 1203 }; 1204 1205 enum bpf_jit_poke_reason { 1206 BPF_POKE_REASON_TAIL_CALL, 1207 }; 1208 1209 /* Descriptor of pokes pointing /into/ the JITed image. */ 1210 struct bpf_jit_poke_descriptor { 1211 void *tailcall_target; 1212 void *tailcall_bypass; 1213 void *bypass_addr; 1214 void *aux; 1215 union { 1216 struct { 1217 struct bpf_map *map; 1218 u32 key; 1219 } tail_call; 1220 }; 1221 bool tailcall_target_stable; 1222 u8 adj_off; 1223 u16 reason; 1224 u32 insn_idx; 1225 }; 1226 1227 /* reg_type info for ctx arguments */ 1228 struct bpf_ctx_arg_aux { 1229 u32 offset; 1230 enum bpf_reg_type reg_type; 1231 u32 btf_id; 1232 }; 1233 1234 struct btf_mod_pair { 1235 struct btf *btf; 1236 struct module *module; 1237 }; 1238 1239 struct bpf_kfunc_desc_tab; 1240 1241 struct bpf_prog_aux { 1242 atomic64_t refcnt; 1243 u32 used_map_cnt; 1244 u32 used_btf_cnt; 1245 u32 max_ctx_offset; 1246 u32 max_pkt_offset; 1247 u32 max_tp_access; 1248 u32 stack_depth; 1249 u32 id; 1250 u32 func_cnt; /* used by non-func prog as the number of func progs */ 1251 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ 1252 u32 attach_btf_id; /* in-kernel BTF type id to attach to */ 1253 u32 ctx_arg_info_size; 1254 u32 max_rdonly_access; 1255 u32 max_rdwr_access; 1256 struct btf *attach_btf; 1257 const struct bpf_ctx_arg_aux *ctx_arg_info; 1258 struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */ 1259 struct bpf_prog *dst_prog; 1260 struct bpf_trampoline *dst_trampoline; 1261 enum bpf_prog_type saved_dst_prog_type; 1262 enum bpf_attach_type saved_dst_attach_type; 1263 bool verifier_zext; /* Zero extensions has been inserted by verifier. */ 1264 bool offload_requested; 1265 bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ 1266 bool func_proto_unreliable; 1267 bool sleepable; 1268 bool tail_call_reachable; 1269 bool xdp_has_frags; 1270 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ 1271 const struct btf_type *attach_func_proto; 1272 /* function name for valid attach_btf_id */ 1273 const char *attach_func_name; 1274 struct bpf_prog **func; 1275 void *jit_data; /* JIT specific data. arch dependent */ 1276 struct bpf_jit_poke_descriptor *poke_tab; 1277 struct bpf_kfunc_desc_tab *kfunc_tab; 1278 struct bpf_kfunc_btf_tab *kfunc_btf_tab; 1279 u32 size_poke_tab; 1280 struct bpf_ksym ksym; 1281 const struct bpf_prog_ops *ops; 1282 struct bpf_map **used_maps; 1283 struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */ 1284 struct btf_mod_pair *used_btfs; 1285 struct bpf_prog *prog; 1286 struct user_struct *user; 1287 u64 load_time; /* ns since boottime */ 1288 u32 verified_insns; 1289 int cgroup_atype; /* enum cgroup_bpf_attach_type */ 1290 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 1291 char name[BPF_OBJ_NAME_LEN]; 1292 #ifdef CONFIG_SECURITY 1293 void *security; 1294 #endif 1295 struct bpf_prog_offload *offload; 1296 struct btf *btf; 1297 struct bpf_func_info *func_info; 1298 struct bpf_func_info_aux *func_info_aux; 1299 /* bpf_line_info loaded from userspace. linfo->insn_off 1300 * has the xlated insn offset. 1301 * Both the main and sub prog share the same linfo. 1302 * The subprog can access its first linfo by 1303 * using the linfo_idx. 1304 */ 1305 struct bpf_line_info *linfo; 1306 /* jited_linfo is the jited addr of the linfo. It has a 1307 * one to one mapping to linfo: 1308 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. 1309 * Both the main and sub prog share the same jited_linfo. 1310 * The subprog can access its first jited_linfo by 1311 * using the linfo_idx. 1312 */ 1313 void **jited_linfo; 1314 u32 func_info_cnt; 1315 u32 nr_linfo; 1316 /* subprog can use linfo_idx to access its first linfo and 1317 * jited_linfo. 1318 * main prog always has linfo_idx == 0 1319 */ 1320 u32 linfo_idx; 1321 u32 num_exentries; 1322 struct exception_table_entry *extable; 1323 union { 1324 struct work_struct work; 1325 struct rcu_head rcu; 1326 }; 1327 }; 1328 1329 struct bpf_prog { 1330 u16 pages; /* Number of allocated pages */ 1331 u16 jited:1, /* Is our filter JIT'ed? */ 1332 jit_requested:1,/* archs need to JIT the prog */ 1333 gpl_compatible:1, /* Is filter GPL compatible? */ 1334 cb_access:1, /* Is control block accessed? */ 1335 dst_needed:1, /* Do we need dst entry? */ 1336 blinding_requested:1, /* needs constant blinding */ 1337 blinded:1, /* Was blinded */ 1338 is_func:1, /* program is a bpf function */ 1339 kprobe_override:1, /* Do we override a kprobe? */ 1340 has_callchain_buf:1, /* callchain buffer allocated? */ 1341 enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */ 1342 call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */ 1343 call_get_func_ip:1, /* Do we call get_func_ip() */ 1344 tstamp_type_access:1; /* Accessed __sk_buff->tstamp_type */ 1345 enum bpf_prog_type type; /* Type of BPF program */ 1346 enum bpf_attach_type expected_attach_type; /* For some prog types */ 1347 u32 len; /* Number of filter blocks */ 1348 u32 jited_len; /* Size of jited insns in bytes */ 1349 u8 tag[BPF_TAG_SIZE]; 1350 struct bpf_prog_stats __percpu *stats; 1351 int __percpu *active; 1352 unsigned int (*bpf_func)(const void *ctx, 1353 const struct bpf_insn *insn); 1354 struct bpf_prog_aux *aux; /* Auxiliary fields */ 1355 struct sock_fprog_kern *orig_prog; /* Original BPF program */ 1356 /* Instructions for interpreter */ 1357 union { 1358 DECLARE_FLEX_ARRAY(struct sock_filter, insns); 1359 DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi); 1360 }; 1361 }; 1362 1363 struct bpf_array_aux { 1364 /* Programs with direct jumps into programs part of this array. */ 1365 struct list_head poke_progs; 1366 struct bpf_map *map; 1367 struct mutex poke_mutex; 1368 struct work_struct work; 1369 }; 1370 1371 struct bpf_link { 1372 atomic64_t refcnt; 1373 u32 id; 1374 enum bpf_link_type type; 1375 const struct bpf_link_ops *ops; 1376 struct bpf_prog *prog; 1377 struct work_struct work; 1378 }; 1379 1380 struct bpf_link_ops { 1381 void (*release)(struct bpf_link *link); 1382 void (*dealloc)(struct bpf_link *link); 1383 int (*detach)(struct bpf_link *link); 1384 int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog, 1385 struct bpf_prog *old_prog); 1386 void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq); 1387 int (*fill_link_info)(const struct bpf_link *link, 1388 struct bpf_link_info *info); 1389 }; 1390 1391 struct bpf_tramp_link { 1392 struct bpf_link link; 1393 struct hlist_node tramp_hlist; 1394 u64 cookie; 1395 }; 1396 1397 struct bpf_shim_tramp_link { 1398 struct bpf_tramp_link link; 1399 struct bpf_trampoline *trampoline; 1400 }; 1401 1402 struct bpf_tracing_link { 1403 struct bpf_tramp_link link; 1404 enum bpf_attach_type attach_type; 1405 struct bpf_trampoline *trampoline; 1406 struct bpf_prog *tgt_prog; 1407 }; 1408 1409 struct bpf_link_primer { 1410 struct bpf_link *link; 1411 struct file *file; 1412 int fd; 1413 u32 id; 1414 }; 1415 1416 struct bpf_struct_ops_value; 1417 struct btf_member; 1418 1419 #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64 1420 struct bpf_struct_ops { 1421 const struct bpf_verifier_ops *verifier_ops; 1422 int (*init)(struct btf *btf); 1423 int (*check_member)(const struct btf_type *t, 1424 const struct btf_member *member); 1425 int (*init_member)(const struct btf_type *t, 1426 const struct btf_member *member, 1427 void *kdata, const void *udata); 1428 int (*reg)(void *kdata); 1429 void (*unreg)(void *kdata); 1430 const struct btf_type *type; 1431 const struct btf_type *value_type; 1432 const char *name; 1433 struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; 1434 u32 type_id; 1435 u32 value_id; 1436 }; 1437 1438 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) 1439 #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) 1440 const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id); 1441 void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log); 1442 bool bpf_struct_ops_get(const void *kdata); 1443 void bpf_struct_ops_put(const void *kdata); 1444 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, 1445 void *value); 1446 int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, 1447 struct bpf_tramp_link *link, 1448 const struct btf_func_model *model, 1449 void *image, void *image_end); 1450 static inline bool bpf_try_module_get(const void *data, struct module *owner) 1451 { 1452 if (owner == BPF_MODULE_OWNER) 1453 return bpf_struct_ops_get(data); 1454 else 1455 return try_module_get(owner); 1456 } 1457 static inline void bpf_module_put(const void *data, struct module *owner) 1458 { 1459 if (owner == BPF_MODULE_OWNER) 1460 bpf_struct_ops_put(data); 1461 else 1462 module_put(owner); 1463 } 1464 1465 #ifdef CONFIG_NET 1466 /* Define it here to avoid the use of forward declaration */ 1467 struct bpf_dummy_ops_state { 1468 int val; 1469 }; 1470 1471 struct bpf_dummy_ops { 1472 int (*test_1)(struct bpf_dummy_ops_state *cb); 1473 int (*test_2)(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2, 1474 char a3, unsigned long a4); 1475 }; 1476 1477 int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, 1478 union bpf_attr __user *uattr); 1479 #endif 1480 #else 1481 static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) 1482 { 1483 return NULL; 1484 } 1485 static inline void bpf_struct_ops_init(struct btf *btf, 1486 struct bpf_verifier_log *log) 1487 { 1488 } 1489 static inline bool bpf_try_module_get(const void *data, struct module *owner) 1490 { 1491 return try_module_get(owner); 1492 } 1493 static inline void bpf_module_put(const void *data, struct module *owner) 1494 { 1495 module_put(owner); 1496 } 1497 static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, 1498 void *key, 1499 void *value) 1500 { 1501 return -EINVAL; 1502 } 1503 #endif 1504 1505 #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM) 1506 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, 1507 int cgroup_atype); 1508 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog); 1509 #else 1510 static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, 1511 int cgroup_atype) 1512 { 1513 return -EOPNOTSUPP; 1514 } 1515 static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog) 1516 { 1517 } 1518 #endif 1519 1520 struct bpf_array { 1521 struct bpf_map map; 1522 u32 elem_size; 1523 u32 index_mask; 1524 struct bpf_array_aux *aux; 1525 union { 1526 char value[0] __aligned(8); 1527 void *ptrs[0] __aligned(8); 1528 void __percpu *pptrs[0] __aligned(8); 1529 }; 1530 }; 1531 1532 #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ 1533 #define MAX_TAIL_CALL_CNT 33 1534 1535 /* Maximum number of loops for bpf_loop */ 1536 #define BPF_MAX_LOOPS BIT(23) 1537 1538 #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ 1539 BPF_F_RDONLY_PROG | \ 1540 BPF_F_WRONLY | \ 1541 BPF_F_WRONLY_PROG) 1542 1543 #define BPF_MAP_CAN_READ BIT(0) 1544 #define BPF_MAP_CAN_WRITE BIT(1) 1545 1546 /* Maximum number of user-producer ring buffer samples that can be drained in 1547 * a call to bpf_user_ringbuf_drain(). 1548 */ 1549 #define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024) 1550 1551 static inline u32 bpf_map_flags_to_cap(struct bpf_map *map) 1552 { 1553 u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 1554 1555 /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is 1556 * not possible. 1557 */ 1558 if (access_flags & BPF_F_RDONLY_PROG) 1559 return BPF_MAP_CAN_READ; 1560 else if (access_flags & BPF_F_WRONLY_PROG) 1561 return BPF_MAP_CAN_WRITE; 1562 else 1563 return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE; 1564 } 1565 1566 static inline bool bpf_map_flags_access_ok(u32 access_flags) 1567 { 1568 return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) != 1569 (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 1570 } 1571 1572 struct bpf_event_entry { 1573 struct perf_event *event; 1574 struct file *perf_file; 1575 struct file *map_file; 1576 struct rcu_head rcu; 1577 }; 1578 1579 static inline bool map_type_contains_progs(struct bpf_map *map) 1580 { 1581 return map->map_type == BPF_MAP_TYPE_PROG_ARRAY || 1582 map->map_type == BPF_MAP_TYPE_DEVMAP || 1583 map->map_type == BPF_MAP_TYPE_CPUMAP; 1584 } 1585 1586 bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp); 1587 int bpf_prog_calc_tag(struct bpf_prog *fp); 1588 1589 const struct bpf_func_proto *bpf_get_trace_printk_proto(void); 1590 const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void); 1591 1592 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, 1593 unsigned long off, unsigned long len); 1594 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, 1595 const struct bpf_insn *src, 1596 struct bpf_insn *dst, 1597 struct bpf_prog *prog, 1598 u32 *target_size); 1599 1600 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 1601 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); 1602 1603 /* an array of programs to be executed under rcu_lock. 1604 * 1605 * Typical usage: 1606 * ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run); 1607 * 1608 * the structure returned by bpf_prog_array_alloc() should be populated 1609 * with program pointers and the last pointer must be NULL. 1610 * The user has to keep refcnt on the program and make sure the program 1611 * is removed from the array before bpf_prog_put(). 1612 * The 'struct bpf_prog_array *' should only be replaced with xchg() 1613 * since other cpus are walking the array of pointers in parallel. 1614 */ 1615 struct bpf_prog_array_item { 1616 struct bpf_prog *prog; 1617 union { 1618 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 1619 u64 bpf_cookie; 1620 }; 1621 }; 1622 1623 struct bpf_prog_array { 1624 struct rcu_head rcu; 1625 struct bpf_prog_array_item items[]; 1626 }; 1627 1628 struct bpf_empty_prog_array { 1629 struct bpf_prog_array hdr; 1630 struct bpf_prog *null_prog; 1631 }; 1632 1633 /* to avoid allocating empty bpf_prog_array for cgroups that 1634 * don't have bpf program attached use one global 'bpf_empty_prog_array' 1635 * It will not be modified the caller of bpf_prog_array_alloc() 1636 * (since caller requested prog_cnt == 0) 1637 * that pointer should be 'freed' by bpf_prog_array_free() 1638 */ 1639 extern struct bpf_empty_prog_array bpf_empty_prog_array; 1640 1641 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); 1642 void bpf_prog_array_free(struct bpf_prog_array *progs); 1643 /* Use when traversal over the bpf_prog_array uses tasks_trace rcu */ 1644 void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs); 1645 int bpf_prog_array_length(struct bpf_prog_array *progs); 1646 bool bpf_prog_array_is_empty(struct bpf_prog_array *array); 1647 int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs, 1648 __u32 __user *prog_ids, u32 cnt); 1649 1650 void bpf_prog_array_delete_safe(struct bpf_prog_array *progs, 1651 struct bpf_prog *old_prog); 1652 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index); 1653 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index, 1654 struct bpf_prog *prog); 1655 int bpf_prog_array_copy_info(struct bpf_prog_array *array, 1656 u32 *prog_ids, u32 request_cnt, 1657 u32 *prog_cnt); 1658 int bpf_prog_array_copy(struct bpf_prog_array *old_array, 1659 struct bpf_prog *exclude_prog, 1660 struct bpf_prog *include_prog, 1661 u64 bpf_cookie, 1662 struct bpf_prog_array **new_array); 1663 1664 struct bpf_run_ctx {}; 1665 1666 struct bpf_cg_run_ctx { 1667 struct bpf_run_ctx run_ctx; 1668 const struct bpf_prog_array_item *prog_item; 1669 int retval; 1670 }; 1671 1672 struct bpf_trace_run_ctx { 1673 struct bpf_run_ctx run_ctx; 1674 u64 bpf_cookie; 1675 }; 1676 1677 struct bpf_tramp_run_ctx { 1678 struct bpf_run_ctx run_ctx; 1679 u64 bpf_cookie; 1680 struct bpf_run_ctx *saved_run_ctx; 1681 }; 1682 1683 static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx) 1684 { 1685 struct bpf_run_ctx *old_ctx = NULL; 1686 1687 #ifdef CONFIG_BPF_SYSCALL 1688 old_ctx = current->bpf_ctx; 1689 current->bpf_ctx = new_ctx; 1690 #endif 1691 return old_ctx; 1692 } 1693 1694 static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx) 1695 { 1696 #ifdef CONFIG_BPF_SYSCALL 1697 current->bpf_ctx = old_ctx; 1698 #endif 1699 } 1700 1701 /* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */ 1702 #define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0) 1703 /* BPF program asks to set CN on the packet. */ 1704 #define BPF_RET_SET_CN (1 << 0) 1705 1706 typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx); 1707 1708 static __always_inline u32 1709 bpf_prog_run_array(const struct bpf_prog_array *array, 1710 const void *ctx, bpf_prog_run_fn run_prog) 1711 { 1712 const struct bpf_prog_array_item *item; 1713 const struct bpf_prog *prog; 1714 struct bpf_run_ctx *old_run_ctx; 1715 struct bpf_trace_run_ctx run_ctx; 1716 u32 ret = 1; 1717 1718 RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held"); 1719 1720 if (unlikely(!array)) 1721 return ret; 1722 1723 migrate_disable(); 1724 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 1725 item = &array->items[0]; 1726 while ((prog = READ_ONCE(item->prog))) { 1727 run_ctx.bpf_cookie = item->bpf_cookie; 1728 ret &= run_prog(prog, ctx); 1729 item++; 1730 } 1731 bpf_reset_run_ctx(old_run_ctx); 1732 migrate_enable(); 1733 return ret; 1734 } 1735 1736 /* Notes on RCU design for bpf_prog_arrays containing sleepable programs: 1737 * 1738 * We use the tasks_trace rcu flavor read section to protect the bpf_prog_array 1739 * overall. As a result, we must use the bpf_prog_array_free_sleepable 1740 * in order to use the tasks_trace rcu grace period. 1741 * 1742 * When a non-sleepable program is inside the array, we take the rcu read 1743 * section and disable preemption for that program alone, so it can access 1744 * rcu-protected dynamically sized maps. 1745 */ 1746 static __always_inline u32 1747 bpf_prog_run_array_sleepable(const struct bpf_prog_array __rcu *array_rcu, 1748 const void *ctx, bpf_prog_run_fn run_prog) 1749 { 1750 const struct bpf_prog_array_item *item; 1751 const struct bpf_prog *prog; 1752 const struct bpf_prog_array *array; 1753 struct bpf_run_ctx *old_run_ctx; 1754 struct bpf_trace_run_ctx run_ctx; 1755 u32 ret = 1; 1756 1757 might_fault(); 1758 1759 rcu_read_lock_trace(); 1760 migrate_disable(); 1761 1762 array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held()); 1763 if (unlikely(!array)) 1764 goto out; 1765 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 1766 item = &array->items[0]; 1767 while ((prog = READ_ONCE(item->prog))) { 1768 if (!prog->aux->sleepable) 1769 rcu_read_lock(); 1770 1771 run_ctx.bpf_cookie = item->bpf_cookie; 1772 ret &= run_prog(prog, ctx); 1773 item++; 1774 1775 if (!prog->aux->sleepable) 1776 rcu_read_unlock(); 1777 } 1778 bpf_reset_run_ctx(old_run_ctx); 1779 out: 1780 migrate_enable(); 1781 rcu_read_unlock_trace(); 1782 return ret; 1783 } 1784 1785 #ifdef CONFIG_BPF_SYSCALL 1786 DECLARE_PER_CPU(int, bpf_prog_active); 1787 extern struct mutex bpf_stats_enabled_mutex; 1788 1789 /* 1790 * Block execution of BPF programs attached to instrumentation (perf, 1791 * kprobes, tracepoints) to prevent deadlocks on map operations as any of 1792 * these events can happen inside a region which holds a map bucket lock 1793 * and can deadlock on it. 1794 */ 1795 static inline void bpf_disable_instrumentation(void) 1796 { 1797 migrate_disable(); 1798 this_cpu_inc(bpf_prog_active); 1799 } 1800 1801 static inline void bpf_enable_instrumentation(void) 1802 { 1803 this_cpu_dec(bpf_prog_active); 1804 migrate_enable(); 1805 } 1806 1807 extern const struct file_operations bpf_map_fops; 1808 extern const struct file_operations bpf_prog_fops; 1809 extern const struct file_operations bpf_iter_fops; 1810 1811 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 1812 extern const struct bpf_prog_ops _name ## _prog_ops; \ 1813 extern const struct bpf_verifier_ops _name ## _verifier_ops; 1814 #define BPF_MAP_TYPE(_id, _ops) \ 1815 extern const struct bpf_map_ops _ops; 1816 #define BPF_LINK_TYPE(_id, _name) 1817 #include <linux/bpf_types.h> 1818 #undef BPF_PROG_TYPE 1819 #undef BPF_MAP_TYPE 1820 #undef BPF_LINK_TYPE 1821 1822 extern const struct bpf_prog_ops bpf_offload_prog_ops; 1823 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; 1824 extern const struct bpf_verifier_ops xdp_analyzer_ops; 1825 1826 struct bpf_prog *bpf_prog_get(u32 ufd); 1827 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1828 bool attach_drv); 1829 void bpf_prog_add(struct bpf_prog *prog, int i); 1830 void bpf_prog_sub(struct bpf_prog *prog, int i); 1831 void bpf_prog_inc(struct bpf_prog *prog); 1832 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); 1833 void bpf_prog_put(struct bpf_prog *prog); 1834 1835 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock); 1836 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock); 1837 1838 struct btf_field *btf_record_find(const struct btf_record *rec, 1839 u32 offset, enum btf_field_type type); 1840 void btf_record_free(struct btf_record *rec); 1841 void bpf_map_free_record(struct bpf_map *map); 1842 struct btf_record *btf_record_dup(const struct btf_record *rec); 1843 bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b); 1844 void bpf_obj_free_timer(const struct btf_record *rec, void *obj); 1845 void bpf_obj_free_fields(const struct btf_record *rec, void *obj); 1846 1847 struct bpf_map *bpf_map_get(u32 ufd); 1848 struct bpf_map *bpf_map_get_with_uref(u32 ufd); 1849 struct bpf_map *__bpf_map_get(struct fd f); 1850 void bpf_map_inc(struct bpf_map *map); 1851 void bpf_map_inc_with_uref(struct bpf_map *map); 1852 struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map); 1853 void bpf_map_put_with_uref(struct bpf_map *map); 1854 void bpf_map_put(struct bpf_map *map); 1855 void *bpf_map_area_alloc(u64 size, int numa_node); 1856 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node); 1857 void bpf_map_area_free(void *base); 1858 bool bpf_map_write_active(const struct bpf_map *map); 1859 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); 1860 int generic_map_lookup_batch(struct bpf_map *map, 1861 const union bpf_attr *attr, 1862 union bpf_attr __user *uattr); 1863 int generic_map_update_batch(struct bpf_map *map, struct file *map_file, 1864 const union bpf_attr *attr, 1865 union bpf_attr __user *uattr); 1866 int generic_map_delete_batch(struct bpf_map *map, 1867 const union bpf_attr *attr, 1868 union bpf_attr __user *uattr); 1869 struct bpf_map *bpf_map_get_curr_or_next(u32 *id); 1870 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); 1871 1872 #ifdef CONFIG_MEMCG_KMEM 1873 void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 1874 int node); 1875 void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags); 1876 void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, 1877 size_t align, gfp_t flags); 1878 #else 1879 static inline void * 1880 bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 1881 int node) 1882 { 1883 return kmalloc_node(size, flags, node); 1884 } 1885 1886 static inline void * 1887 bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) 1888 { 1889 return kzalloc(size, flags); 1890 } 1891 1892 static inline void __percpu * 1893 bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, 1894 gfp_t flags) 1895 { 1896 return __alloc_percpu_gfp(size, align, flags); 1897 } 1898 #endif 1899 1900 extern int sysctl_unprivileged_bpf_disabled; 1901 1902 static inline bool bpf_allow_ptr_leaks(void) 1903 { 1904 return perfmon_capable(); 1905 } 1906 1907 static inline bool bpf_allow_uninit_stack(void) 1908 { 1909 return perfmon_capable(); 1910 } 1911 1912 static inline bool bpf_bypass_spec_v1(void) 1913 { 1914 return perfmon_capable(); 1915 } 1916 1917 static inline bool bpf_bypass_spec_v4(void) 1918 { 1919 return perfmon_capable(); 1920 } 1921 1922 int bpf_map_new_fd(struct bpf_map *map, int flags); 1923 int bpf_prog_new_fd(struct bpf_prog *prog); 1924 1925 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 1926 const struct bpf_link_ops *ops, struct bpf_prog *prog); 1927 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer); 1928 int bpf_link_settle(struct bpf_link_primer *primer); 1929 void bpf_link_cleanup(struct bpf_link_primer *primer); 1930 void bpf_link_inc(struct bpf_link *link); 1931 void bpf_link_put(struct bpf_link *link); 1932 int bpf_link_new_fd(struct bpf_link *link); 1933 struct file *bpf_link_new_file(struct bpf_link *link, int *reserved_fd); 1934 struct bpf_link *bpf_link_get_from_fd(u32 ufd); 1935 struct bpf_link *bpf_link_get_curr_or_next(u32 *id); 1936 1937 int bpf_obj_pin_user(u32 ufd, const char __user *pathname); 1938 int bpf_obj_get_user(const char __user *pathname, int flags); 1939 1940 #define BPF_ITER_FUNC_PREFIX "bpf_iter_" 1941 #define DEFINE_BPF_ITER_FUNC(target, args...) \ 1942 extern int bpf_iter_ ## target(args); \ 1943 int __init bpf_iter_ ## target(args) { return 0; } 1944 1945 /* 1946 * The task type of iterators. 1947 * 1948 * For BPF task iterators, they can be parameterized with various 1949 * parameters to visit only some of tasks. 1950 * 1951 * BPF_TASK_ITER_ALL (default) 1952 * Iterate over resources of every task. 1953 * 1954 * BPF_TASK_ITER_TID 1955 * Iterate over resources of a task/tid. 1956 * 1957 * BPF_TASK_ITER_TGID 1958 * Iterate over resources of every task of a process / task group. 1959 */ 1960 enum bpf_iter_task_type { 1961 BPF_TASK_ITER_ALL = 0, 1962 BPF_TASK_ITER_TID, 1963 BPF_TASK_ITER_TGID, 1964 }; 1965 1966 struct bpf_iter_aux_info { 1967 /* for map_elem iter */ 1968 struct bpf_map *map; 1969 1970 /* for cgroup iter */ 1971 struct { 1972 struct cgroup *start; /* starting cgroup */ 1973 enum bpf_cgroup_iter_order order; 1974 } cgroup; 1975 struct { 1976 enum bpf_iter_task_type type; 1977 u32 pid; 1978 } task; 1979 }; 1980 1981 typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, 1982 union bpf_iter_link_info *linfo, 1983 struct bpf_iter_aux_info *aux); 1984 typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux); 1985 typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux, 1986 struct seq_file *seq); 1987 typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux, 1988 struct bpf_link_info *info); 1989 typedef const struct bpf_func_proto * 1990 (*bpf_iter_get_func_proto_t)(enum bpf_func_id func_id, 1991 const struct bpf_prog *prog); 1992 1993 enum bpf_iter_feature { 1994 BPF_ITER_RESCHED = BIT(0), 1995 }; 1996 1997 #define BPF_ITER_CTX_ARG_MAX 2 1998 struct bpf_iter_reg { 1999 const char *target; 2000 bpf_iter_attach_target_t attach_target; 2001 bpf_iter_detach_target_t detach_target; 2002 bpf_iter_show_fdinfo_t show_fdinfo; 2003 bpf_iter_fill_link_info_t fill_link_info; 2004 bpf_iter_get_func_proto_t get_func_proto; 2005 u32 ctx_arg_info_size; 2006 u32 feature; 2007 struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX]; 2008 const struct bpf_iter_seq_info *seq_info; 2009 }; 2010 2011 struct bpf_iter_meta { 2012 __bpf_md_ptr(struct seq_file *, seq); 2013 u64 session_id; 2014 u64 seq_num; 2015 }; 2016 2017 struct bpf_iter__bpf_map_elem { 2018 __bpf_md_ptr(struct bpf_iter_meta *, meta); 2019 __bpf_md_ptr(struct bpf_map *, map); 2020 __bpf_md_ptr(void *, key); 2021 __bpf_md_ptr(void *, value); 2022 }; 2023 2024 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info); 2025 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info); 2026 bool bpf_iter_prog_supported(struct bpf_prog *prog); 2027 const struct bpf_func_proto * 2028 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); 2029 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog); 2030 int bpf_iter_new_fd(struct bpf_link *link); 2031 bool bpf_link_is_iter(struct bpf_link *link); 2032 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop); 2033 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx); 2034 void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux, 2035 struct seq_file *seq); 2036 int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, 2037 struct bpf_link_info *info); 2038 2039 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 2040 struct bpf_func_state *caller, 2041 struct bpf_func_state *callee); 2042 2043 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); 2044 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); 2045 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, 2046 u64 flags); 2047 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, 2048 u64 flags); 2049 2050 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); 2051 2052 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, 2053 void *key, void *value, u64 map_flags); 2054 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 2055 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, 2056 void *key, void *value, u64 map_flags); 2057 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 2058 2059 int bpf_get_file_flag(int flags); 2060 int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size, 2061 size_t actual_size); 2062 2063 /* verify correctness of eBPF program */ 2064 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr); 2065 2066 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 2067 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); 2068 #endif 2069 2070 struct btf *bpf_get_btf_vmlinux(void); 2071 2072 /* Map specifics */ 2073 struct xdp_frame; 2074 struct sk_buff; 2075 struct bpf_dtab_netdev; 2076 struct bpf_cpu_map_entry; 2077 2078 void __dev_flush(void); 2079 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, 2080 struct net_device *dev_rx); 2081 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, 2082 struct net_device *dev_rx); 2083 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, 2084 struct bpf_map *map, bool exclude_ingress); 2085 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, 2086 struct bpf_prog *xdp_prog); 2087 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, 2088 struct bpf_prog *xdp_prog, struct bpf_map *map, 2089 bool exclude_ingress); 2090 2091 void __cpu_map_flush(void); 2092 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, 2093 struct net_device *dev_rx); 2094 int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, 2095 struct sk_buff *skb); 2096 2097 /* Return map's numa specified by userspace */ 2098 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) 2099 { 2100 return (attr->map_flags & BPF_F_NUMA_NODE) ? 2101 attr->numa_node : NUMA_NO_NODE; 2102 } 2103 2104 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); 2105 int array_map_alloc_check(union bpf_attr *attr); 2106 2107 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 2108 union bpf_attr __user *uattr); 2109 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 2110 union bpf_attr __user *uattr); 2111 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 2112 const union bpf_attr *kattr, 2113 union bpf_attr __user *uattr); 2114 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 2115 const union bpf_attr *kattr, 2116 union bpf_attr __user *uattr); 2117 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, 2118 const union bpf_attr *kattr, 2119 union bpf_attr __user *uattr); 2120 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, 2121 const union bpf_attr *kattr, 2122 union bpf_attr __user *uattr); 2123 bool btf_ctx_access(int off, int size, enum bpf_access_type type, 2124 const struct bpf_prog *prog, 2125 struct bpf_insn_access_aux *info); 2126 2127 static inline bool bpf_tracing_ctx_access(int off, int size, 2128 enum bpf_access_type type) 2129 { 2130 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 2131 return false; 2132 if (type != BPF_READ) 2133 return false; 2134 if (off % size != 0) 2135 return false; 2136 return true; 2137 } 2138 2139 static inline bool bpf_tracing_btf_ctx_access(int off, int size, 2140 enum bpf_access_type type, 2141 const struct bpf_prog *prog, 2142 struct bpf_insn_access_aux *info) 2143 { 2144 if (!bpf_tracing_ctx_access(off, size, type)) 2145 return false; 2146 return btf_ctx_access(off, size, type, prog, info); 2147 } 2148 2149 int btf_struct_access(struct bpf_verifier_log *log, 2150 const struct bpf_reg_state *reg, 2151 int off, int size, enum bpf_access_type atype, 2152 u32 *next_btf_id, enum bpf_type_flag *flag); 2153 bool btf_struct_ids_match(struct bpf_verifier_log *log, 2154 const struct btf *btf, u32 id, int off, 2155 const struct btf *need_btf, u32 need_type_id, 2156 bool strict); 2157 2158 int btf_distill_func_proto(struct bpf_verifier_log *log, 2159 struct btf *btf, 2160 const struct btf_type *func_proto, 2161 const char *func_name, 2162 struct btf_func_model *m); 2163 2164 struct bpf_reg_state; 2165 int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, 2166 struct bpf_reg_state *regs); 2167 int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, 2168 struct bpf_reg_state *regs); 2169 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, 2170 struct bpf_reg_state *reg); 2171 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, 2172 struct btf *btf, const struct btf_type *t); 2173 2174 struct bpf_prog *bpf_prog_by_id(u32 id); 2175 struct bpf_link *bpf_link_by_id(u32 id); 2176 2177 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); 2178 void bpf_task_storage_free(struct task_struct *task); 2179 void bpf_cgrp_storage_free(struct cgroup *cgroup); 2180 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); 2181 const struct btf_func_model * 2182 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 2183 const struct bpf_insn *insn); 2184 struct bpf_core_ctx { 2185 struct bpf_verifier_log *log; 2186 const struct btf *btf; 2187 }; 2188 2189 int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo, 2190 int relo_idx, void *insn); 2191 2192 static inline bool unprivileged_ebpf_enabled(void) 2193 { 2194 return !sysctl_unprivileged_bpf_disabled; 2195 } 2196 2197 /* Not all bpf prog type has the bpf_ctx. 2198 * For the bpf prog type that has initialized the bpf_ctx, 2199 * this function can be used to decide if a kernel function 2200 * is called by a bpf program. 2201 */ 2202 static inline bool has_current_bpf_ctx(void) 2203 { 2204 return !!current->bpf_ctx; 2205 } 2206 2207 void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog); 2208 #else /* !CONFIG_BPF_SYSCALL */ 2209 static inline struct bpf_prog *bpf_prog_get(u32 ufd) 2210 { 2211 return ERR_PTR(-EOPNOTSUPP); 2212 } 2213 2214 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, 2215 enum bpf_prog_type type, 2216 bool attach_drv) 2217 { 2218 return ERR_PTR(-EOPNOTSUPP); 2219 } 2220 2221 static inline void bpf_prog_add(struct bpf_prog *prog, int i) 2222 { 2223 } 2224 2225 static inline void bpf_prog_sub(struct bpf_prog *prog, int i) 2226 { 2227 } 2228 2229 static inline void bpf_prog_put(struct bpf_prog *prog) 2230 { 2231 } 2232 2233 static inline void bpf_prog_inc(struct bpf_prog *prog) 2234 { 2235 } 2236 2237 static inline struct bpf_prog *__must_check 2238 bpf_prog_inc_not_zero(struct bpf_prog *prog) 2239 { 2240 return ERR_PTR(-EOPNOTSUPP); 2241 } 2242 2243 static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 2244 const struct bpf_link_ops *ops, 2245 struct bpf_prog *prog) 2246 { 2247 } 2248 2249 static inline int bpf_link_prime(struct bpf_link *link, 2250 struct bpf_link_primer *primer) 2251 { 2252 return -EOPNOTSUPP; 2253 } 2254 2255 static inline int bpf_link_settle(struct bpf_link_primer *primer) 2256 { 2257 return -EOPNOTSUPP; 2258 } 2259 2260 static inline void bpf_link_cleanup(struct bpf_link_primer *primer) 2261 { 2262 } 2263 2264 static inline void bpf_link_inc(struct bpf_link *link) 2265 { 2266 } 2267 2268 static inline void bpf_link_put(struct bpf_link *link) 2269 { 2270 } 2271 2272 static inline int bpf_obj_get_user(const char __user *pathname, int flags) 2273 { 2274 return -EOPNOTSUPP; 2275 } 2276 2277 static inline void __dev_flush(void) 2278 { 2279 } 2280 2281 struct xdp_frame; 2282 struct bpf_dtab_netdev; 2283 struct bpf_cpu_map_entry; 2284 2285 static inline 2286 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, 2287 struct net_device *dev_rx) 2288 { 2289 return 0; 2290 } 2291 2292 static inline 2293 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, 2294 struct net_device *dev_rx) 2295 { 2296 return 0; 2297 } 2298 2299 static inline 2300 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, 2301 struct bpf_map *map, bool exclude_ingress) 2302 { 2303 return 0; 2304 } 2305 2306 struct sk_buff; 2307 2308 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, 2309 struct sk_buff *skb, 2310 struct bpf_prog *xdp_prog) 2311 { 2312 return 0; 2313 } 2314 2315 static inline 2316 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, 2317 struct bpf_prog *xdp_prog, struct bpf_map *map, 2318 bool exclude_ingress) 2319 { 2320 return 0; 2321 } 2322 2323 static inline void __cpu_map_flush(void) 2324 { 2325 } 2326 2327 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, 2328 struct xdp_frame *xdpf, 2329 struct net_device *dev_rx) 2330 { 2331 return 0; 2332 } 2333 2334 static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, 2335 struct sk_buff *skb) 2336 { 2337 return -EOPNOTSUPP; 2338 } 2339 2340 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, 2341 enum bpf_prog_type type) 2342 { 2343 return ERR_PTR(-EOPNOTSUPP); 2344 } 2345 2346 static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog, 2347 const union bpf_attr *kattr, 2348 union bpf_attr __user *uattr) 2349 { 2350 return -ENOTSUPP; 2351 } 2352 2353 static inline int bpf_prog_test_run_skb(struct bpf_prog *prog, 2354 const union bpf_attr *kattr, 2355 union bpf_attr __user *uattr) 2356 { 2357 return -ENOTSUPP; 2358 } 2359 2360 static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog, 2361 const union bpf_attr *kattr, 2362 union bpf_attr __user *uattr) 2363 { 2364 return -ENOTSUPP; 2365 } 2366 2367 static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 2368 const union bpf_attr *kattr, 2369 union bpf_attr __user *uattr) 2370 { 2371 return -ENOTSUPP; 2372 } 2373 2374 static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, 2375 const union bpf_attr *kattr, 2376 union bpf_attr __user *uattr) 2377 { 2378 return -ENOTSUPP; 2379 } 2380 2381 static inline void bpf_map_put(struct bpf_map *map) 2382 { 2383 } 2384 2385 static inline struct bpf_prog *bpf_prog_by_id(u32 id) 2386 { 2387 return ERR_PTR(-ENOTSUPP); 2388 } 2389 2390 static inline int btf_struct_access(struct bpf_verifier_log *log, 2391 const struct bpf_reg_state *reg, 2392 int off, int size, enum bpf_access_type atype, 2393 u32 *next_btf_id, enum bpf_type_flag *flag) 2394 { 2395 return -EACCES; 2396 } 2397 2398 static inline const struct bpf_func_proto * 2399 bpf_base_func_proto(enum bpf_func_id func_id) 2400 { 2401 return NULL; 2402 } 2403 2404 static inline void bpf_task_storage_free(struct task_struct *task) 2405 { 2406 } 2407 2408 static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) 2409 { 2410 return false; 2411 } 2412 2413 static inline const struct btf_func_model * 2414 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 2415 const struct bpf_insn *insn) 2416 { 2417 return NULL; 2418 } 2419 2420 static inline bool unprivileged_ebpf_enabled(void) 2421 { 2422 return false; 2423 } 2424 2425 static inline bool has_current_bpf_ctx(void) 2426 { 2427 return false; 2428 } 2429 2430 static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog) 2431 { 2432 } 2433 2434 static inline void bpf_cgrp_storage_free(struct cgroup *cgroup) 2435 { 2436 } 2437 #endif /* CONFIG_BPF_SYSCALL */ 2438 2439 void __bpf_free_used_btfs(struct bpf_prog_aux *aux, 2440 struct btf_mod_pair *used_btfs, u32 len); 2441 2442 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, 2443 enum bpf_prog_type type) 2444 { 2445 return bpf_prog_get_type_dev(ufd, type, false); 2446 } 2447 2448 void __bpf_free_used_maps(struct bpf_prog_aux *aux, 2449 struct bpf_map **used_maps, u32 len); 2450 2451 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); 2452 2453 int bpf_prog_offload_compile(struct bpf_prog *prog); 2454 void bpf_prog_offload_destroy(struct bpf_prog *prog); 2455 int bpf_prog_offload_info_fill(struct bpf_prog_info *info, 2456 struct bpf_prog *prog); 2457 2458 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map); 2459 2460 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value); 2461 int bpf_map_offload_update_elem(struct bpf_map *map, 2462 void *key, void *value, u64 flags); 2463 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key); 2464 int bpf_map_offload_get_next_key(struct bpf_map *map, 2465 void *key, void *next_key); 2466 2467 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map); 2468 2469 struct bpf_offload_dev * 2470 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv); 2471 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev); 2472 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev); 2473 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, 2474 struct net_device *netdev); 2475 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, 2476 struct net_device *netdev); 2477 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); 2478 2479 void unpriv_ebpf_notify(int new_state); 2480 2481 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) 2482 int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr); 2483 2484 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) 2485 { 2486 return aux->offload_requested; 2487 } 2488 2489 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 2490 { 2491 return unlikely(map->ops == &bpf_map_offload_ops); 2492 } 2493 2494 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr); 2495 void bpf_map_offload_map_free(struct bpf_map *map); 2496 int bpf_prog_test_run_syscall(struct bpf_prog *prog, 2497 const union bpf_attr *kattr, 2498 union bpf_attr __user *uattr); 2499 2500 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); 2501 int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); 2502 int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags); 2503 int sock_map_bpf_prog_query(const union bpf_attr *attr, 2504 union bpf_attr __user *uattr); 2505 2506 void sock_map_unhash(struct sock *sk); 2507 void sock_map_destroy(struct sock *sk); 2508 void sock_map_close(struct sock *sk, long timeout); 2509 #else 2510 static inline int bpf_prog_offload_init(struct bpf_prog *prog, 2511 union bpf_attr *attr) 2512 { 2513 return -EOPNOTSUPP; 2514 } 2515 2516 static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux) 2517 { 2518 return false; 2519 } 2520 2521 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 2522 { 2523 return false; 2524 } 2525 2526 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) 2527 { 2528 return ERR_PTR(-EOPNOTSUPP); 2529 } 2530 2531 static inline void bpf_map_offload_map_free(struct bpf_map *map) 2532 { 2533 } 2534 2535 static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog, 2536 const union bpf_attr *kattr, 2537 union bpf_attr __user *uattr) 2538 { 2539 return -ENOTSUPP; 2540 } 2541 2542 #ifdef CONFIG_BPF_SYSCALL 2543 static inline int sock_map_get_from_fd(const union bpf_attr *attr, 2544 struct bpf_prog *prog) 2545 { 2546 return -EINVAL; 2547 } 2548 2549 static inline int sock_map_prog_detach(const union bpf_attr *attr, 2550 enum bpf_prog_type ptype) 2551 { 2552 return -EOPNOTSUPP; 2553 } 2554 2555 static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, 2556 u64 flags) 2557 { 2558 return -EOPNOTSUPP; 2559 } 2560 2561 static inline int sock_map_bpf_prog_query(const union bpf_attr *attr, 2562 union bpf_attr __user *uattr) 2563 { 2564 return -EINVAL; 2565 } 2566 #endif /* CONFIG_BPF_SYSCALL */ 2567 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ 2568 2569 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) 2570 void bpf_sk_reuseport_detach(struct sock *sk); 2571 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, 2572 void *value); 2573 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, 2574 void *value, u64 map_flags); 2575 #else 2576 static inline void bpf_sk_reuseport_detach(struct sock *sk) 2577 { 2578 } 2579 2580 #ifdef CONFIG_BPF_SYSCALL 2581 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, 2582 void *key, void *value) 2583 { 2584 return -EOPNOTSUPP; 2585 } 2586 2587 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, 2588 void *key, void *value, 2589 u64 map_flags) 2590 { 2591 return -EOPNOTSUPP; 2592 } 2593 #endif /* CONFIG_BPF_SYSCALL */ 2594 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ 2595 2596 /* verifier prototypes for helper functions called from eBPF programs */ 2597 extern const struct bpf_func_proto bpf_map_lookup_elem_proto; 2598 extern const struct bpf_func_proto bpf_map_update_elem_proto; 2599 extern const struct bpf_func_proto bpf_map_delete_elem_proto; 2600 extern const struct bpf_func_proto bpf_map_push_elem_proto; 2601 extern const struct bpf_func_proto bpf_map_pop_elem_proto; 2602 extern const struct bpf_func_proto bpf_map_peek_elem_proto; 2603 extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto; 2604 2605 extern const struct bpf_func_proto bpf_get_prandom_u32_proto; 2606 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; 2607 extern const struct bpf_func_proto bpf_get_numa_node_id_proto; 2608 extern const struct bpf_func_proto bpf_tail_call_proto; 2609 extern const struct bpf_func_proto bpf_ktime_get_ns_proto; 2610 extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto; 2611 extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto; 2612 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; 2613 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; 2614 extern const struct bpf_func_proto bpf_get_current_comm_proto; 2615 extern const struct bpf_func_proto bpf_get_stackid_proto; 2616 extern const struct bpf_func_proto bpf_get_stack_proto; 2617 extern const struct bpf_func_proto bpf_get_task_stack_proto; 2618 extern const struct bpf_func_proto bpf_get_stackid_proto_pe; 2619 extern const struct bpf_func_proto bpf_get_stack_proto_pe; 2620 extern const struct bpf_func_proto bpf_sock_map_update_proto; 2621 extern const struct bpf_func_proto bpf_sock_hash_update_proto; 2622 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; 2623 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto; 2624 extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto; 2625 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; 2626 extern const struct bpf_func_proto bpf_msg_redirect_map_proto; 2627 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; 2628 extern const struct bpf_func_proto bpf_sk_redirect_map_proto; 2629 extern const struct bpf_func_proto bpf_spin_lock_proto; 2630 extern const struct bpf_func_proto bpf_spin_unlock_proto; 2631 extern const struct bpf_func_proto bpf_get_local_storage_proto; 2632 extern const struct bpf_func_proto bpf_strtol_proto; 2633 extern const struct bpf_func_proto bpf_strtoul_proto; 2634 extern const struct bpf_func_proto bpf_tcp_sock_proto; 2635 extern const struct bpf_func_proto bpf_jiffies64_proto; 2636 extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto; 2637 extern const struct bpf_func_proto bpf_event_output_data_proto; 2638 extern const struct bpf_func_proto bpf_ringbuf_output_proto; 2639 extern const struct bpf_func_proto bpf_ringbuf_reserve_proto; 2640 extern const struct bpf_func_proto bpf_ringbuf_submit_proto; 2641 extern const struct bpf_func_proto bpf_ringbuf_discard_proto; 2642 extern const struct bpf_func_proto bpf_ringbuf_query_proto; 2643 extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto; 2644 extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto; 2645 extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto; 2646 extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto; 2647 extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto; 2648 extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto; 2649 extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto; 2650 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto; 2651 extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto; 2652 extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto; 2653 extern const struct bpf_func_proto bpf_copy_from_user_proto; 2654 extern const struct bpf_func_proto bpf_snprintf_btf_proto; 2655 extern const struct bpf_func_proto bpf_snprintf_proto; 2656 extern const struct bpf_func_proto bpf_per_cpu_ptr_proto; 2657 extern const struct bpf_func_proto bpf_this_cpu_ptr_proto; 2658 extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto; 2659 extern const struct bpf_func_proto bpf_sock_from_file_proto; 2660 extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto; 2661 extern const struct bpf_func_proto bpf_task_storage_get_recur_proto; 2662 extern const struct bpf_func_proto bpf_task_storage_get_proto; 2663 extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto; 2664 extern const struct bpf_func_proto bpf_task_storage_delete_proto; 2665 extern const struct bpf_func_proto bpf_for_each_map_elem_proto; 2666 extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto; 2667 extern const struct bpf_func_proto bpf_sk_setsockopt_proto; 2668 extern const struct bpf_func_proto bpf_sk_getsockopt_proto; 2669 extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto; 2670 extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto; 2671 extern const struct bpf_func_proto bpf_find_vma_proto; 2672 extern const struct bpf_func_proto bpf_loop_proto; 2673 extern const struct bpf_func_proto bpf_copy_from_user_task_proto; 2674 extern const struct bpf_func_proto bpf_set_retval_proto; 2675 extern const struct bpf_func_proto bpf_get_retval_proto; 2676 extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto; 2677 extern const struct bpf_func_proto bpf_cgrp_storage_get_proto; 2678 extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto; 2679 2680 const struct bpf_func_proto *tracing_prog_func_proto( 2681 enum bpf_func_id func_id, const struct bpf_prog *prog); 2682 2683 /* Shared helpers among cBPF and eBPF. */ 2684 void bpf_user_rnd_init_once(void); 2685 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 2686 u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 2687 2688 #if defined(CONFIG_NET) 2689 bool bpf_sock_common_is_valid_access(int off, int size, 2690 enum bpf_access_type type, 2691 struct bpf_insn_access_aux *info); 2692 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, 2693 struct bpf_insn_access_aux *info); 2694 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 2695 const struct bpf_insn *si, 2696 struct bpf_insn *insn_buf, 2697 struct bpf_prog *prog, 2698 u32 *target_size); 2699 #else 2700 static inline bool bpf_sock_common_is_valid_access(int off, int size, 2701 enum bpf_access_type type, 2702 struct bpf_insn_access_aux *info) 2703 { 2704 return false; 2705 } 2706 static inline bool bpf_sock_is_valid_access(int off, int size, 2707 enum bpf_access_type type, 2708 struct bpf_insn_access_aux *info) 2709 { 2710 return false; 2711 } 2712 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 2713 const struct bpf_insn *si, 2714 struct bpf_insn *insn_buf, 2715 struct bpf_prog *prog, 2716 u32 *target_size) 2717 { 2718 return 0; 2719 } 2720 #endif 2721 2722 #ifdef CONFIG_INET 2723 struct sk_reuseport_kern { 2724 struct sk_buff *skb; 2725 struct sock *sk; 2726 struct sock *selected_sk; 2727 struct sock *migrating_sk; 2728 void *data_end; 2729 u32 hash; 2730 u32 reuseport_id; 2731 bool bind_inany; 2732 }; 2733 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 2734 struct bpf_insn_access_aux *info); 2735 2736 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 2737 const struct bpf_insn *si, 2738 struct bpf_insn *insn_buf, 2739 struct bpf_prog *prog, 2740 u32 *target_size); 2741 2742 bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 2743 struct bpf_insn_access_aux *info); 2744 2745 u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 2746 const struct bpf_insn *si, 2747 struct bpf_insn *insn_buf, 2748 struct bpf_prog *prog, 2749 u32 *target_size); 2750 #else 2751 static inline bool bpf_tcp_sock_is_valid_access(int off, int size, 2752 enum bpf_access_type type, 2753 struct bpf_insn_access_aux *info) 2754 { 2755 return false; 2756 } 2757 2758 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 2759 const struct bpf_insn *si, 2760 struct bpf_insn *insn_buf, 2761 struct bpf_prog *prog, 2762 u32 *target_size) 2763 { 2764 return 0; 2765 } 2766 static inline bool bpf_xdp_sock_is_valid_access(int off, int size, 2767 enum bpf_access_type type, 2768 struct bpf_insn_access_aux *info) 2769 { 2770 return false; 2771 } 2772 2773 static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 2774 const struct bpf_insn *si, 2775 struct bpf_insn *insn_buf, 2776 struct bpf_prog *prog, 2777 u32 *target_size) 2778 { 2779 return 0; 2780 } 2781 #endif /* CONFIG_INET */ 2782 2783 enum bpf_text_poke_type { 2784 BPF_MOD_CALL, 2785 BPF_MOD_JUMP, 2786 }; 2787 2788 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 2789 void *addr1, void *addr2); 2790 2791 void *bpf_arch_text_copy(void *dst, void *src, size_t len); 2792 int bpf_arch_text_invalidate(void *dst, size_t len); 2793 2794 struct btf_id_set; 2795 bool btf_id_set_contains(const struct btf_id_set *set, u32 id); 2796 2797 #define MAX_BPRINTF_VARARGS 12 2798 2799 int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args, 2800 u32 **bin_buf, u32 num_args); 2801 void bpf_bprintf_cleanup(void); 2802 2803 /* the implementation of the opaque uapi struct bpf_dynptr */ 2804 struct bpf_dynptr_kern { 2805 void *data; 2806 /* Size represents the number of usable bytes of dynptr data. 2807 * If for example the offset is at 4 for a local dynptr whose data is 2808 * of type u64, the number of usable bytes is 4. 2809 * 2810 * The upper 8 bits are reserved. It is as follows: 2811 * Bits 0 - 23 = size 2812 * Bits 24 - 30 = dynptr type 2813 * Bit 31 = whether dynptr is read-only 2814 */ 2815 u32 size; 2816 u32 offset; 2817 } __aligned(8); 2818 2819 enum bpf_dynptr_type { 2820 BPF_DYNPTR_TYPE_INVALID, 2821 /* Points to memory that is local to the bpf program */ 2822 BPF_DYNPTR_TYPE_LOCAL, 2823 /* Underlying data is a kernel-produced ringbuf record */ 2824 BPF_DYNPTR_TYPE_RINGBUF, 2825 }; 2826 2827 void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data, 2828 enum bpf_dynptr_type type, u32 offset, u32 size); 2829 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr); 2830 int bpf_dynptr_check_size(u32 size); 2831 u32 bpf_dynptr_get_size(const struct bpf_dynptr_kern *ptr); 2832 2833 #ifdef CONFIG_BPF_LSM 2834 void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype); 2835 void bpf_cgroup_atype_put(int cgroup_atype); 2836 #else 2837 static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {} 2838 static inline void bpf_cgroup_atype_put(int cgroup_atype) {} 2839 #endif /* CONFIG_BPF_LSM */ 2840 2841 struct key; 2842 2843 #ifdef CONFIG_KEYS 2844 struct bpf_key { 2845 struct key *key; 2846 bool has_ref; 2847 }; 2848 #endif /* CONFIG_KEYS */ 2849 2850 static inline bool type_is_alloc(u32 type) 2851 { 2852 return type & MEM_ALLOC; 2853 } 2854 2855 #endif /* _LINUX_BPF_H */ 2856