1 /* 2 * BPF Jit compiler for s390. 3 * 4 * Copyright IBM Corp. 2012 5 * 6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 7 */ 8 #include <linux/moduleloader.h> 9 #include <linux/netdevice.h> 10 #include <linux/filter.h> 11 #include <asm/cacheflush.h> 12 #include <asm/processor.h> 13 #include <asm/facility.h> 14 15 /* 16 * Conventions: 17 * %r2 = skb pointer 18 * %r3 = offset parameter 19 * %r4 = scratch register / length parameter 20 * %r5 = BPF A accumulator 21 * %r8 = return address 22 * %r9 = save register for skb pointer 23 * %r10 = skb->data 24 * %r11 = skb->len - skb->data_len (headlen) 25 * %r12 = BPF X accumulator 26 * %r13 = literal pool pointer 27 * 0(%r15) - 63(%r15) scratch memory array with BPF_MEMWORDS 28 */ 29 int bpf_jit_enable __read_mostly; 30 31 /* 32 * assembly code in arch/x86/net/bpf_jit.S 33 */ 34 extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[]; 35 extern u8 sk_load_word_ind[], sk_load_half_ind[], sk_load_byte_ind[]; 36 37 struct bpf_jit { 38 unsigned int seen; 39 u8 *start; 40 u8 *prg; 41 u8 *mid; 42 u8 *lit; 43 u8 *end; 44 u8 *base_ip; 45 u8 *ret0_ip; 46 u8 *exit_ip; 47 unsigned int off_load_word; 48 unsigned int off_load_half; 49 unsigned int off_load_byte; 50 unsigned int off_load_bmsh; 51 unsigned int off_load_iword; 52 unsigned int off_load_ihalf; 53 unsigned int off_load_ibyte; 54 }; 55 56 #define BPF_SIZE_MAX 4096 /* Max size for program */ 57 58 #define SEEN_DATAREF 1 /* might call external helpers */ 59 #define SEEN_XREG 2 /* ebx is used */ 60 #define SEEN_MEM 4 /* use mem[] for temporary storage */ 61 #define SEEN_RET0 8 /* pc_ret0 points to a valid return 0 */ 62 #define SEEN_LITERAL 16 /* code uses literals */ 63 #define SEEN_LOAD_WORD 32 /* code uses sk_load_word */ 64 #define SEEN_LOAD_HALF 64 /* code uses sk_load_half */ 65 #define SEEN_LOAD_BYTE 128 /* code uses sk_load_byte */ 66 #define SEEN_LOAD_BMSH 256 /* code uses sk_load_byte_msh */ 67 #define SEEN_LOAD_IWORD 512 /* code uses sk_load_word_ind */ 68 #define SEEN_LOAD_IHALF 1024 /* code uses sk_load_half_ind */ 69 #define SEEN_LOAD_IBYTE 2048 /* code uses sk_load_byte_ind */ 70 71 #define EMIT2(op) \ 72 ({ \ 73 if (jit->prg + 2 <= jit->mid) \ 74 *(u16 *) jit->prg = op; \ 75 jit->prg += 2; \ 76 }) 77 78 #define EMIT4(op) \ 79 ({ \ 80 if (jit->prg + 4 <= jit->mid) \ 81 *(u32 *) jit->prg = op; \ 82 jit->prg += 4; \ 83 }) 84 85 #define EMIT4_DISP(op, disp) \ 86 ({ \ 87 unsigned int __disp = (disp) & 0xfff; \ 88 EMIT4(op | __disp); \ 89 }) 90 91 #define EMIT4_IMM(op, imm) \ 92 ({ \ 93 unsigned int __imm = (imm) & 0xffff; \ 94 EMIT4(op | __imm); \ 95 }) 96 97 #define EMIT4_PCREL(op, pcrel) \ 98 ({ \ 99 long __pcrel = ((pcrel) >> 1) & 0xffff; \ 100 EMIT4(op | __pcrel); \ 101 }) 102 103 #define EMIT6(op1, op2) \ 104 ({ \ 105 if (jit->prg + 6 <= jit->mid) { \ 106 *(u32 *) jit->prg = op1; \ 107 *(u16 *) (jit->prg + 4) = op2; \ 108 } \ 109 jit->prg += 6; \ 110 }) 111 112 #define EMIT6_DISP(op1, op2, disp) \ 113 ({ \ 114 unsigned int __disp = (disp) & 0xfff; \ 115 EMIT6(op1 | __disp, op2); \ 116 }) 117 118 #define EMIT6_IMM(op, imm) \ 119 ({ \ 120 unsigned int __imm = (imm); \ 121 EMIT6(op | (__imm >> 16), __imm & 0xffff); \ 122 }) 123 124 #define EMIT_CONST(val) \ 125 ({ \ 126 unsigned int ret; \ 127 ret = (unsigned int) (jit->lit - jit->base_ip); \ 128 jit->seen |= SEEN_LITERAL; \ 129 if (jit->lit + 4 <= jit->end) \ 130 *(u32 *) jit->lit = val; \ 131 jit->lit += 4; \ 132 ret; \ 133 }) 134 135 #define EMIT_FN_CONST(bit, fn) \ 136 ({ \ 137 unsigned int ret; \ 138 ret = (unsigned int) (jit->lit - jit->base_ip); \ 139 if (jit->seen & bit) { \ 140 jit->seen |= SEEN_LITERAL; \ 141 if (jit->lit + 8 <= jit->end) \ 142 *(void **) jit->lit = fn; \ 143 jit->lit += 8; \ 144 } \ 145 ret; \ 146 }) 147 148 static void bpf_jit_prologue(struct bpf_jit *jit) 149 { 150 /* Save registers and create stack frame if necessary */ 151 if (jit->seen & SEEN_DATAREF) { 152 /* stmg %r8,%r15,88(%r15) */ 153 EMIT6(0xeb8ff058, 0x0024); 154 /* lgr %r14,%r15 */ 155 EMIT4(0xb90400ef); 156 /* ahi %r15,<offset> */ 157 EMIT4_IMM(0xa7fa0000, (jit->seen & SEEN_MEM) ? -112 : -80); 158 /* stg %r14,152(%r15) */ 159 EMIT6(0xe3e0f098, 0x0024); 160 } else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) 161 /* stmg %r12,%r13,120(%r15) */ 162 EMIT6(0xebcdf078, 0x0024); 163 else if (jit->seen & SEEN_XREG) 164 /* stg %r12,120(%r15) */ 165 EMIT6(0xe3c0f078, 0x0024); 166 else if (jit->seen & SEEN_LITERAL) 167 /* stg %r13,128(%r15) */ 168 EMIT6(0xe3d0f080, 0x0024); 169 170 /* Setup literal pool */ 171 if (jit->seen & SEEN_LITERAL) { 172 /* basr %r13,0 */ 173 EMIT2(0x0dd0); 174 jit->base_ip = jit->prg; 175 } 176 jit->off_load_word = EMIT_FN_CONST(SEEN_LOAD_WORD, sk_load_word); 177 jit->off_load_half = EMIT_FN_CONST(SEEN_LOAD_HALF, sk_load_half); 178 jit->off_load_byte = EMIT_FN_CONST(SEEN_LOAD_BYTE, sk_load_byte); 179 jit->off_load_bmsh = EMIT_FN_CONST(SEEN_LOAD_BMSH, sk_load_byte_msh); 180 jit->off_load_iword = EMIT_FN_CONST(SEEN_LOAD_IWORD, sk_load_word_ind); 181 jit->off_load_ihalf = EMIT_FN_CONST(SEEN_LOAD_IHALF, sk_load_half_ind); 182 jit->off_load_ibyte = EMIT_FN_CONST(SEEN_LOAD_IBYTE, sk_load_byte_ind); 183 184 /* Filter needs to access skb data */ 185 if (jit->seen & SEEN_DATAREF) { 186 /* l %r11,<len>(%r2) */ 187 EMIT4_DISP(0x58b02000, offsetof(struct sk_buff, len)); 188 /* s %r11,<data_len>(%r2) */ 189 EMIT4_DISP(0x5bb02000, offsetof(struct sk_buff, data_len)); 190 /* lg %r10,<data>(%r2) */ 191 EMIT6_DISP(0xe3a02000, 0x0004, 192 offsetof(struct sk_buff, data)); 193 } 194 } 195 196 static void bpf_jit_epilogue(struct bpf_jit *jit) 197 { 198 /* Return 0 */ 199 if (jit->seen & SEEN_RET0) { 200 jit->ret0_ip = jit->prg; 201 /* lghi %r2,0 */ 202 EMIT4(0xa7290000); 203 } 204 jit->exit_ip = jit->prg; 205 /* Restore registers */ 206 if (jit->seen & SEEN_DATAREF) 207 /* lmg %r8,%r15,<offset>(%r15) */ 208 EMIT6_DISP(0xeb8ff000, 0x0004, 209 (jit->seen & SEEN_MEM) ? 200 : 168); 210 else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) 211 /* lmg %r12,%r13,120(%r15) */ 212 EMIT6(0xebcdf078, 0x0004); 213 else if (jit->seen & SEEN_XREG) 214 /* lg %r12,120(%r15) */ 215 EMIT6(0xe3c0f078, 0x0004); 216 else if (jit->seen & SEEN_LITERAL) 217 /* lg %r13,128(%r15) */ 218 EMIT6(0xe3d0f080, 0x0004); 219 /* br %r14 */ 220 EMIT2(0x07fe); 221 } 222 223 /* 224 * make sure we dont leak kernel information to user 225 */ 226 static void bpf_jit_noleaks(struct bpf_jit *jit, struct sock_filter *filter) 227 { 228 /* Clear temporary memory if (seen & SEEN_MEM) */ 229 if (jit->seen & SEEN_MEM) 230 /* xc 0(64,%r15),0(%r15) */ 231 EMIT6(0xd73ff000, 0xf000); 232 /* Clear X if (seen & SEEN_XREG) */ 233 if (jit->seen & SEEN_XREG) 234 /* lhi %r12,0 */ 235 EMIT4(0xa7c80000); 236 /* Clear A if the first register does not set it. */ 237 switch (filter[0].code) { 238 case BPF_S_LD_W_ABS: 239 case BPF_S_LD_H_ABS: 240 case BPF_S_LD_B_ABS: 241 case BPF_S_LD_W_LEN: 242 case BPF_S_LD_W_IND: 243 case BPF_S_LD_H_IND: 244 case BPF_S_LD_B_IND: 245 case BPF_S_LDX_B_MSH: 246 case BPF_S_LD_IMM: 247 case BPF_S_LD_MEM: 248 case BPF_S_MISC_TXA: 249 case BPF_S_ANC_PROTOCOL: 250 case BPF_S_ANC_PKTTYPE: 251 case BPF_S_ANC_IFINDEX: 252 case BPF_S_ANC_MARK: 253 case BPF_S_ANC_QUEUE: 254 case BPF_S_ANC_HATYPE: 255 case BPF_S_ANC_RXHASH: 256 case BPF_S_ANC_CPU: 257 case BPF_S_RET_K: 258 /* first instruction sets A register */ 259 break; 260 default: /* A = 0 */ 261 /* lhi %r5,0 */ 262 EMIT4(0xa7580000); 263 } 264 } 265 266 static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter, 267 unsigned int *addrs, int i, int last) 268 { 269 unsigned int K; 270 int offset; 271 unsigned int mask; 272 273 K = filter->k; 274 switch (filter->code) { 275 case BPF_S_ALU_ADD_X: /* A += X */ 276 jit->seen |= SEEN_XREG; 277 /* ar %r5,%r12 */ 278 EMIT2(0x1a5c); 279 break; 280 case BPF_S_ALU_ADD_K: /* A += K */ 281 if (!K) 282 break; 283 if (K <= 16383) 284 /* ahi %r5,<K> */ 285 EMIT4_IMM(0xa75a0000, K); 286 else if (test_facility(21)) 287 /* alfi %r5,<K> */ 288 EMIT6_IMM(0xc25b0000, K); 289 else 290 /* a %r5,<d(K)>(%r13) */ 291 EMIT4_DISP(0x5a50d000, EMIT_CONST(K)); 292 break; 293 case BPF_S_ALU_SUB_X: /* A -= X */ 294 jit->seen |= SEEN_XREG; 295 /* sr %r5,%r12 */ 296 EMIT2(0x1b5c); 297 break; 298 case BPF_S_ALU_SUB_K: /* A -= K */ 299 if (!K) 300 break; 301 if (K <= 16384) 302 /* ahi %r5,-K */ 303 EMIT4_IMM(0xa75a0000, -K); 304 else if (test_facility(21)) 305 /* alfi %r5,-K */ 306 EMIT6_IMM(0xc25b0000, -K); 307 else 308 /* s %r5,<d(K)>(%r13) */ 309 EMIT4_DISP(0x5b50d000, EMIT_CONST(K)); 310 break; 311 case BPF_S_ALU_MUL_X: /* A *= X */ 312 jit->seen |= SEEN_XREG; 313 /* msr %r5,%r12 */ 314 EMIT4(0xb252005c); 315 break; 316 case BPF_S_ALU_MUL_K: /* A *= K */ 317 if (K <= 16383) 318 /* mhi %r5,K */ 319 EMIT4_IMM(0xa75c0000, K); 320 else if (test_facility(34)) 321 /* msfi %r5,<K> */ 322 EMIT6_IMM(0xc2510000, K); 323 else 324 /* ms %r5,<d(K)>(%r13) */ 325 EMIT4_DISP(0x7150d000, EMIT_CONST(K)); 326 break; 327 case BPF_S_ALU_DIV_X: /* A /= X */ 328 jit->seen |= SEEN_XREG | SEEN_RET0; 329 /* ltr %r12,%r12 */ 330 EMIT2(0x12cc); 331 /* jz <ret0> */ 332 EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); 333 /* lhi %r4,0 */ 334 EMIT4(0xa7480000); 335 /* dr %r4,%r12 */ 336 EMIT2(0x1d4c); 337 break; 338 case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K) */ 339 /* m %r4,<d(K)>(%r13) */ 340 EMIT4_DISP(0x5c40d000, EMIT_CONST(K)); 341 /* lr %r5,%r4 */ 342 EMIT2(0x1854); 343 break; 344 case BPF_S_ALU_AND_X: /* A &= X */ 345 jit->seen |= SEEN_XREG; 346 /* nr %r5,%r12 */ 347 EMIT2(0x145c); 348 break; 349 case BPF_S_ALU_AND_K: /* A &= K */ 350 if (test_facility(21)) 351 /* nilf %r5,<K> */ 352 EMIT6_IMM(0xc05b0000, K); 353 else 354 /* n %r5,<d(K)>(%r13) */ 355 EMIT4_DISP(0x5450d000, EMIT_CONST(K)); 356 break; 357 case BPF_S_ALU_OR_X: /* A |= X */ 358 jit->seen |= SEEN_XREG; 359 /* or %r5,%r12 */ 360 EMIT2(0x165c); 361 break; 362 case BPF_S_ALU_OR_K: /* A |= K */ 363 if (test_facility(21)) 364 /* oilf %r5,<K> */ 365 EMIT6_IMM(0xc05d0000, K); 366 else 367 /* o %r5,<d(K)>(%r13) */ 368 EMIT4_DISP(0x5650d000, EMIT_CONST(K)); 369 break; 370 case BPF_S_ANC_ALU_XOR_X: /* A ^= X; */ 371 jit->seen |= SEEN_XREG; 372 /* xr %r5,%r12 */ 373 EMIT2(0x175c); 374 break; 375 case BPF_S_ALU_LSH_X: /* A <<= X; */ 376 jit->seen |= SEEN_XREG; 377 /* sll %r5,0(%r12) */ 378 EMIT4(0x8950c000); 379 break; 380 case BPF_S_ALU_LSH_K: /* A <<= K */ 381 if (K == 0) 382 break; 383 /* sll %r5,K */ 384 EMIT4_DISP(0x89500000, K); 385 break; 386 case BPF_S_ALU_RSH_X: /* A >>= X; */ 387 jit->seen |= SEEN_XREG; 388 /* srl %r5,0(%r12) */ 389 EMIT4(0x8850c000); 390 break; 391 case BPF_S_ALU_RSH_K: /* A >>= K; */ 392 if (K == 0) 393 break; 394 /* srl %r5,K */ 395 EMIT4_DISP(0x88500000, K); 396 break; 397 case BPF_S_ALU_NEG: /* A = -A */ 398 /* lnr %r5,%r5 */ 399 EMIT2(0x1155); 400 break; 401 case BPF_S_JMP_JA: /* ip += K */ 402 offset = addrs[i + K] + jit->start - jit->prg; 403 EMIT4_PCREL(0xa7f40000, offset); 404 break; 405 case BPF_S_JMP_JGT_K: /* ip += (A > K) ? jt : jf */ 406 mask = 0x200000; /* jh */ 407 goto kbranch; 408 case BPF_S_JMP_JGE_K: /* ip += (A >= K) ? jt : jf */ 409 mask = 0xa00000; /* jhe */ 410 goto kbranch; 411 case BPF_S_JMP_JEQ_K: /* ip += (A == K) ? jt : jf */ 412 mask = 0x800000; /* je */ 413 kbranch: /* Emit compare if the branch targets are different */ 414 if (filter->jt != filter->jf) { 415 if (K <= 16383) 416 /* chi %r5,<K> */ 417 EMIT4_IMM(0xa75e0000, K); 418 else if (test_facility(21)) 419 /* clfi %r5,<K> */ 420 EMIT6_IMM(0xc25f0000, K); 421 else 422 /* c %r5,<d(K)>(%r13) */ 423 EMIT4_DISP(0x5950d000, EMIT_CONST(K)); 424 } 425 branch: if (filter->jt == filter->jf) { 426 if (filter->jt == 0) 427 break; 428 /* j <jt> */ 429 offset = addrs[i + filter->jt] + jit->start - jit->prg; 430 EMIT4_PCREL(0xa7f40000, offset); 431 break; 432 } 433 if (filter->jt != 0) { 434 /* brc <mask>,<jt> */ 435 offset = addrs[i + filter->jt] + jit->start - jit->prg; 436 EMIT4_PCREL(0xa7040000 | mask, offset); 437 } 438 if (filter->jf != 0) { 439 /* brc <mask^15>,<jf> */ 440 offset = addrs[i + filter->jf] + jit->start - jit->prg; 441 EMIT4_PCREL(0xa7040000 | (mask ^ 0xf00000), offset); 442 } 443 break; 444 case BPF_S_JMP_JSET_K: /* ip += (A & K) ? jt : jf */ 445 mask = 0x700000; /* jnz */ 446 /* Emit test if the branch targets are different */ 447 if (filter->jt != filter->jf) { 448 if (K > 65535) { 449 /* lr %r4,%r5 */ 450 EMIT2(0x1845); 451 /* n %r4,<d(K)>(%r13) */ 452 EMIT4_DISP(0x5440d000, EMIT_CONST(K)); 453 } else 454 /* tmll %r5,K */ 455 EMIT4_IMM(0xa7510000, K); 456 } 457 goto branch; 458 case BPF_S_JMP_JGT_X: /* ip += (A > X) ? jt : jf */ 459 mask = 0x200000; /* jh */ 460 goto xbranch; 461 case BPF_S_JMP_JGE_X: /* ip += (A >= X) ? jt : jf */ 462 mask = 0xa00000; /* jhe */ 463 goto xbranch; 464 case BPF_S_JMP_JEQ_X: /* ip += (A == X) ? jt : jf */ 465 mask = 0x800000; /* je */ 466 xbranch: /* Emit compare if the branch targets are different */ 467 if (filter->jt != filter->jf) { 468 jit->seen |= SEEN_XREG; 469 /* cr %r5,%r12 */ 470 EMIT2(0x195c); 471 } 472 goto branch; 473 case BPF_S_JMP_JSET_X: /* ip += (A & X) ? jt : jf */ 474 mask = 0x700000; /* jnz */ 475 /* Emit test if the branch targets are different */ 476 if (filter->jt != filter->jf) { 477 jit->seen |= SEEN_XREG; 478 /* lr %r4,%r5 */ 479 EMIT2(0x1845); 480 /* nr %r4,%r12 */ 481 EMIT2(0x144c); 482 } 483 goto branch; 484 case BPF_S_LD_W_ABS: /* A = *(u32 *) (skb->data+K) */ 485 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_WORD; 486 offset = jit->off_load_word; 487 goto load_abs; 488 case BPF_S_LD_H_ABS: /* A = *(u16 *) (skb->data+K) */ 489 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_HALF; 490 offset = jit->off_load_half; 491 goto load_abs; 492 case BPF_S_LD_B_ABS: /* A = *(u8 *) (skb->data+K) */ 493 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_BYTE; 494 offset = jit->off_load_byte; 495 load_abs: if ((int) K < 0) 496 goto out; 497 call_fn: /* lg %r1,<d(function)>(%r13) */ 498 EMIT6_DISP(0xe310d000, 0x0004, offset); 499 /* l %r3,<d(K)>(%r13) */ 500 EMIT4_DISP(0x5830d000, EMIT_CONST(K)); 501 /* basr %r8,%r1 */ 502 EMIT2(0x0d81); 503 /* jnz <ret0> */ 504 EMIT4_PCREL(0xa7740000, (jit->ret0_ip - jit->prg)); 505 break; 506 case BPF_S_LD_W_IND: /* A = *(u32 *) (skb->data+K+X) */ 507 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IWORD; 508 offset = jit->off_load_iword; 509 goto call_fn; 510 case BPF_S_LD_H_IND: /* A = *(u16 *) (skb->data+K+X) */ 511 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IHALF; 512 offset = jit->off_load_ihalf; 513 goto call_fn; 514 case BPF_S_LD_B_IND: /* A = *(u8 *) (skb->data+K+X) */ 515 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IBYTE; 516 offset = jit->off_load_ibyte; 517 goto call_fn; 518 case BPF_S_LDX_B_MSH: 519 /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */ 520 jit->seen |= SEEN_RET0; 521 if ((int) K < 0) { 522 /* j <ret0> */ 523 EMIT4_PCREL(0xa7f40000, (jit->ret0_ip - jit->prg)); 524 break; 525 } 526 jit->seen |= SEEN_DATAREF | SEEN_LOAD_BMSH; 527 offset = jit->off_load_bmsh; 528 goto call_fn; 529 case BPF_S_LD_W_LEN: /* A = skb->len; */ 530 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); 531 /* l %r5,<d(len)>(%r2) */ 532 EMIT4_DISP(0x58502000, offsetof(struct sk_buff, len)); 533 break; 534 case BPF_S_LDX_W_LEN: /* X = skb->len; */ 535 jit->seen |= SEEN_XREG; 536 /* l %r12,<d(len)>(%r2) */ 537 EMIT4_DISP(0x58c02000, offsetof(struct sk_buff, len)); 538 break; 539 case BPF_S_LD_IMM: /* A = K */ 540 if (K <= 16383) 541 /* lhi %r5,K */ 542 EMIT4_IMM(0xa7580000, K); 543 else if (test_facility(21)) 544 /* llilf %r5,<K> */ 545 EMIT6_IMM(0xc05f0000, K); 546 else 547 /* l %r5,<d(K)>(%r13) */ 548 EMIT4_DISP(0x5850d000, EMIT_CONST(K)); 549 break; 550 case BPF_S_LDX_IMM: /* X = K */ 551 jit->seen |= SEEN_XREG; 552 if (K <= 16383) 553 /* lhi %r12,<K> */ 554 EMIT4_IMM(0xa7c80000, K); 555 else if (test_facility(21)) 556 /* llilf %r12,<K> */ 557 EMIT6_IMM(0xc0cf0000, K); 558 else 559 /* l %r12,<d(K)>(%r13) */ 560 EMIT4_DISP(0x58c0d000, EMIT_CONST(K)); 561 break; 562 case BPF_S_LD_MEM: /* A = mem[K] */ 563 jit->seen |= SEEN_MEM; 564 /* l %r5,<K>(%r15) */ 565 EMIT4_DISP(0x5850f000, 566 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 567 break; 568 case BPF_S_LDX_MEM: /* X = mem[K] */ 569 jit->seen |= SEEN_XREG | SEEN_MEM; 570 /* l %r12,<K>(%r15) */ 571 EMIT4_DISP(0x58c0f000, 572 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 573 break; 574 case BPF_S_MISC_TAX: /* X = A */ 575 jit->seen |= SEEN_XREG; 576 /* lr %r12,%r5 */ 577 EMIT2(0x18c5); 578 break; 579 case BPF_S_MISC_TXA: /* A = X */ 580 jit->seen |= SEEN_XREG; 581 /* lr %r5,%r12 */ 582 EMIT2(0x185c); 583 break; 584 case BPF_S_RET_K: 585 if (K == 0) { 586 jit->seen |= SEEN_RET0; 587 if (last) 588 break; 589 /* j <ret0> */ 590 EMIT4_PCREL(0xa7f40000, jit->ret0_ip - jit->prg); 591 } else { 592 if (K <= 16383) 593 /* lghi %r2,K */ 594 EMIT4_IMM(0xa7290000, K); 595 else 596 /* llgf %r2,<K>(%r13) */ 597 EMIT6_DISP(0xe320d000, 0x0016, EMIT_CONST(K)); 598 /* j <exit> */ 599 if (last && !(jit->seen & SEEN_RET0)) 600 break; 601 EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); 602 } 603 break; 604 case BPF_S_RET_A: 605 /* llgfr %r2,%r5 */ 606 EMIT4(0xb9160025); 607 /* j <exit> */ 608 EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); 609 break; 610 case BPF_S_ST: /* mem[K] = A */ 611 jit->seen |= SEEN_MEM; 612 /* st %r5,<K>(%r15) */ 613 EMIT4_DISP(0x5050f000, 614 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 615 break; 616 case BPF_S_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */ 617 jit->seen |= SEEN_XREG | SEEN_MEM; 618 /* st %r12,<K>(%r15) */ 619 EMIT4_DISP(0x50c0f000, 620 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 621 break; 622 case BPF_S_ANC_PROTOCOL: /* A = ntohs(skb->protocol); */ 623 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); 624 /* lhi %r5,0 */ 625 EMIT4(0xa7580000); 626 /* icm %r5,3,<d(protocol)>(%r2) */ 627 EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, protocol)); 628 break; 629 case BPF_S_ANC_IFINDEX: /* if (!skb->dev) return 0; 630 * A = skb->dev->ifindex */ 631 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); 632 jit->seen |= SEEN_RET0; 633 /* lg %r1,<d(dev)>(%r2) */ 634 EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); 635 /* ltgr %r1,%r1 */ 636 EMIT4(0xb9020011); 637 /* jz <ret0> */ 638 EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); 639 /* l %r5,<d(ifindex)>(%r1) */ 640 EMIT4_DISP(0x58501000, offsetof(struct net_device, ifindex)); 641 break; 642 case BPF_S_ANC_MARK: /* A = skb->mark */ 643 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); 644 /* l %r5,<d(mark)>(%r2) */ 645 EMIT4_DISP(0x58502000, offsetof(struct sk_buff, mark)); 646 break; 647 case BPF_S_ANC_QUEUE: /* A = skb->queue_mapping */ 648 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2); 649 /* lhi %r5,0 */ 650 EMIT4(0xa7580000); 651 /* icm %r5,3,<d(queue_mapping)>(%r2) */ 652 EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, queue_mapping)); 653 break; 654 case BPF_S_ANC_HATYPE: /* if (!skb->dev) return 0; 655 * A = skb->dev->type */ 656 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2); 657 jit->seen |= SEEN_RET0; 658 /* lg %r1,<d(dev)>(%r2) */ 659 EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); 660 /* ltgr %r1,%r1 */ 661 EMIT4(0xb9020011); 662 /* jz <ret0> */ 663 EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); 664 /* lhi %r5,0 */ 665 EMIT4(0xa7580000); 666 /* icm %r5,3,<d(type)>(%r1) */ 667 EMIT4_DISP(0xbf531000, offsetof(struct net_device, type)); 668 break; 669 case BPF_S_ANC_RXHASH: /* A = skb->rxhash */ 670 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4); 671 /* l %r5,<d(rxhash)>(%r2) */ 672 EMIT4_DISP(0x58502000, offsetof(struct sk_buff, rxhash)); 673 break; 674 case BPF_S_ANC_CPU: /* A = smp_processor_id() */ 675 #ifdef CONFIG_SMP 676 /* l %r5,<d(cpu_nr)> */ 677 EMIT4_DISP(0x58500000, offsetof(struct _lowcore, cpu_nr)); 678 #else 679 /* lhi %r5,0 */ 680 EMIT4(0xa7580000); 681 #endif 682 break; 683 default: /* too complex, give up */ 684 goto out; 685 } 686 addrs[i] = jit->prg - jit->start; 687 return 0; 688 out: 689 return -1; 690 } 691 692 void bpf_jit_compile(struct sk_filter *fp) 693 { 694 unsigned long size, prg_len, lit_len; 695 struct bpf_jit jit, cjit; 696 unsigned int *addrs; 697 int pass, i; 698 699 if (!bpf_jit_enable) 700 return; 701 addrs = kmalloc(fp->len * sizeof(*addrs), GFP_KERNEL); 702 if (addrs == NULL) 703 return; 704 memset(addrs, 0, fp->len * sizeof(*addrs)); 705 memset(&jit, 0, sizeof(cjit)); 706 memset(&cjit, 0, sizeof(cjit)); 707 708 for (pass = 0; pass < 10; pass++) { 709 jit.prg = jit.start; 710 jit.lit = jit.mid; 711 712 bpf_jit_prologue(&jit); 713 bpf_jit_noleaks(&jit, fp->insns); 714 for (i = 0; i < fp->len; i++) { 715 if (bpf_jit_insn(&jit, fp->insns + i, addrs, i, 716 i == fp->len - 1)) 717 goto out; 718 } 719 bpf_jit_epilogue(&jit); 720 if (jit.start) { 721 WARN_ON(jit.prg > cjit.prg || jit.lit > cjit.lit); 722 if (memcmp(&jit, &cjit, sizeof(jit)) == 0) 723 break; 724 } else if (jit.prg == cjit.prg && jit.lit == cjit.lit) { 725 prg_len = jit.prg - jit.start; 726 lit_len = jit.lit - jit.mid; 727 size = max_t(unsigned long, prg_len + lit_len, 728 sizeof(struct work_struct)); 729 if (size >= BPF_SIZE_MAX) 730 goto out; 731 jit.start = module_alloc(size); 732 if (!jit.start) 733 goto out; 734 jit.prg = jit.mid = jit.start + prg_len; 735 jit.lit = jit.end = jit.start + prg_len + lit_len; 736 jit.base_ip += (unsigned long) jit.start; 737 jit.exit_ip += (unsigned long) jit.start; 738 jit.ret0_ip += (unsigned long) jit.start; 739 } 740 cjit = jit; 741 } 742 if (bpf_jit_enable > 1) { 743 pr_err("flen=%d proglen=%lu pass=%d image=%p\n", 744 fp->len, jit.end - jit.start, pass, jit.start); 745 if (jit.start) { 746 printk(KERN_ERR "JIT code:\n"); 747 print_fn_code(jit.start, jit.mid - jit.start); 748 print_hex_dump(KERN_ERR, "JIT literals:\n", 749 DUMP_PREFIX_ADDRESS, 16, 1, 750 jit.mid, jit.end - jit.mid, false); 751 } 752 } 753 if (jit.start) 754 fp->bpf_func = (void *) jit.start; 755 out: 756 kfree(addrs); 757 } 758 759 static void jit_free_defer(struct work_struct *arg) 760 { 761 module_free(NULL, arg); 762 } 763 764 /* run from softirq, we must use a work_struct to call 765 * module_free() from process context 766 */ 767 void bpf_jit_free(struct sk_filter *fp) 768 { 769 struct work_struct *work; 770 771 if (fp->bpf_func == sk_run_filter) 772 return; 773 work = (struct work_struct *)fp->bpf_func; 774 INIT_WORK(work, jit_free_defer); 775 schedule_work(work); 776 } 777