1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-arm64.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> 7 */ 8 9 /* 10 * aarch64 -mbig-endian generates mixed endianness code vs data: 11 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature 12 * matches code endianness. 13 */ 14 #define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */ 15 16 #ifdef __AARCH64EB__ 17 #define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */ 18 #else 19 #define RSEQ_SIG_DATA RSEQ_SIG_CODE 20 #endif 21 22 #define RSEQ_SIG RSEQ_SIG_DATA 23 24 #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory") 25 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory") 26 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory") 27 28 #define rseq_smp_load_acquire(p) \ 29 __extension__ ({ \ 30 __typeof(*p) ____p1; \ 31 switch (sizeof(*p)) { \ 32 case 1: \ 33 asm volatile ("ldarb %w0, %1" \ 34 : "=r" (*(__u8 *)p) \ 35 : "Q" (*p) : "memory"); \ 36 break; \ 37 case 2: \ 38 asm volatile ("ldarh %w0, %1" \ 39 : "=r" (*(__u16 *)p) \ 40 : "Q" (*p) : "memory"); \ 41 break; \ 42 case 4: \ 43 asm volatile ("ldar %w0, %1" \ 44 : "=r" (*(__u32 *)p) \ 45 : "Q" (*p) : "memory"); \ 46 break; \ 47 case 8: \ 48 asm volatile ("ldar %0, %1" \ 49 : "=r" (*(__u64 *)p) \ 50 : "Q" (*p) : "memory"); \ 51 break; \ 52 } \ 53 ____p1; \ 54 }) 55 56 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 57 58 #define rseq_smp_store_release(p, v) \ 59 do { \ 60 switch (sizeof(*p)) { \ 61 case 1: \ 62 asm volatile ("stlrb %w1, %0" \ 63 : "=Q" (*p) \ 64 : "r" ((__u8)v) \ 65 : "memory"); \ 66 break; \ 67 case 2: \ 68 asm volatile ("stlrh %w1, %0" \ 69 : "=Q" (*p) \ 70 : "r" ((__u16)v) \ 71 : "memory"); \ 72 break; \ 73 case 4: \ 74 asm volatile ("stlr %w1, %0" \ 75 : "=Q" (*p) \ 76 : "r" ((__u32)v) \ 77 : "memory"); \ 78 break; \ 79 case 8: \ 80 asm volatile ("stlr %1, %0" \ 81 : "=Q" (*p) \ 82 : "r" ((__u64)v) \ 83 : "memory"); \ 84 break; \ 85 } \ 86 } while (0) 87 88 #ifdef RSEQ_SKIP_FASTPATH 89 #include "rseq-skip.h" 90 #else /* !RSEQ_SKIP_FASTPATH */ 91 92 #define RSEQ_ASM_TMP_REG32 "w15" 93 #define RSEQ_ASM_TMP_REG "x15" 94 #define RSEQ_ASM_TMP_REG_2 "x14" 95 96 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ 97 post_commit_offset, abort_ip) \ 98 " .pushsection __rseq_cs, \"aw\"\n" \ 99 " .balign 32\n" \ 100 __rseq_str(label) ":\n" \ 101 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \ 102 " .quad " __rseq_str(start_ip) ", " \ 103 __rseq_str(post_commit_offset) ", " \ 104 __rseq_str(abort_ip) "\n" \ 105 " .popsection\n\t" \ 106 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \ 107 " .quad " __rseq_str(label) "b\n" \ 108 " .popsection\n" 109 110 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 111 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 112 (post_commit_ip - start_ip), abort_ip) 113 114 /* 115 * Exit points of a rseq critical section consist of all instructions outside 116 * of the critical section where a critical section can either branch to or 117 * reach through the normal course of its execution. The abort IP and the 118 * post-commit IP are already part of the __rseq_cs section and should not be 119 * explicitly defined as additional exit points. Knowing all exit points is 120 * useful to assist debuggers stepping over the critical section. 121 */ 122 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 123 " .pushsection __rseq_exit_point_array, \"aw\"\n" \ 124 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \ 125 " .popsection\n" 126 127 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 128 RSEQ_INJECT_ASM(1) \ 129 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \ 130 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 131 ", :lo12:" __rseq_str(cs_label) "\n" \ 132 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \ 133 __rseq_str(label) ":\n" 134 135 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ 136 " b 222f\n" \ 137 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \ 138 __rseq_str(label) ":\n" \ 139 " b %l[" __rseq_str(abort_label) "]\n" \ 140 "222:\n" 141 142 #define RSEQ_ASM_OP_STORE(value, var) \ 143 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" 144 145 #define RSEQ_ASM_OP_STORE_RELEASE(value, var) \ 146 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" 147 148 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ 149 RSEQ_ASM_OP_STORE(value, var) \ 150 __rseq_str(post_commit_label) ":\n" 151 152 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \ 153 RSEQ_ASM_OP_STORE_RELEASE(value, var) \ 154 __rseq_str(post_commit_label) ":\n" 155 156 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ 157 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ 158 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 159 ", %[" __rseq_str(expect) "]\n" \ 160 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n" 161 162 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \ 163 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \ 164 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \ 165 ", %w[" __rseq_str(expect) "]\n" \ 166 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n" 167 168 #define RSEQ_ASM_OP_CMPNE(var, expect, label) \ 169 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ 170 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 171 ", %[" __rseq_str(expect) "]\n" \ 172 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n" 173 174 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 175 RSEQ_INJECT_ASM(2) \ 176 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label) 177 178 #define RSEQ_ASM_OP_R_LOAD(var) \ 179 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" 180 181 #define RSEQ_ASM_OP_R_STORE(var) \ 182 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" 183 184 #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \ 185 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \ 186 ", %[" __rseq_str(offset) "]]\n" 187 188 #define RSEQ_ASM_OP_R_ADD(count) \ 189 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 190 ", %[" __rseq_str(count) "]\n" 191 192 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ 193 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ 194 __rseq_str(post_commit_label) ":\n" 195 196 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \ 197 " cbz %[" __rseq_str(len) "], 333f\n" \ 198 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \ 199 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \ 200 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \ 201 ", " RSEQ_ASM_TMP_REG_2 "]\n" \ 202 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \ 203 ", " RSEQ_ASM_TMP_REG_2 "]\n" \ 204 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \ 205 "333:\n" 206 207 static inline __attribute__((always_inline)) 208 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 209 { 210 RSEQ_INJECT_C(9) 211 212 __asm__ __volatile__ goto ( 213 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 214 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 215 #ifdef RSEQ_COMPARE_TWICE 216 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 217 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 218 #endif 219 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 220 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 221 RSEQ_INJECT_ASM(3) 222 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 223 RSEQ_INJECT_ASM(4) 224 #ifdef RSEQ_COMPARE_TWICE 225 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 226 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 227 #endif 228 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 229 RSEQ_INJECT_ASM(5) 230 RSEQ_ASM_DEFINE_ABORT(4, abort) 231 : /* gcc asm goto does not allow outputs */ 232 : [cpu_id] "r" (cpu), 233 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 234 [rseq_cs] "m" (__rseq_abi.rseq_cs), 235 [v] "Qo" (*v), 236 [expect] "r" (expect), 237 [newv] "r" (newv) 238 RSEQ_INJECT_INPUT 239 : "memory", RSEQ_ASM_TMP_REG 240 : abort, cmpfail 241 #ifdef RSEQ_COMPARE_TWICE 242 , error1, error2 243 #endif 244 ); 245 246 return 0; 247 abort: 248 RSEQ_INJECT_FAILED 249 return -1; 250 cmpfail: 251 return 1; 252 #ifdef RSEQ_COMPARE_TWICE 253 error1: 254 rseq_bug("cpu_id comparison failed"); 255 error2: 256 rseq_bug("expected value comparison failed"); 257 #endif 258 } 259 260 static inline __attribute__((always_inline)) 261 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 262 off_t voffp, intptr_t *load, int cpu) 263 { 264 RSEQ_INJECT_C(9) 265 266 __asm__ __volatile__ goto ( 267 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 268 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 269 #ifdef RSEQ_COMPARE_TWICE 270 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 271 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 272 #endif 273 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 274 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 275 RSEQ_INJECT_ASM(3) 276 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 277 RSEQ_INJECT_ASM(4) 278 #ifdef RSEQ_COMPARE_TWICE 279 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 280 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 281 #endif 282 RSEQ_ASM_OP_R_LOAD(v) 283 RSEQ_ASM_OP_R_STORE(load) 284 RSEQ_ASM_OP_R_LOAD_OFF(voffp) 285 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 286 RSEQ_INJECT_ASM(5) 287 RSEQ_ASM_DEFINE_ABORT(4, abort) 288 : /* gcc asm goto does not allow outputs */ 289 : [cpu_id] "r" (cpu), 290 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 291 [rseq_cs] "m" (__rseq_abi.rseq_cs), 292 [v] "Qo" (*v), 293 [expectnot] "r" (expectnot), 294 [load] "Qo" (*load), 295 [voffp] "r" (voffp) 296 RSEQ_INJECT_INPUT 297 : "memory", RSEQ_ASM_TMP_REG 298 : abort, cmpfail 299 #ifdef RSEQ_COMPARE_TWICE 300 , error1, error2 301 #endif 302 ); 303 return 0; 304 abort: 305 RSEQ_INJECT_FAILED 306 return -1; 307 cmpfail: 308 return 1; 309 #ifdef RSEQ_COMPARE_TWICE 310 error1: 311 rseq_bug("cpu_id comparison failed"); 312 error2: 313 rseq_bug("expected value comparison failed"); 314 #endif 315 } 316 317 static inline __attribute__((always_inline)) 318 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 319 { 320 RSEQ_INJECT_C(9) 321 322 __asm__ __volatile__ goto ( 323 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 324 #ifdef RSEQ_COMPARE_TWICE 325 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 326 #endif 327 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 328 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 329 RSEQ_INJECT_ASM(3) 330 #ifdef RSEQ_COMPARE_TWICE 331 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 332 #endif 333 RSEQ_ASM_OP_R_LOAD(v) 334 RSEQ_ASM_OP_R_ADD(count) 335 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 336 RSEQ_INJECT_ASM(4) 337 RSEQ_ASM_DEFINE_ABORT(4, abort) 338 : /* gcc asm goto does not allow outputs */ 339 : [cpu_id] "r" (cpu), 340 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 341 [rseq_cs] "m" (__rseq_abi.rseq_cs), 342 [v] "Qo" (*v), 343 [count] "r" (count) 344 RSEQ_INJECT_INPUT 345 : "memory", RSEQ_ASM_TMP_REG 346 : abort 347 #ifdef RSEQ_COMPARE_TWICE 348 , error1 349 #endif 350 ); 351 return 0; 352 abort: 353 RSEQ_INJECT_FAILED 354 return -1; 355 #ifdef RSEQ_COMPARE_TWICE 356 error1: 357 rseq_bug("cpu_id comparison failed"); 358 #endif 359 } 360 361 static inline __attribute__((always_inline)) 362 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 363 intptr_t *v2, intptr_t newv2, 364 intptr_t newv, int cpu) 365 { 366 RSEQ_INJECT_C(9) 367 368 __asm__ __volatile__ goto ( 369 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 370 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 371 #ifdef RSEQ_COMPARE_TWICE 372 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 373 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 374 #endif 375 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 376 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 377 RSEQ_INJECT_ASM(3) 378 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 379 RSEQ_INJECT_ASM(4) 380 #ifdef RSEQ_COMPARE_TWICE 381 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 382 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 383 #endif 384 RSEQ_ASM_OP_STORE(newv2, v2) 385 RSEQ_INJECT_ASM(5) 386 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 387 RSEQ_INJECT_ASM(6) 388 RSEQ_ASM_DEFINE_ABORT(4, abort) 389 : /* gcc asm goto does not allow outputs */ 390 : [cpu_id] "r" (cpu), 391 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 392 [rseq_cs] "m" (__rseq_abi.rseq_cs), 393 [expect] "r" (expect), 394 [v] "Qo" (*v), 395 [newv] "r" (newv), 396 [v2] "Qo" (*v2), 397 [newv2] "r" (newv2) 398 RSEQ_INJECT_INPUT 399 : "memory", RSEQ_ASM_TMP_REG 400 : abort, cmpfail 401 #ifdef RSEQ_COMPARE_TWICE 402 , error1, error2 403 #endif 404 ); 405 406 return 0; 407 abort: 408 RSEQ_INJECT_FAILED 409 return -1; 410 cmpfail: 411 return 1; 412 #ifdef RSEQ_COMPARE_TWICE 413 error1: 414 rseq_bug("cpu_id comparison failed"); 415 error2: 416 rseq_bug("expected value comparison failed"); 417 #endif 418 } 419 420 static inline __attribute__((always_inline)) 421 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 422 intptr_t *v2, intptr_t newv2, 423 intptr_t newv, int cpu) 424 { 425 RSEQ_INJECT_C(9) 426 427 __asm__ __volatile__ goto ( 428 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 429 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 430 #ifdef RSEQ_COMPARE_TWICE 431 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 432 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 433 #endif 434 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 435 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 436 RSEQ_INJECT_ASM(3) 437 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 438 RSEQ_INJECT_ASM(4) 439 #ifdef RSEQ_COMPARE_TWICE 440 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 441 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 442 #endif 443 RSEQ_ASM_OP_STORE(newv2, v2) 444 RSEQ_INJECT_ASM(5) 445 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 446 RSEQ_INJECT_ASM(6) 447 RSEQ_ASM_DEFINE_ABORT(4, abort) 448 : /* gcc asm goto does not allow outputs */ 449 : [cpu_id] "r" (cpu), 450 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 451 [rseq_cs] "m" (__rseq_abi.rseq_cs), 452 [expect] "r" (expect), 453 [v] "Qo" (*v), 454 [newv] "r" (newv), 455 [v2] "Qo" (*v2), 456 [newv2] "r" (newv2) 457 RSEQ_INJECT_INPUT 458 : "memory", RSEQ_ASM_TMP_REG 459 : abort, cmpfail 460 #ifdef RSEQ_COMPARE_TWICE 461 , error1, error2 462 #endif 463 ); 464 465 return 0; 466 abort: 467 RSEQ_INJECT_FAILED 468 return -1; 469 cmpfail: 470 return 1; 471 #ifdef RSEQ_COMPARE_TWICE 472 error1: 473 rseq_bug("cpu_id comparison failed"); 474 error2: 475 rseq_bug("expected value comparison failed"); 476 #endif 477 } 478 479 static inline __attribute__((always_inline)) 480 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 481 intptr_t *v2, intptr_t expect2, 482 intptr_t newv, int cpu) 483 { 484 RSEQ_INJECT_C(9) 485 486 __asm__ __volatile__ goto ( 487 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 488 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 489 #ifdef RSEQ_COMPARE_TWICE 490 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 491 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 492 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) 493 #endif 494 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 495 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 496 RSEQ_INJECT_ASM(3) 497 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 498 RSEQ_INJECT_ASM(4) 499 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 500 RSEQ_INJECT_ASM(5) 501 #ifdef RSEQ_COMPARE_TWICE 502 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 503 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 504 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 505 #endif 506 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 507 RSEQ_INJECT_ASM(6) 508 RSEQ_ASM_DEFINE_ABORT(4, abort) 509 : /* gcc asm goto does not allow outputs */ 510 : [cpu_id] "r" (cpu), 511 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 512 [rseq_cs] "m" (__rseq_abi.rseq_cs), 513 [v] "Qo" (*v), 514 [expect] "r" (expect), 515 [v2] "Qo" (*v2), 516 [expect2] "r" (expect2), 517 [newv] "r" (newv) 518 RSEQ_INJECT_INPUT 519 : "memory", RSEQ_ASM_TMP_REG 520 : abort, cmpfail 521 #ifdef RSEQ_COMPARE_TWICE 522 , error1, error2, error3 523 #endif 524 ); 525 526 return 0; 527 abort: 528 RSEQ_INJECT_FAILED 529 return -1; 530 cmpfail: 531 return 1; 532 #ifdef RSEQ_COMPARE_TWICE 533 error1: 534 rseq_bug("cpu_id comparison failed"); 535 error2: 536 rseq_bug("expected value comparison failed"); 537 error3: 538 rseq_bug("2nd expected value comparison failed"); 539 #endif 540 } 541 542 static inline __attribute__((always_inline)) 543 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 544 void *dst, void *src, size_t len, 545 intptr_t newv, int cpu) 546 { 547 RSEQ_INJECT_C(9) 548 549 __asm__ __volatile__ goto ( 550 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 551 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 552 #ifdef RSEQ_COMPARE_TWICE 553 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 554 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 555 #endif 556 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 557 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 558 RSEQ_INJECT_ASM(3) 559 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 560 RSEQ_INJECT_ASM(4) 561 #ifdef RSEQ_COMPARE_TWICE 562 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 563 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 564 #endif 565 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 566 RSEQ_INJECT_ASM(5) 567 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 568 RSEQ_INJECT_ASM(6) 569 RSEQ_ASM_DEFINE_ABORT(4, abort) 570 : /* gcc asm goto does not allow outputs */ 571 : [cpu_id] "r" (cpu), 572 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 573 [rseq_cs] "m" (__rseq_abi.rseq_cs), 574 [expect] "r" (expect), 575 [v] "Qo" (*v), 576 [newv] "r" (newv), 577 [dst] "r" (dst), 578 [src] "r" (src), 579 [len] "r" (len) 580 RSEQ_INJECT_INPUT 581 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 582 : abort, cmpfail 583 #ifdef RSEQ_COMPARE_TWICE 584 , error1, error2 585 #endif 586 ); 587 588 return 0; 589 abort: 590 RSEQ_INJECT_FAILED 591 return -1; 592 cmpfail: 593 return 1; 594 #ifdef RSEQ_COMPARE_TWICE 595 error1: 596 rseq_bug("cpu_id comparison failed"); 597 error2: 598 rseq_bug("expected value comparison failed"); 599 #endif 600 } 601 602 static inline __attribute__((always_inline)) 603 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 604 void *dst, void *src, size_t len, 605 intptr_t newv, int cpu) 606 { 607 RSEQ_INJECT_C(9) 608 609 __asm__ __volatile__ goto ( 610 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 611 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 612 #ifdef RSEQ_COMPARE_TWICE 613 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 614 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 615 #endif 616 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 617 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 618 RSEQ_INJECT_ASM(3) 619 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 620 RSEQ_INJECT_ASM(4) 621 #ifdef RSEQ_COMPARE_TWICE 622 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 623 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 624 #endif 625 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 626 RSEQ_INJECT_ASM(5) 627 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 628 RSEQ_INJECT_ASM(6) 629 RSEQ_ASM_DEFINE_ABORT(4, abort) 630 : /* gcc asm goto does not allow outputs */ 631 : [cpu_id] "r" (cpu), 632 [current_cpu_id] "Qo" (__rseq_abi.cpu_id), 633 [rseq_cs] "m" (__rseq_abi.rseq_cs), 634 [expect] "r" (expect), 635 [v] "Qo" (*v), 636 [newv] "r" (newv), 637 [dst] "r" (dst), 638 [src] "r" (src), 639 [len] "r" (len) 640 RSEQ_INJECT_INPUT 641 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 642 : abort, cmpfail 643 #ifdef RSEQ_COMPARE_TWICE 644 , error1, error2 645 #endif 646 ); 647 648 return 0; 649 abort: 650 RSEQ_INJECT_FAILED 651 return -1; 652 cmpfail: 653 return 1; 654 #ifdef RSEQ_COMPARE_TWICE 655 error1: 656 rseq_bug("cpu_id comparison failed"); 657 error2: 658 rseq_bug("expected value comparison failed"); 659 #endif 660 } 661 662 #endif /* !RSEQ_SKIP_FASTPATH */ 663