1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org) 7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc. 8 */ 9 #ifndef _ASM_BITOPS_H 10 #define _ASM_BITOPS_H 11 12 #ifndef _LINUX_BITOPS_H 13 #error only <linux/bitops.h> can be included directly 14 #endif 15 16 #include <linux/compiler.h> 17 #include <linux/irqflags.h> 18 #include <linux/types.h> 19 #include <asm/barrier.h> 20 #include <asm/bug.h> 21 #include <asm/byteorder.h> /* sigh ... */ 22 #include <asm/cpu-features.h> 23 #include <asm/sgidefs.h> 24 #include <asm/war.h> 25 26 #if _MIPS_SZLONG == 32 27 #define SZLONG_LOG 5 28 #define SZLONG_MASK 31UL 29 #define __LL "ll " 30 #define __SC "sc " 31 #define __INS "ins " 32 #define __EXT "ext " 33 #elif _MIPS_SZLONG == 64 34 #define SZLONG_LOG 6 35 #define SZLONG_MASK 63UL 36 #define __LL "lld " 37 #define __SC "scd " 38 #define __INS "dins " 39 #define __EXT "dext " 40 #endif 41 42 /* 43 * clear_bit() doesn't provide any barrier for the compiler. 44 */ 45 #define smp_mb__before_clear_bit() smp_mb__before_llsc() 46 #define smp_mb__after_clear_bit() smp_llsc_mb() 47 48 /* 49 * set_bit - Atomically set a bit in memory 50 * @nr: the bit to set 51 * @addr: the address to start counting from 52 * 53 * This function is atomic and may not be reordered. See __set_bit() 54 * if you do not require the atomic guarantees. 55 * Note that @nr may be almost arbitrarily large; this function is not 56 * restricted to acting on a single-word quantity. 57 */ 58 static inline void set_bit(unsigned long nr, volatile unsigned long *addr) 59 { 60 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 61 unsigned short bit = nr & SZLONG_MASK; 62 unsigned long temp; 63 64 if (kernel_uses_llsc && R10000_LLSC_WAR) { 65 __asm__ __volatile__( 66 " .set mips3 \n" 67 "1: " __LL "%0, %1 # set_bit \n" 68 " or %0, %2 \n" 69 " " __SC "%0, %1 \n" 70 " beqzl %0, 1b \n" 71 " .set mips0 \n" 72 : "=&r" (temp), "=m" (*m) 73 : "ir" (1UL << bit), "m" (*m)); 74 #ifdef CONFIG_CPU_MIPSR2 75 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) { 76 __asm__ __volatile__( 77 "1: " __LL "%0, %1 # set_bit \n" 78 " " __INS "%0, %4, %2, 1 \n" 79 " " __SC "%0, %1 \n" 80 " beqz %0, 2f \n" 81 " .subsection 2 \n" 82 "2: b 1b \n" 83 " .previous \n" 84 : "=&r" (temp), "=m" (*m) 85 : "ir" (bit), "m" (*m), "r" (~0)); 86 #endif /* CONFIG_CPU_MIPSR2 */ 87 } else if (kernel_uses_llsc) { 88 __asm__ __volatile__( 89 " .set mips3 \n" 90 "1: " __LL "%0, %1 # set_bit \n" 91 " or %0, %2 \n" 92 " " __SC "%0, %1 \n" 93 " beqz %0, 2f \n" 94 " .subsection 2 \n" 95 "2: b 1b \n" 96 " .previous \n" 97 " .set mips0 \n" 98 : "=&r" (temp), "=m" (*m) 99 : "ir" (1UL << bit), "m" (*m)); 100 } else { 101 volatile unsigned long *a = addr; 102 unsigned long mask; 103 unsigned long flags; 104 105 a += nr >> SZLONG_LOG; 106 mask = 1UL << bit; 107 raw_local_irq_save(flags); 108 *a |= mask; 109 raw_local_irq_restore(flags); 110 } 111 } 112 113 /* 114 * clear_bit - Clears a bit in memory 115 * @nr: Bit to clear 116 * @addr: Address to start counting from 117 * 118 * clear_bit() is atomic and may not be reordered. However, it does 119 * not contain a memory barrier, so if it is used for locking purposes, 120 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() 121 * in order to ensure changes are visible on other processors. 122 */ 123 static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) 124 { 125 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 126 unsigned short bit = nr & SZLONG_MASK; 127 unsigned long temp; 128 129 if (kernel_uses_llsc && R10000_LLSC_WAR) { 130 __asm__ __volatile__( 131 " .set mips3 \n" 132 "1: " __LL "%0, %1 # clear_bit \n" 133 " and %0, %2 \n" 134 " " __SC "%0, %1 \n" 135 " beqzl %0, 1b \n" 136 " .set mips0 \n" 137 : "=&r" (temp), "=m" (*m) 138 : "ir" (~(1UL << bit)), "m" (*m)); 139 #ifdef CONFIG_CPU_MIPSR2 140 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) { 141 __asm__ __volatile__( 142 "1: " __LL "%0, %1 # clear_bit \n" 143 " " __INS "%0, $0, %2, 1 \n" 144 " " __SC "%0, %1 \n" 145 " beqz %0, 2f \n" 146 " .subsection 2 \n" 147 "2: b 1b \n" 148 " .previous \n" 149 : "=&r" (temp), "=m" (*m) 150 : "ir" (bit), "m" (*m)); 151 #endif /* CONFIG_CPU_MIPSR2 */ 152 } else if (kernel_uses_llsc) { 153 __asm__ __volatile__( 154 " .set mips3 \n" 155 "1: " __LL "%0, %1 # clear_bit \n" 156 " and %0, %2 \n" 157 " " __SC "%0, %1 \n" 158 " beqz %0, 2f \n" 159 " .subsection 2 \n" 160 "2: b 1b \n" 161 " .previous \n" 162 " .set mips0 \n" 163 : "=&r" (temp), "=m" (*m) 164 : "ir" (~(1UL << bit)), "m" (*m)); 165 } else { 166 volatile unsigned long *a = addr; 167 unsigned long mask; 168 unsigned long flags; 169 170 a += nr >> SZLONG_LOG; 171 mask = 1UL << bit; 172 raw_local_irq_save(flags); 173 *a &= ~mask; 174 raw_local_irq_restore(flags); 175 } 176 } 177 178 /* 179 * clear_bit_unlock - Clears a bit in memory 180 * @nr: Bit to clear 181 * @addr: Address to start counting from 182 * 183 * clear_bit() is atomic and implies release semantics before the memory 184 * operation. It can be used for an unlock. 185 */ 186 static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr) 187 { 188 smp_mb__before_clear_bit(); 189 clear_bit(nr, addr); 190 } 191 192 /* 193 * change_bit - Toggle a bit in memory 194 * @nr: Bit to change 195 * @addr: Address to start counting from 196 * 197 * change_bit() is atomic and may not be reordered. 198 * Note that @nr may be almost arbitrarily large; this function is not 199 * restricted to acting on a single-word quantity. 200 */ 201 static inline void change_bit(unsigned long nr, volatile unsigned long *addr) 202 { 203 unsigned short bit = nr & SZLONG_MASK; 204 205 if (kernel_uses_llsc && R10000_LLSC_WAR) { 206 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 207 unsigned long temp; 208 209 __asm__ __volatile__( 210 " .set mips3 \n" 211 "1: " __LL "%0, %1 # change_bit \n" 212 " xor %0, %2 \n" 213 " " __SC "%0, %1 \n" 214 " beqzl %0, 1b \n" 215 " .set mips0 \n" 216 : "=&r" (temp), "=m" (*m) 217 : "ir" (1UL << bit), "m" (*m)); 218 } else if (kernel_uses_llsc) { 219 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 220 unsigned long temp; 221 222 __asm__ __volatile__( 223 " .set mips3 \n" 224 "1: " __LL "%0, %1 # change_bit \n" 225 " xor %0, %2 \n" 226 " " __SC "%0, %1 \n" 227 " beqz %0, 2f \n" 228 " .subsection 2 \n" 229 "2: b 1b \n" 230 " .previous \n" 231 " .set mips0 \n" 232 : "=&r" (temp), "=m" (*m) 233 : "ir" (1UL << bit), "m" (*m)); 234 } else { 235 volatile unsigned long *a = addr; 236 unsigned long mask; 237 unsigned long flags; 238 239 a += nr >> SZLONG_LOG; 240 mask = 1UL << bit; 241 raw_local_irq_save(flags); 242 *a ^= mask; 243 raw_local_irq_restore(flags); 244 } 245 } 246 247 /* 248 * test_and_set_bit - Set a bit and return its old value 249 * @nr: Bit to set 250 * @addr: Address to count from 251 * 252 * This operation is atomic and cannot be reordered. 253 * It also implies a memory barrier. 254 */ 255 static inline int test_and_set_bit(unsigned long nr, 256 volatile unsigned long *addr) 257 { 258 unsigned short bit = nr & SZLONG_MASK; 259 unsigned long res; 260 261 smp_mb__before_llsc(); 262 263 if (kernel_uses_llsc && R10000_LLSC_WAR) { 264 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 265 unsigned long temp; 266 267 __asm__ __volatile__( 268 " .set mips3 \n" 269 "1: " __LL "%0, %1 # test_and_set_bit \n" 270 " or %2, %0, %3 \n" 271 " " __SC "%2, %1 \n" 272 " beqzl %2, 1b \n" 273 " and %2, %0, %3 \n" 274 " .set mips0 \n" 275 : "=&r" (temp), "=m" (*m), "=&r" (res) 276 : "r" (1UL << bit), "m" (*m) 277 : "memory"); 278 } else if (kernel_uses_llsc) { 279 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 280 unsigned long temp; 281 282 __asm__ __volatile__( 283 " .set push \n" 284 " .set noreorder \n" 285 " .set mips3 \n" 286 "1: " __LL "%0, %1 # test_and_set_bit \n" 287 " or %2, %0, %3 \n" 288 " " __SC "%2, %1 \n" 289 " beqz %2, 2f \n" 290 " and %2, %0, %3 \n" 291 " .subsection 2 \n" 292 "2: b 1b \n" 293 " nop \n" 294 " .previous \n" 295 " .set pop \n" 296 : "=&r" (temp), "=m" (*m), "=&r" (res) 297 : "r" (1UL << bit), "m" (*m) 298 : "memory"); 299 } else { 300 volatile unsigned long *a = addr; 301 unsigned long mask; 302 unsigned long flags; 303 304 a += nr >> SZLONG_LOG; 305 mask = 1UL << bit; 306 raw_local_irq_save(flags); 307 res = (mask & *a); 308 *a |= mask; 309 raw_local_irq_restore(flags); 310 } 311 312 smp_llsc_mb(); 313 314 return res != 0; 315 } 316 317 /* 318 * test_and_set_bit_lock - Set a bit and return its old value 319 * @nr: Bit to set 320 * @addr: Address to count from 321 * 322 * This operation is atomic and implies acquire ordering semantics 323 * after the memory operation. 324 */ 325 static inline int test_and_set_bit_lock(unsigned long nr, 326 volatile unsigned long *addr) 327 { 328 unsigned short bit = nr & SZLONG_MASK; 329 unsigned long res; 330 331 if (kernel_uses_llsc && R10000_LLSC_WAR) { 332 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 333 unsigned long temp; 334 335 __asm__ __volatile__( 336 " .set mips3 \n" 337 "1: " __LL "%0, %1 # test_and_set_bit \n" 338 " or %2, %0, %3 \n" 339 " " __SC "%2, %1 \n" 340 " beqzl %2, 1b \n" 341 " and %2, %0, %3 \n" 342 " .set mips0 \n" 343 : "=&r" (temp), "=m" (*m), "=&r" (res) 344 : "r" (1UL << bit), "m" (*m) 345 : "memory"); 346 } else if (kernel_uses_llsc) { 347 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 348 unsigned long temp; 349 350 __asm__ __volatile__( 351 " .set push \n" 352 " .set noreorder \n" 353 " .set mips3 \n" 354 "1: " __LL "%0, %1 # test_and_set_bit \n" 355 " or %2, %0, %3 \n" 356 " " __SC "%2, %1 \n" 357 " beqz %2, 2f \n" 358 " and %2, %0, %3 \n" 359 " .subsection 2 \n" 360 "2: b 1b \n" 361 " nop \n" 362 " .previous \n" 363 " .set pop \n" 364 : "=&r" (temp), "=m" (*m), "=&r" (res) 365 : "r" (1UL << bit), "m" (*m) 366 : "memory"); 367 } else { 368 volatile unsigned long *a = addr; 369 unsigned long mask; 370 unsigned long flags; 371 372 a += nr >> SZLONG_LOG; 373 mask = 1UL << bit; 374 raw_local_irq_save(flags); 375 res = (mask & *a); 376 *a |= mask; 377 raw_local_irq_restore(flags); 378 } 379 380 smp_llsc_mb(); 381 382 return res != 0; 383 } 384 /* 385 * test_and_clear_bit - Clear a bit and return its old value 386 * @nr: Bit to clear 387 * @addr: Address to count from 388 * 389 * This operation is atomic and cannot be reordered. 390 * It also implies a memory barrier. 391 */ 392 static inline int test_and_clear_bit(unsigned long nr, 393 volatile unsigned long *addr) 394 { 395 unsigned short bit = nr & SZLONG_MASK; 396 unsigned long res; 397 398 smp_mb__before_llsc(); 399 400 if (kernel_uses_llsc && R10000_LLSC_WAR) { 401 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 402 unsigned long temp; 403 404 __asm__ __volatile__( 405 " .set mips3 \n" 406 "1: " __LL "%0, %1 # test_and_clear_bit \n" 407 " or %2, %0, %3 \n" 408 " xor %2, %3 \n" 409 " " __SC "%2, %1 \n" 410 " beqzl %2, 1b \n" 411 " and %2, %0, %3 \n" 412 " .set mips0 \n" 413 : "=&r" (temp), "=m" (*m), "=&r" (res) 414 : "r" (1UL << bit), "m" (*m) 415 : "memory"); 416 #ifdef CONFIG_CPU_MIPSR2 417 } else if (kernel_uses_llsc && __builtin_constant_p(nr)) { 418 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 419 unsigned long temp; 420 421 __asm__ __volatile__( 422 "1: " __LL "%0, %1 # test_and_clear_bit \n" 423 " " __EXT "%2, %0, %3, 1 \n" 424 " " __INS "%0, $0, %3, 1 \n" 425 " " __SC "%0, %1 \n" 426 " beqz %0, 2f \n" 427 " .subsection 2 \n" 428 "2: b 1b \n" 429 " .previous \n" 430 : "=&r" (temp), "=m" (*m), "=&r" (res) 431 : "ir" (bit), "m" (*m) 432 : "memory"); 433 #endif 434 } else if (kernel_uses_llsc) { 435 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 436 unsigned long temp; 437 438 __asm__ __volatile__( 439 " .set push \n" 440 " .set noreorder \n" 441 " .set mips3 \n" 442 "1: " __LL "%0, %1 # test_and_clear_bit \n" 443 " or %2, %0, %3 \n" 444 " xor %2, %3 \n" 445 " " __SC "%2, %1 \n" 446 " beqz %2, 2f \n" 447 " and %2, %0, %3 \n" 448 " .subsection 2 \n" 449 "2: b 1b \n" 450 " nop \n" 451 " .previous \n" 452 " .set pop \n" 453 : "=&r" (temp), "=m" (*m), "=&r" (res) 454 : "r" (1UL << bit), "m" (*m) 455 : "memory"); 456 } else { 457 volatile unsigned long *a = addr; 458 unsigned long mask; 459 unsigned long flags; 460 461 a += nr >> SZLONG_LOG; 462 mask = 1UL << bit; 463 raw_local_irq_save(flags); 464 res = (mask & *a); 465 *a &= ~mask; 466 raw_local_irq_restore(flags); 467 } 468 469 smp_llsc_mb(); 470 471 return res != 0; 472 } 473 474 /* 475 * test_and_change_bit - Change a bit and return its old value 476 * @nr: Bit to change 477 * @addr: Address to count from 478 * 479 * This operation is atomic and cannot be reordered. 480 * It also implies a memory barrier. 481 */ 482 static inline int test_and_change_bit(unsigned long nr, 483 volatile unsigned long *addr) 484 { 485 unsigned short bit = nr & SZLONG_MASK; 486 unsigned long res; 487 488 smp_mb__before_llsc(); 489 490 if (kernel_uses_llsc && R10000_LLSC_WAR) { 491 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 492 unsigned long temp; 493 494 __asm__ __volatile__( 495 " .set mips3 \n" 496 "1: " __LL "%0, %1 # test_and_change_bit \n" 497 " xor %2, %0, %3 \n" 498 " " __SC "%2, %1 \n" 499 " beqzl %2, 1b \n" 500 " and %2, %0, %3 \n" 501 " .set mips0 \n" 502 : "=&r" (temp), "=m" (*m), "=&r" (res) 503 : "r" (1UL << bit), "m" (*m) 504 : "memory"); 505 } else if (kernel_uses_llsc) { 506 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); 507 unsigned long temp; 508 509 __asm__ __volatile__( 510 " .set push \n" 511 " .set noreorder \n" 512 " .set mips3 \n" 513 "1: " __LL "%0, %1 # test_and_change_bit \n" 514 " xor %2, %0, %3 \n" 515 " " __SC "\t%2, %1 \n" 516 " beqz %2, 2f \n" 517 " and %2, %0, %3 \n" 518 " .subsection 2 \n" 519 "2: b 1b \n" 520 " nop \n" 521 " .previous \n" 522 " .set pop \n" 523 : "=&r" (temp), "=m" (*m), "=&r" (res) 524 : "r" (1UL << bit), "m" (*m) 525 : "memory"); 526 } else { 527 volatile unsigned long *a = addr; 528 unsigned long mask; 529 unsigned long flags; 530 531 a += nr >> SZLONG_LOG; 532 mask = 1UL << bit; 533 raw_local_irq_save(flags); 534 res = (mask & *a); 535 *a ^= mask; 536 raw_local_irq_restore(flags); 537 } 538 539 smp_llsc_mb(); 540 541 return res != 0; 542 } 543 544 #include <asm-generic/bitops/non-atomic.h> 545 546 /* 547 * __clear_bit_unlock - Clears a bit in memory 548 * @nr: Bit to clear 549 * @addr: Address to start counting from 550 * 551 * __clear_bit() is non-atomic and implies release semantics before the memory 552 * operation. It can be used for an unlock if no other CPUs can concurrently 553 * modify other bits in the word. 554 */ 555 static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr) 556 { 557 smp_mb(); 558 __clear_bit(nr, addr); 559 } 560 561 /* 562 * Return the bit position (0..63) of the most significant 1 bit in a word 563 * Returns -1 if no 1 bit exists 564 */ 565 static inline unsigned long __fls(unsigned long word) 566 { 567 int num; 568 569 if (BITS_PER_LONG == 32 && 570 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) { 571 __asm__( 572 " .set push \n" 573 " .set mips32 \n" 574 " clz %0, %1 \n" 575 " .set pop \n" 576 : "=r" (num) 577 : "r" (word)); 578 579 return 31 - num; 580 } 581 582 if (BITS_PER_LONG == 64 && 583 __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) { 584 __asm__( 585 " .set push \n" 586 " .set mips64 \n" 587 " dclz %0, %1 \n" 588 " .set pop \n" 589 : "=r" (num) 590 : "r" (word)); 591 592 return 63 - num; 593 } 594 595 num = BITS_PER_LONG - 1; 596 597 #if BITS_PER_LONG == 64 598 if (!(word & (~0ul << 32))) { 599 num -= 32; 600 word <<= 32; 601 } 602 #endif 603 if (!(word & (~0ul << (BITS_PER_LONG-16)))) { 604 num -= 16; 605 word <<= 16; 606 } 607 if (!(word & (~0ul << (BITS_PER_LONG-8)))) { 608 num -= 8; 609 word <<= 8; 610 } 611 if (!(word & (~0ul << (BITS_PER_LONG-4)))) { 612 num -= 4; 613 word <<= 4; 614 } 615 if (!(word & (~0ul << (BITS_PER_LONG-2)))) { 616 num -= 2; 617 word <<= 2; 618 } 619 if (!(word & (~0ul << (BITS_PER_LONG-1)))) 620 num -= 1; 621 return num; 622 } 623 624 /* 625 * __ffs - find first bit in word. 626 * @word: The word to search 627 * 628 * Returns 0..SZLONG-1 629 * Undefined if no bit exists, so code should check against 0 first. 630 */ 631 static inline unsigned long __ffs(unsigned long word) 632 { 633 return __fls(word & -word); 634 } 635 636 /* 637 * fls - find last bit set. 638 * @word: The word to search 639 * 640 * This is defined the same way as ffs. 641 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. 642 */ 643 static inline int fls(int x) 644 { 645 int r; 646 647 if (__builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) { 648 __asm__("clz %0, %1" : "=r" (x) : "r" (x)); 649 650 return 32 - x; 651 } 652 653 r = 32; 654 if (!x) 655 return 0; 656 if (!(x & 0xffff0000u)) { 657 x <<= 16; 658 r -= 16; 659 } 660 if (!(x & 0xff000000u)) { 661 x <<= 8; 662 r -= 8; 663 } 664 if (!(x & 0xf0000000u)) { 665 x <<= 4; 666 r -= 4; 667 } 668 if (!(x & 0xc0000000u)) { 669 x <<= 2; 670 r -= 2; 671 } 672 if (!(x & 0x80000000u)) { 673 x <<= 1; 674 r -= 1; 675 } 676 return r; 677 } 678 679 #include <asm-generic/bitops/fls64.h> 680 681 /* 682 * ffs - find first bit set. 683 * @word: The word to search 684 * 685 * This is defined the same way as 686 * the libc and compiler builtin ffs routines, therefore 687 * differs in spirit from the above ffz (man ffs). 688 */ 689 static inline int ffs(int word) 690 { 691 if (!word) 692 return 0; 693 694 return fls(word & -word); 695 } 696 697 #include <asm-generic/bitops/ffz.h> 698 #include <asm-generic/bitops/find.h> 699 700 #ifdef __KERNEL__ 701 702 #include <asm-generic/bitops/sched.h> 703 704 #include <asm/arch_hweight.h> 705 #include <asm-generic/bitops/const_hweight.h> 706 707 #include <asm-generic/bitops/ext2-non-atomic.h> 708 #include <asm-generic/bitops/ext2-atomic.h> 709 #include <asm-generic/bitops/minix.h> 710 711 #endif /* __KERNEL__ */ 712 713 #endif /* _ASM_BITOPS_H */ 714