1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Module-based torture test facility for locking 4 * 5 * Copyright (C) IBM Corporation, 2014 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Davidlohr Bueso <dave@stgolabs.net> 9 * Based on kernel/rcu/torture.c. 10 */ 11 12 #define pr_fmt(fmt) fmt 13 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/kthread.h> 17 #include <linux/sched/rt.h> 18 #include <linux/spinlock.h> 19 #include <linux/mutex.h> 20 #include <linux/rwsem.h> 21 #include <linux/smp.h> 22 #include <linux/interrupt.h> 23 #include <linux/sched.h> 24 #include <uapi/linux/sched/types.h> 25 #include <linux/rtmutex.h> 26 #include <linux/atomic.h> 27 #include <linux/moduleparam.h> 28 #include <linux/delay.h> 29 #include <linux/slab.h> 30 #include <linux/percpu-rwsem.h> 31 #include <linux/torture.h> 32 33 MODULE_LICENSE("GPL"); 34 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); 35 36 torture_param(int, nwriters_stress, -1, 37 "Number of write-locking stress-test threads"); 38 torture_param(int, nreaders_stress, -1, 39 "Number of read-locking stress-test threads"); 40 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 41 torture_param(int, onoff_interval, 0, 42 "Time between CPU hotplugs (s), 0=disable"); 43 torture_param(int, shuffle_interval, 3, 44 "Number of jiffies between shuffles, 0=disable"); 45 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable."); 46 torture_param(int, stat_interval, 60, 47 "Number of seconds between stats printk()s"); 48 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); 49 torture_param(int, verbose, 1, 50 "Enable verbose debugging printk()s"); 51 52 static char *torture_type = "spin_lock"; 53 module_param(torture_type, charp, 0444); 54 MODULE_PARM_DESC(torture_type, 55 "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)"); 56 57 static struct task_struct *stats_task; 58 static struct task_struct **writer_tasks; 59 static struct task_struct **reader_tasks; 60 61 static bool lock_is_write_held; 62 static bool lock_is_read_held; 63 64 struct lock_stress_stats { 65 long n_lock_fail; 66 long n_lock_acquired; 67 }; 68 69 /* Forward reference. */ 70 static void lock_torture_cleanup(void); 71 72 /* 73 * Operations vector for selecting different types of tests. 74 */ 75 struct lock_torture_ops { 76 void (*init)(void); 77 int (*writelock)(void); 78 void (*write_delay)(struct torture_random_state *trsp); 79 void (*task_boost)(struct torture_random_state *trsp); 80 void (*writeunlock)(void); 81 int (*readlock)(void); 82 void (*read_delay)(struct torture_random_state *trsp); 83 void (*readunlock)(void); 84 85 unsigned long flags; /* for irq spinlocks */ 86 const char *name; 87 }; 88 89 struct lock_torture_cxt { 90 int nrealwriters_stress; 91 int nrealreaders_stress; 92 bool debug_lock; 93 atomic_t n_lock_torture_errors; 94 struct lock_torture_ops *cur_ops; 95 struct lock_stress_stats *lwsa; /* writer statistics */ 96 struct lock_stress_stats *lrsa; /* reader statistics */ 97 }; 98 static struct lock_torture_cxt cxt = { 0, 0, false, 99 ATOMIC_INIT(0), 100 NULL, NULL}; 101 /* 102 * Definitions for lock torture testing. 103 */ 104 105 static int torture_lock_busted_write_lock(void) 106 { 107 return 0; /* BUGGY, do not use in real life!!! */ 108 } 109 110 static void torture_lock_busted_write_delay(struct torture_random_state *trsp) 111 { 112 const unsigned long longdelay_ms = 100; 113 114 /* We want a long delay occasionally to force massive contention. */ 115 if (!(torture_random(trsp) % 116 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 117 mdelay(longdelay_ms); 118 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 119 torture_preempt_schedule(); /* Allow test to be preempted. */ 120 } 121 122 static void torture_lock_busted_write_unlock(void) 123 { 124 /* BUGGY, do not use in real life!!! */ 125 } 126 127 static void torture_boost_dummy(struct torture_random_state *trsp) 128 { 129 /* Only rtmutexes care about priority */ 130 } 131 132 static struct lock_torture_ops lock_busted_ops = { 133 .writelock = torture_lock_busted_write_lock, 134 .write_delay = torture_lock_busted_write_delay, 135 .task_boost = torture_boost_dummy, 136 .writeunlock = torture_lock_busted_write_unlock, 137 .readlock = NULL, 138 .read_delay = NULL, 139 .readunlock = NULL, 140 .name = "lock_busted" 141 }; 142 143 static DEFINE_SPINLOCK(torture_spinlock); 144 145 static int torture_spin_lock_write_lock(void) __acquires(torture_spinlock) 146 { 147 spin_lock(&torture_spinlock); 148 return 0; 149 } 150 151 static void torture_spin_lock_write_delay(struct torture_random_state *trsp) 152 { 153 const unsigned long shortdelay_us = 2; 154 const unsigned long longdelay_ms = 100; 155 156 /* We want a short delay mostly to emulate likely code, and 157 * we want a long delay occasionally to force massive contention. 158 */ 159 if (!(torture_random(trsp) % 160 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 161 mdelay(longdelay_ms); 162 if (!(torture_random(trsp) % 163 (cxt.nrealwriters_stress * 2 * shortdelay_us))) 164 udelay(shortdelay_us); 165 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 166 torture_preempt_schedule(); /* Allow test to be preempted. */ 167 } 168 169 static void torture_spin_lock_write_unlock(void) __releases(torture_spinlock) 170 { 171 spin_unlock(&torture_spinlock); 172 } 173 174 static struct lock_torture_ops spin_lock_ops = { 175 .writelock = torture_spin_lock_write_lock, 176 .write_delay = torture_spin_lock_write_delay, 177 .task_boost = torture_boost_dummy, 178 .writeunlock = torture_spin_lock_write_unlock, 179 .readlock = NULL, 180 .read_delay = NULL, 181 .readunlock = NULL, 182 .name = "spin_lock" 183 }; 184 185 static int torture_spin_lock_write_lock_irq(void) 186 __acquires(torture_spinlock) 187 { 188 unsigned long flags; 189 190 spin_lock_irqsave(&torture_spinlock, flags); 191 cxt.cur_ops->flags = flags; 192 return 0; 193 } 194 195 static void torture_lock_spin_write_unlock_irq(void) 196 __releases(torture_spinlock) 197 { 198 spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags); 199 } 200 201 static struct lock_torture_ops spin_lock_irq_ops = { 202 .writelock = torture_spin_lock_write_lock_irq, 203 .write_delay = torture_spin_lock_write_delay, 204 .task_boost = torture_boost_dummy, 205 .writeunlock = torture_lock_spin_write_unlock_irq, 206 .readlock = NULL, 207 .read_delay = NULL, 208 .readunlock = NULL, 209 .name = "spin_lock_irq" 210 }; 211 212 static DEFINE_RWLOCK(torture_rwlock); 213 214 static int torture_rwlock_write_lock(void) __acquires(torture_rwlock) 215 { 216 write_lock(&torture_rwlock); 217 return 0; 218 } 219 220 static void torture_rwlock_write_delay(struct torture_random_state *trsp) 221 { 222 const unsigned long shortdelay_us = 2; 223 const unsigned long longdelay_ms = 100; 224 225 /* We want a short delay mostly to emulate likely code, and 226 * we want a long delay occasionally to force massive contention. 227 */ 228 if (!(torture_random(trsp) % 229 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 230 mdelay(longdelay_ms); 231 else 232 udelay(shortdelay_us); 233 } 234 235 static void torture_rwlock_write_unlock(void) __releases(torture_rwlock) 236 { 237 write_unlock(&torture_rwlock); 238 } 239 240 static int torture_rwlock_read_lock(void) __acquires(torture_rwlock) 241 { 242 read_lock(&torture_rwlock); 243 return 0; 244 } 245 246 static void torture_rwlock_read_delay(struct torture_random_state *trsp) 247 { 248 const unsigned long shortdelay_us = 10; 249 const unsigned long longdelay_ms = 100; 250 251 /* We want a short delay mostly to emulate likely code, and 252 * we want a long delay occasionally to force massive contention. 253 */ 254 if (!(torture_random(trsp) % 255 (cxt.nrealreaders_stress * 2000 * longdelay_ms))) 256 mdelay(longdelay_ms); 257 else 258 udelay(shortdelay_us); 259 } 260 261 static void torture_rwlock_read_unlock(void) __releases(torture_rwlock) 262 { 263 read_unlock(&torture_rwlock); 264 } 265 266 static struct lock_torture_ops rw_lock_ops = { 267 .writelock = torture_rwlock_write_lock, 268 .write_delay = torture_rwlock_write_delay, 269 .task_boost = torture_boost_dummy, 270 .writeunlock = torture_rwlock_write_unlock, 271 .readlock = torture_rwlock_read_lock, 272 .read_delay = torture_rwlock_read_delay, 273 .readunlock = torture_rwlock_read_unlock, 274 .name = "rw_lock" 275 }; 276 277 static int torture_rwlock_write_lock_irq(void) __acquires(torture_rwlock) 278 { 279 unsigned long flags; 280 281 write_lock_irqsave(&torture_rwlock, flags); 282 cxt.cur_ops->flags = flags; 283 return 0; 284 } 285 286 static void torture_rwlock_write_unlock_irq(void) 287 __releases(torture_rwlock) 288 { 289 write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags); 290 } 291 292 static int torture_rwlock_read_lock_irq(void) __acquires(torture_rwlock) 293 { 294 unsigned long flags; 295 296 read_lock_irqsave(&torture_rwlock, flags); 297 cxt.cur_ops->flags = flags; 298 return 0; 299 } 300 301 static void torture_rwlock_read_unlock_irq(void) 302 __releases(torture_rwlock) 303 { 304 read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags); 305 } 306 307 static struct lock_torture_ops rw_lock_irq_ops = { 308 .writelock = torture_rwlock_write_lock_irq, 309 .write_delay = torture_rwlock_write_delay, 310 .task_boost = torture_boost_dummy, 311 .writeunlock = torture_rwlock_write_unlock_irq, 312 .readlock = torture_rwlock_read_lock_irq, 313 .read_delay = torture_rwlock_read_delay, 314 .readunlock = torture_rwlock_read_unlock_irq, 315 .name = "rw_lock_irq" 316 }; 317 318 static DEFINE_MUTEX(torture_mutex); 319 320 static int torture_mutex_lock(void) __acquires(torture_mutex) 321 { 322 mutex_lock(&torture_mutex); 323 return 0; 324 } 325 326 static void torture_mutex_delay(struct torture_random_state *trsp) 327 { 328 const unsigned long longdelay_ms = 100; 329 330 /* We want a long delay occasionally to force massive contention. */ 331 if (!(torture_random(trsp) % 332 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 333 mdelay(longdelay_ms * 5); 334 else 335 mdelay(longdelay_ms / 5); 336 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 337 torture_preempt_schedule(); /* Allow test to be preempted. */ 338 } 339 340 static void torture_mutex_unlock(void) __releases(torture_mutex) 341 { 342 mutex_unlock(&torture_mutex); 343 } 344 345 static struct lock_torture_ops mutex_lock_ops = { 346 .writelock = torture_mutex_lock, 347 .write_delay = torture_mutex_delay, 348 .task_boost = torture_boost_dummy, 349 .writeunlock = torture_mutex_unlock, 350 .readlock = NULL, 351 .read_delay = NULL, 352 .readunlock = NULL, 353 .name = "mutex_lock" 354 }; 355 356 #include <linux/ww_mutex.h> 357 static DEFINE_WD_CLASS(torture_ww_class); 358 static DEFINE_WW_MUTEX(torture_ww_mutex_0, &torture_ww_class); 359 static DEFINE_WW_MUTEX(torture_ww_mutex_1, &torture_ww_class); 360 static DEFINE_WW_MUTEX(torture_ww_mutex_2, &torture_ww_class); 361 362 static int torture_ww_mutex_lock(void) 363 __acquires(torture_ww_mutex_0) 364 __acquires(torture_ww_mutex_1) 365 __acquires(torture_ww_mutex_2) 366 { 367 LIST_HEAD(list); 368 struct reorder_lock { 369 struct list_head link; 370 struct ww_mutex *lock; 371 } locks[3], *ll, *ln; 372 struct ww_acquire_ctx ctx; 373 374 locks[0].lock = &torture_ww_mutex_0; 375 list_add(&locks[0].link, &list); 376 377 locks[1].lock = &torture_ww_mutex_1; 378 list_add(&locks[1].link, &list); 379 380 locks[2].lock = &torture_ww_mutex_2; 381 list_add(&locks[2].link, &list); 382 383 ww_acquire_init(&ctx, &torture_ww_class); 384 385 list_for_each_entry(ll, &list, link) { 386 int err; 387 388 err = ww_mutex_lock(ll->lock, &ctx); 389 if (!err) 390 continue; 391 392 ln = ll; 393 list_for_each_entry_continue_reverse(ln, &list, link) 394 ww_mutex_unlock(ln->lock); 395 396 if (err != -EDEADLK) 397 return err; 398 399 ww_mutex_lock_slow(ll->lock, &ctx); 400 list_move(&ll->link, &list); 401 } 402 403 ww_acquire_fini(&ctx); 404 return 0; 405 } 406 407 static void torture_ww_mutex_unlock(void) 408 __releases(torture_ww_mutex_0) 409 __releases(torture_ww_mutex_1) 410 __releases(torture_ww_mutex_2) 411 { 412 ww_mutex_unlock(&torture_ww_mutex_0); 413 ww_mutex_unlock(&torture_ww_mutex_1); 414 ww_mutex_unlock(&torture_ww_mutex_2); 415 } 416 417 static struct lock_torture_ops ww_mutex_lock_ops = { 418 .writelock = torture_ww_mutex_lock, 419 .write_delay = torture_mutex_delay, 420 .task_boost = torture_boost_dummy, 421 .writeunlock = torture_ww_mutex_unlock, 422 .readlock = NULL, 423 .read_delay = NULL, 424 .readunlock = NULL, 425 .name = "ww_mutex_lock" 426 }; 427 428 #ifdef CONFIG_RT_MUTEXES 429 static DEFINE_RT_MUTEX(torture_rtmutex); 430 431 static int torture_rtmutex_lock(void) __acquires(torture_rtmutex) 432 { 433 rt_mutex_lock(&torture_rtmutex); 434 return 0; 435 } 436 437 static void torture_rtmutex_boost(struct torture_random_state *trsp) 438 { 439 int policy; 440 struct sched_param param; 441 const unsigned int factor = 50000; /* yes, quite arbitrary */ 442 443 if (!rt_task(current)) { 444 /* 445 * Boost priority once every ~50k operations. When the 446 * task tries to take the lock, the rtmutex it will account 447 * for the new priority, and do any corresponding pi-dance. 448 */ 449 if (trsp && !(torture_random(trsp) % 450 (cxt.nrealwriters_stress * factor))) { 451 policy = SCHED_FIFO; 452 param.sched_priority = MAX_RT_PRIO - 1; 453 } else /* common case, do nothing */ 454 return; 455 } else { 456 /* 457 * The task will remain boosted for another ~500k operations, 458 * then restored back to its original prio, and so forth. 459 * 460 * When @trsp is nil, we want to force-reset the task for 461 * stopping the kthread. 462 */ 463 if (!trsp || !(torture_random(trsp) % 464 (cxt.nrealwriters_stress * factor * 2))) { 465 policy = SCHED_NORMAL; 466 param.sched_priority = 0; 467 } else /* common case, do nothing */ 468 return; 469 } 470 471 sched_setscheduler_nocheck(current, policy, ¶m); 472 } 473 474 static void torture_rtmutex_delay(struct torture_random_state *trsp) 475 { 476 const unsigned long shortdelay_us = 2; 477 const unsigned long longdelay_ms = 100; 478 479 /* 480 * We want a short delay mostly to emulate likely code, and 481 * we want a long delay occasionally to force massive contention. 482 */ 483 if (!(torture_random(trsp) % 484 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 485 mdelay(longdelay_ms); 486 if (!(torture_random(trsp) % 487 (cxt.nrealwriters_stress * 2 * shortdelay_us))) 488 udelay(shortdelay_us); 489 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 490 torture_preempt_schedule(); /* Allow test to be preempted. */ 491 } 492 493 static void torture_rtmutex_unlock(void) __releases(torture_rtmutex) 494 { 495 rt_mutex_unlock(&torture_rtmutex); 496 } 497 498 static struct lock_torture_ops rtmutex_lock_ops = { 499 .writelock = torture_rtmutex_lock, 500 .write_delay = torture_rtmutex_delay, 501 .task_boost = torture_rtmutex_boost, 502 .writeunlock = torture_rtmutex_unlock, 503 .readlock = NULL, 504 .read_delay = NULL, 505 .readunlock = NULL, 506 .name = "rtmutex_lock" 507 }; 508 #endif 509 510 static DECLARE_RWSEM(torture_rwsem); 511 static int torture_rwsem_down_write(void) __acquires(torture_rwsem) 512 { 513 down_write(&torture_rwsem); 514 return 0; 515 } 516 517 static void torture_rwsem_write_delay(struct torture_random_state *trsp) 518 { 519 const unsigned long longdelay_ms = 100; 520 521 /* We want a long delay occasionally to force massive contention. */ 522 if (!(torture_random(trsp) % 523 (cxt.nrealwriters_stress * 2000 * longdelay_ms))) 524 mdelay(longdelay_ms * 10); 525 else 526 mdelay(longdelay_ms / 10); 527 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) 528 torture_preempt_schedule(); /* Allow test to be preempted. */ 529 } 530 531 static void torture_rwsem_up_write(void) __releases(torture_rwsem) 532 { 533 up_write(&torture_rwsem); 534 } 535 536 static int torture_rwsem_down_read(void) __acquires(torture_rwsem) 537 { 538 down_read(&torture_rwsem); 539 return 0; 540 } 541 542 static void torture_rwsem_read_delay(struct torture_random_state *trsp) 543 { 544 const unsigned long longdelay_ms = 100; 545 546 /* We want a long delay occasionally to force massive contention. */ 547 if (!(torture_random(trsp) % 548 (cxt.nrealreaders_stress * 2000 * longdelay_ms))) 549 mdelay(longdelay_ms * 2); 550 else 551 mdelay(longdelay_ms / 2); 552 if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000))) 553 torture_preempt_schedule(); /* Allow test to be preempted. */ 554 } 555 556 static void torture_rwsem_up_read(void) __releases(torture_rwsem) 557 { 558 up_read(&torture_rwsem); 559 } 560 561 static struct lock_torture_ops rwsem_lock_ops = { 562 .writelock = torture_rwsem_down_write, 563 .write_delay = torture_rwsem_write_delay, 564 .task_boost = torture_boost_dummy, 565 .writeunlock = torture_rwsem_up_write, 566 .readlock = torture_rwsem_down_read, 567 .read_delay = torture_rwsem_read_delay, 568 .readunlock = torture_rwsem_up_read, 569 .name = "rwsem_lock" 570 }; 571 572 #include <linux/percpu-rwsem.h> 573 static struct percpu_rw_semaphore pcpu_rwsem; 574 575 void torture_percpu_rwsem_init(void) 576 { 577 BUG_ON(percpu_init_rwsem(&pcpu_rwsem)); 578 } 579 580 static int torture_percpu_rwsem_down_write(void) __acquires(pcpu_rwsem) 581 { 582 percpu_down_write(&pcpu_rwsem); 583 return 0; 584 } 585 586 static void torture_percpu_rwsem_up_write(void) __releases(pcpu_rwsem) 587 { 588 percpu_up_write(&pcpu_rwsem); 589 } 590 591 static int torture_percpu_rwsem_down_read(void) __acquires(pcpu_rwsem) 592 { 593 percpu_down_read(&pcpu_rwsem); 594 return 0; 595 } 596 597 static void torture_percpu_rwsem_up_read(void) __releases(pcpu_rwsem) 598 { 599 percpu_up_read(&pcpu_rwsem); 600 } 601 602 static struct lock_torture_ops percpu_rwsem_lock_ops = { 603 .init = torture_percpu_rwsem_init, 604 .writelock = torture_percpu_rwsem_down_write, 605 .write_delay = torture_rwsem_write_delay, 606 .task_boost = torture_boost_dummy, 607 .writeunlock = torture_percpu_rwsem_up_write, 608 .readlock = torture_percpu_rwsem_down_read, 609 .read_delay = torture_rwsem_read_delay, 610 .readunlock = torture_percpu_rwsem_up_read, 611 .name = "percpu_rwsem_lock" 612 }; 613 614 /* 615 * Lock torture writer kthread. Repeatedly acquires and releases 616 * the lock, checking for duplicate acquisitions. 617 */ 618 static int lock_torture_writer(void *arg) 619 { 620 struct lock_stress_stats *lwsp = arg; 621 DEFINE_TORTURE_RANDOM(rand); 622 623 VERBOSE_TOROUT_STRING("lock_torture_writer task started"); 624 set_user_nice(current, MAX_NICE); 625 626 do { 627 if ((torture_random(&rand) & 0xfffff) == 0) 628 schedule_timeout_uninterruptible(1); 629 630 cxt.cur_ops->task_boost(&rand); 631 cxt.cur_ops->writelock(); 632 if (WARN_ON_ONCE(lock_is_write_held)) 633 lwsp->n_lock_fail++; 634 lock_is_write_held = 1; 635 if (WARN_ON_ONCE(lock_is_read_held)) 636 lwsp->n_lock_fail++; /* rare, but... */ 637 638 lwsp->n_lock_acquired++; 639 cxt.cur_ops->write_delay(&rand); 640 lock_is_write_held = 0; 641 cxt.cur_ops->writeunlock(); 642 643 stutter_wait("lock_torture_writer"); 644 } while (!torture_must_stop()); 645 646 cxt.cur_ops->task_boost(NULL); /* reset prio */ 647 torture_kthread_stopping("lock_torture_writer"); 648 return 0; 649 } 650 651 /* 652 * Lock torture reader kthread. Repeatedly acquires and releases 653 * the reader lock. 654 */ 655 static int lock_torture_reader(void *arg) 656 { 657 struct lock_stress_stats *lrsp = arg; 658 DEFINE_TORTURE_RANDOM(rand); 659 660 VERBOSE_TOROUT_STRING("lock_torture_reader task started"); 661 set_user_nice(current, MAX_NICE); 662 663 do { 664 if ((torture_random(&rand) & 0xfffff) == 0) 665 schedule_timeout_uninterruptible(1); 666 667 cxt.cur_ops->readlock(); 668 lock_is_read_held = 1; 669 if (WARN_ON_ONCE(lock_is_write_held)) 670 lrsp->n_lock_fail++; /* rare, but... */ 671 672 lrsp->n_lock_acquired++; 673 cxt.cur_ops->read_delay(&rand); 674 lock_is_read_held = 0; 675 cxt.cur_ops->readunlock(); 676 677 stutter_wait("lock_torture_reader"); 678 } while (!torture_must_stop()); 679 torture_kthread_stopping("lock_torture_reader"); 680 return 0; 681 } 682 683 /* 684 * Create an lock-torture-statistics message in the specified buffer. 685 */ 686 static void __torture_print_stats(char *page, 687 struct lock_stress_stats *statp, bool write) 688 { 689 bool fail = 0; 690 int i, n_stress; 691 long max = 0, min = statp ? statp[0].n_lock_acquired : 0; 692 long long sum = 0; 693 694 n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress; 695 for (i = 0; i < n_stress; i++) { 696 if (statp[i].n_lock_fail) 697 fail = true; 698 sum += statp[i].n_lock_acquired; 699 if (max < statp[i].n_lock_acquired) 700 max = statp[i].n_lock_acquired; 701 if (min > statp[i].n_lock_acquired) 702 min = statp[i].n_lock_acquired; 703 } 704 page += sprintf(page, 705 "%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n", 706 write ? "Writes" : "Reads ", 707 sum, max, min, 708 !onoff_interval && max / 2 > min ? "???" : "", 709 fail, fail ? "!!!" : ""); 710 if (fail) 711 atomic_inc(&cxt.n_lock_torture_errors); 712 } 713 714 /* 715 * Print torture statistics. Caller must ensure that there is only one 716 * call to this function at a given time!!! This is normally accomplished 717 * by relying on the module system to only have one copy of the module 718 * loaded, and then by giving the lock_torture_stats kthread full control 719 * (or the init/cleanup functions when lock_torture_stats thread is not 720 * running). 721 */ 722 static void lock_torture_stats_print(void) 723 { 724 int size = cxt.nrealwriters_stress * 200 + 8192; 725 char *buf; 726 727 if (cxt.cur_ops->readlock) 728 size += cxt.nrealreaders_stress * 200 + 8192; 729 730 buf = kmalloc(size, GFP_KERNEL); 731 if (!buf) { 732 pr_err("lock_torture_stats_print: Out of memory, need: %d", 733 size); 734 return; 735 } 736 737 __torture_print_stats(buf, cxt.lwsa, true); 738 pr_alert("%s", buf); 739 kfree(buf); 740 741 if (cxt.cur_ops->readlock) { 742 buf = kmalloc(size, GFP_KERNEL); 743 if (!buf) { 744 pr_err("lock_torture_stats_print: Out of memory, need: %d", 745 size); 746 return; 747 } 748 749 __torture_print_stats(buf, cxt.lrsa, false); 750 pr_alert("%s", buf); 751 kfree(buf); 752 } 753 } 754 755 /* 756 * Periodically prints torture statistics, if periodic statistics printing 757 * was specified via the stat_interval module parameter. 758 * 759 * No need to worry about fullstop here, since this one doesn't reference 760 * volatile state or register callbacks. 761 */ 762 static int lock_torture_stats(void *arg) 763 { 764 VERBOSE_TOROUT_STRING("lock_torture_stats task started"); 765 do { 766 schedule_timeout_interruptible(stat_interval * HZ); 767 lock_torture_stats_print(); 768 torture_shutdown_absorb("lock_torture_stats"); 769 } while (!torture_must_stop()); 770 torture_kthread_stopping("lock_torture_stats"); 771 return 0; 772 } 773 774 static inline void 775 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops, 776 const char *tag) 777 { 778 pr_alert("%s" TORTURE_FLAG 779 "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n", 780 torture_type, tag, cxt.debug_lock ? " [debug]": "", 781 cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval, 782 verbose, shuffle_interval, stutter, shutdown_secs, 783 onoff_interval, onoff_holdoff); 784 } 785 786 static void lock_torture_cleanup(void) 787 { 788 int i; 789 790 if (torture_cleanup_begin()) 791 return; 792 793 /* 794 * Indicates early cleanup, meaning that the test has not run, 795 * such as when passing bogus args when loading the module. As 796 * such, only perform the underlying torture-specific cleanups, 797 * and avoid anything related to locktorture. 798 */ 799 if (!cxt.lwsa && !cxt.lrsa) 800 goto end; 801 802 if (writer_tasks) { 803 for (i = 0; i < cxt.nrealwriters_stress; i++) 804 torture_stop_kthread(lock_torture_writer, 805 writer_tasks[i]); 806 kfree(writer_tasks); 807 writer_tasks = NULL; 808 } 809 810 if (reader_tasks) { 811 for (i = 0; i < cxt.nrealreaders_stress; i++) 812 torture_stop_kthread(lock_torture_reader, 813 reader_tasks[i]); 814 kfree(reader_tasks); 815 reader_tasks = NULL; 816 } 817 818 torture_stop_kthread(lock_torture_stats, stats_task); 819 lock_torture_stats_print(); /* -After- the stats thread is stopped! */ 820 821 if (atomic_read(&cxt.n_lock_torture_errors)) 822 lock_torture_print_module_parms(cxt.cur_ops, 823 "End of test: FAILURE"); 824 else if (torture_onoff_failures()) 825 lock_torture_print_module_parms(cxt.cur_ops, 826 "End of test: LOCK_HOTPLUG"); 827 else 828 lock_torture_print_module_parms(cxt.cur_ops, 829 "End of test: SUCCESS"); 830 831 kfree(cxt.lwsa); 832 cxt.lwsa = NULL; 833 kfree(cxt.lrsa); 834 cxt.lrsa = NULL; 835 836 end: 837 torture_cleanup_end(); 838 } 839 840 static int __init lock_torture_init(void) 841 { 842 int i, j; 843 int firsterr = 0; 844 static struct lock_torture_ops *torture_ops[] = { 845 &lock_busted_ops, 846 &spin_lock_ops, &spin_lock_irq_ops, 847 &rw_lock_ops, &rw_lock_irq_ops, 848 &mutex_lock_ops, 849 &ww_mutex_lock_ops, 850 #ifdef CONFIG_RT_MUTEXES 851 &rtmutex_lock_ops, 852 #endif 853 &rwsem_lock_ops, 854 &percpu_rwsem_lock_ops, 855 }; 856 857 if (!torture_init_begin(torture_type, verbose)) 858 return -EBUSY; 859 860 /* Process args and tell the world that the torturer is on the job. */ 861 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 862 cxt.cur_ops = torture_ops[i]; 863 if (strcmp(torture_type, cxt.cur_ops->name) == 0) 864 break; 865 } 866 if (i == ARRAY_SIZE(torture_ops)) { 867 pr_alert("lock-torture: invalid torture type: \"%s\"\n", 868 torture_type); 869 pr_alert("lock-torture types:"); 870 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 871 pr_alert(" %s", torture_ops[i]->name); 872 pr_alert("\n"); 873 firsterr = -EINVAL; 874 goto unwind; 875 } 876 877 if (nwriters_stress == 0 && nreaders_stress == 0) { 878 pr_alert("lock-torture: must run at least one locking thread\n"); 879 firsterr = -EINVAL; 880 goto unwind; 881 } 882 883 if (cxt.cur_ops->init) 884 cxt.cur_ops->init(); 885 886 if (nwriters_stress >= 0) 887 cxt.nrealwriters_stress = nwriters_stress; 888 else 889 cxt.nrealwriters_stress = 2 * num_online_cpus(); 890 891 #ifdef CONFIG_DEBUG_MUTEXES 892 if (str_has_prefix(torture_type, "mutex")) 893 cxt.debug_lock = true; 894 #endif 895 #ifdef CONFIG_DEBUG_RT_MUTEXES 896 if (str_has_prefix(torture_type, "rtmutex")) 897 cxt.debug_lock = true; 898 #endif 899 #ifdef CONFIG_DEBUG_SPINLOCK 900 if ((str_has_prefix(torture_type, "spin")) || 901 (str_has_prefix(torture_type, "rw_lock"))) 902 cxt.debug_lock = true; 903 #endif 904 905 /* Initialize the statistics so that each run gets its own numbers. */ 906 if (nwriters_stress) { 907 lock_is_write_held = 0; 908 cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress, 909 sizeof(*cxt.lwsa), 910 GFP_KERNEL); 911 if (cxt.lwsa == NULL) { 912 VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory"); 913 firsterr = -ENOMEM; 914 goto unwind; 915 } 916 917 for (i = 0; i < cxt.nrealwriters_stress; i++) { 918 cxt.lwsa[i].n_lock_fail = 0; 919 cxt.lwsa[i].n_lock_acquired = 0; 920 } 921 } 922 923 if (cxt.cur_ops->readlock) { 924 if (nreaders_stress >= 0) 925 cxt.nrealreaders_stress = nreaders_stress; 926 else { 927 /* 928 * By default distribute evenly the number of 929 * readers and writers. We still run the same number 930 * of threads as the writer-only locks default. 931 */ 932 if (nwriters_stress < 0) /* user doesn't care */ 933 cxt.nrealwriters_stress = num_online_cpus(); 934 cxt.nrealreaders_stress = cxt.nrealwriters_stress; 935 } 936 937 if (nreaders_stress) { 938 lock_is_read_held = 0; 939 cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress, 940 sizeof(*cxt.lrsa), 941 GFP_KERNEL); 942 if (cxt.lrsa == NULL) { 943 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory"); 944 firsterr = -ENOMEM; 945 kfree(cxt.lwsa); 946 cxt.lwsa = NULL; 947 goto unwind; 948 } 949 950 for (i = 0; i < cxt.nrealreaders_stress; i++) { 951 cxt.lrsa[i].n_lock_fail = 0; 952 cxt.lrsa[i].n_lock_acquired = 0; 953 } 954 } 955 } 956 957 lock_torture_print_module_parms(cxt.cur_ops, "Start of test"); 958 959 /* Prepare torture context. */ 960 if (onoff_interval > 0) { 961 firsterr = torture_onoff_init(onoff_holdoff * HZ, 962 onoff_interval * HZ, NULL); 963 if (firsterr) 964 goto unwind; 965 } 966 if (shuffle_interval > 0) { 967 firsterr = torture_shuffle_init(shuffle_interval); 968 if (firsterr) 969 goto unwind; 970 } 971 if (shutdown_secs > 0) { 972 firsterr = torture_shutdown_init(shutdown_secs, 973 lock_torture_cleanup); 974 if (firsterr) 975 goto unwind; 976 } 977 if (stutter > 0) { 978 firsterr = torture_stutter_init(stutter, stutter); 979 if (firsterr) 980 goto unwind; 981 } 982 983 if (nwriters_stress) { 984 writer_tasks = kcalloc(cxt.nrealwriters_stress, 985 sizeof(writer_tasks[0]), 986 GFP_KERNEL); 987 if (writer_tasks == NULL) { 988 VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory"); 989 firsterr = -ENOMEM; 990 goto unwind; 991 } 992 } 993 994 if (cxt.cur_ops->readlock) { 995 reader_tasks = kcalloc(cxt.nrealreaders_stress, 996 sizeof(reader_tasks[0]), 997 GFP_KERNEL); 998 if (reader_tasks == NULL) { 999 VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory"); 1000 kfree(writer_tasks); 1001 writer_tasks = NULL; 1002 firsterr = -ENOMEM; 1003 goto unwind; 1004 } 1005 } 1006 1007 /* 1008 * Create the kthreads and start torturing (oh, those poor little locks). 1009 * 1010 * TODO: Note that we interleave writers with readers, giving writers a 1011 * slight advantage, by creating its kthread first. This can be modified 1012 * for very specific needs, or even let the user choose the policy, if 1013 * ever wanted. 1014 */ 1015 for (i = 0, j = 0; i < cxt.nrealwriters_stress || 1016 j < cxt.nrealreaders_stress; i++, j++) { 1017 if (i >= cxt.nrealwriters_stress) 1018 goto create_reader; 1019 1020 /* Create writer. */ 1021 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i], 1022 writer_tasks[i]); 1023 if (firsterr) 1024 goto unwind; 1025 1026 create_reader: 1027 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress)) 1028 continue; 1029 /* Create reader. */ 1030 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j], 1031 reader_tasks[j]); 1032 if (firsterr) 1033 goto unwind; 1034 } 1035 if (stat_interval > 0) { 1036 firsterr = torture_create_kthread(lock_torture_stats, NULL, 1037 stats_task); 1038 if (firsterr) 1039 goto unwind; 1040 } 1041 torture_init_end(); 1042 return 0; 1043 1044 unwind: 1045 torture_init_end(); 1046 lock_torture_cleanup(); 1047 return firsterr; 1048 } 1049 1050 module_init(lock_torture_init); 1051 module_exit(lock_torture_cleanup); 1052