1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Detect hard and soft lockups on a system 4 * 5 * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc. 6 * 7 * Note: Most of this code is borrowed heavily from the original softlockup 8 * detector, so thanks to Ingo for the initial implementation. 9 * Some chunks also taken from the old x86-specific nmi watchdog code, thanks 10 * to those contributors as well. 11 */ 12 13 #define pr_fmt(fmt) "watchdog: " fmt 14 15 #include <linux/mm.h> 16 #include <linux/cpu.h> 17 #include <linux/nmi.h> 18 #include <linux/init.h> 19 #include <linux/module.h> 20 #include <linux/sysctl.h> 21 #include <linux/tick.h> 22 #include <linux/sched/clock.h> 23 #include <linux/sched/debug.h> 24 #include <linux/sched/isolation.h> 25 #include <linux/stop_machine.h> 26 27 #include <asm/irq_regs.h> 28 #include <linux/kvm_para.h> 29 30 static DEFINE_MUTEX(watchdog_mutex); 31 32 #if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HARDLOCKUP_DETECTOR_SPARC64) 33 # define WATCHDOG_HARDLOCKUP_DEFAULT 1 34 #else 35 # define WATCHDOG_HARDLOCKUP_DEFAULT 0 36 #endif 37 38 unsigned long __read_mostly watchdog_enabled; 39 int __read_mostly watchdog_user_enabled = 1; 40 static int __read_mostly watchdog_hardlockup_user_enabled = WATCHDOG_HARDLOCKUP_DEFAULT; 41 static int __read_mostly watchdog_softlockup_user_enabled = 1; 42 int __read_mostly watchdog_thresh = 10; 43 static int __read_mostly watchdog_hardlockup_available; 44 45 struct cpumask watchdog_cpumask __read_mostly; 46 unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask); 47 48 #ifdef CONFIG_HARDLOCKUP_DETECTOR 49 50 # ifdef CONFIG_SMP 51 int __read_mostly sysctl_hardlockup_all_cpu_backtrace; 52 # endif /* CONFIG_SMP */ 53 54 /* 55 * Should we panic when a soft-lockup or hard-lockup occurs: 56 */ 57 unsigned int __read_mostly hardlockup_panic = 58 IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC); 59 /* 60 * We may not want to enable hard lockup detection by default in all cases, 61 * for example when running the kernel as a guest on a hypervisor. In these 62 * cases this function can be called to disable hard lockup detection. This 63 * function should only be executed once by the boot processor before the 64 * kernel command line parameters are parsed, because otherwise it is not 65 * possible to override this in hardlockup_panic_setup(). 66 */ 67 void __init hardlockup_detector_disable(void) 68 { 69 watchdog_hardlockup_user_enabled = 0; 70 } 71 72 static int __init hardlockup_panic_setup(char *str) 73 { 74 if (!strncmp(str, "panic", 5)) 75 hardlockup_panic = 1; 76 else if (!strncmp(str, "nopanic", 7)) 77 hardlockup_panic = 0; 78 else if (!strncmp(str, "0", 1)) 79 watchdog_hardlockup_user_enabled = 0; 80 else if (!strncmp(str, "1", 1)) 81 watchdog_hardlockup_user_enabled = 1; 82 return 1; 83 } 84 __setup("nmi_watchdog=", hardlockup_panic_setup); 85 86 #endif /* CONFIG_HARDLOCKUP_DETECTOR */ 87 88 #if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER) 89 90 static DEFINE_PER_CPU(atomic_t, hrtimer_interrupts); 91 static DEFINE_PER_CPU(int, hrtimer_interrupts_saved); 92 static DEFINE_PER_CPU(bool, watchdog_hardlockup_warned); 93 static DEFINE_PER_CPU(bool, watchdog_hardlockup_touched); 94 static unsigned long hard_lockup_nmi_warn; 95 96 notrace void arch_touch_nmi_watchdog(void) 97 { 98 /* 99 * Using __raw here because some code paths have 100 * preemption enabled. If preemption is enabled 101 * then interrupts should be enabled too, in which 102 * case we shouldn't have to worry about the watchdog 103 * going off. 104 */ 105 raw_cpu_write(watchdog_hardlockup_touched, true); 106 } 107 EXPORT_SYMBOL(arch_touch_nmi_watchdog); 108 109 void watchdog_hardlockup_touch_cpu(unsigned int cpu) 110 { 111 per_cpu(watchdog_hardlockup_touched, cpu) = true; 112 } 113 114 static bool is_hardlockup(unsigned int cpu) 115 { 116 int hrint = atomic_read(&per_cpu(hrtimer_interrupts, cpu)); 117 118 if (per_cpu(hrtimer_interrupts_saved, cpu) == hrint) 119 return true; 120 121 /* 122 * NOTE: we don't need any fancy atomic_t or READ_ONCE/WRITE_ONCE 123 * for hrtimer_interrupts_saved. hrtimer_interrupts_saved is 124 * written/read by a single CPU. 125 */ 126 per_cpu(hrtimer_interrupts_saved, cpu) = hrint; 127 128 return false; 129 } 130 131 static void watchdog_hardlockup_kick(void) 132 { 133 int new_interrupts; 134 135 new_interrupts = atomic_inc_return(this_cpu_ptr(&hrtimer_interrupts)); 136 watchdog_buddy_check_hardlockup(new_interrupts); 137 } 138 139 void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs) 140 { 141 if (per_cpu(watchdog_hardlockup_touched, cpu)) { 142 per_cpu(watchdog_hardlockup_touched, cpu) = false; 143 return; 144 } 145 146 /* 147 * Check for a hardlockup by making sure the CPU's timer 148 * interrupt is incrementing. The timer interrupt should have 149 * fired multiple times before we overflow'd. If it hasn't 150 * then this is a good indication the cpu is stuck 151 */ 152 if (is_hardlockup(cpu)) { 153 unsigned int this_cpu = smp_processor_id(); 154 unsigned long flags; 155 156 /* Only print hardlockups once. */ 157 if (per_cpu(watchdog_hardlockup_warned, cpu)) 158 return; 159 160 /* 161 * Prevent multiple hard-lockup reports if one cpu is already 162 * engaged in dumping all cpu back traces. 163 */ 164 if (sysctl_hardlockup_all_cpu_backtrace) { 165 if (test_and_set_bit_lock(0, &hard_lockup_nmi_warn)) 166 return; 167 } 168 169 /* 170 * NOTE: we call printk_cpu_sync_get_irqsave() after printing 171 * the lockup message. While it would be nice to serialize 172 * that printout, we really want to make sure that if some 173 * other CPU somehow locked up while holding the lock associated 174 * with printk_cpu_sync_get_irqsave() that we can still at least 175 * get the message about the lockup out. 176 */ 177 pr_emerg("Watchdog detected hard LOCKUP on cpu %d\n", cpu); 178 printk_cpu_sync_get_irqsave(flags); 179 180 print_modules(); 181 print_irqtrace_events(current); 182 if (cpu == this_cpu) { 183 if (regs) 184 show_regs(regs); 185 else 186 dump_stack(); 187 printk_cpu_sync_put_irqrestore(flags); 188 } else { 189 printk_cpu_sync_put_irqrestore(flags); 190 trigger_single_cpu_backtrace(cpu); 191 } 192 193 if (sysctl_hardlockup_all_cpu_backtrace) { 194 trigger_allbutcpu_cpu_backtrace(cpu); 195 if (!hardlockup_panic) 196 clear_bit_unlock(0, &hard_lockup_nmi_warn); 197 } 198 199 if (hardlockup_panic) 200 nmi_panic(regs, "Hard LOCKUP"); 201 202 per_cpu(watchdog_hardlockup_warned, cpu) = true; 203 } else { 204 per_cpu(watchdog_hardlockup_warned, cpu) = false; 205 } 206 } 207 208 #else /* CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */ 209 210 static inline void watchdog_hardlockup_kick(void) { } 211 212 #endif /* !CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */ 213 214 /* 215 * These functions can be overridden based on the configured hardlockdup detector. 216 * 217 * watchdog_hardlockup_enable/disable can be implemented to start and stop when 218 * softlockup watchdog start and stop. The detector must select the 219 * SOFTLOCKUP_DETECTOR Kconfig. 220 */ 221 void __weak watchdog_hardlockup_enable(unsigned int cpu) { } 222 223 void __weak watchdog_hardlockup_disable(unsigned int cpu) { } 224 225 /* 226 * Watchdog-detector specific API. 227 * 228 * Return 0 when hardlockup watchdog is available, negative value otherwise. 229 * Note that the negative value means that a delayed probe might 230 * succeed later. 231 */ 232 int __weak __init watchdog_hardlockup_probe(void) 233 { 234 return -ENODEV; 235 } 236 237 /** 238 * watchdog_hardlockup_stop - Stop the watchdog for reconfiguration 239 * 240 * The reconfiguration steps are: 241 * watchdog_hardlockup_stop(); 242 * update_variables(); 243 * watchdog_hardlockup_start(); 244 */ 245 void __weak watchdog_hardlockup_stop(void) { } 246 247 /** 248 * watchdog_hardlockup_start - Start the watchdog after reconfiguration 249 * 250 * Counterpart to watchdog_hardlockup_stop(). 251 * 252 * The following variables have been updated in update_variables() and 253 * contain the currently valid configuration: 254 * - watchdog_enabled 255 * - watchdog_thresh 256 * - watchdog_cpumask 257 */ 258 void __weak watchdog_hardlockup_start(void) { } 259 260 /** 261 * lockup_detector_update_enable - Update the sysctl enable bit 262 * 263 * Caller needs to make sure that the hard watchdogs are off, so this 264 * can't race with watchdog_hardlockup_disable(). 265 */ 266 static void lockup_detector_update_enable(void) 267 { 268 watchdog_enabled = 0; 269 if (!watchdog_user_enabled) 270 return; 271 if (watchdog_hardlockup_available && watchdog_hardlockup_user_enabled) 272 watchdog_enabled |= WATCHDOG_HARDLOCKUP_ENABLED; 273 if (watchdog_softlockup_user_enabled) 274 watchdog_enabled |= WATCHDOG_SOFTOCKUP_ENABLED; 275 } 276 277 #ifdef CONFIG_SOFTLOCKUP_DETECTOR 278 279 /* 280 * Delay the soflockup report when running a known slow code. 281 * It does _not_ affect the timestamp of the last successdul reschedule. 282 */ 283 #define SOFTLOCKUP_DELAY_REPORT ULONG_MAX 284 285 #ifdef CONFIG_SMP 286 int __read_mostly sysctl_softlockup_all_cpu_backtrace; 287 #endif 288 289 static struct cpumask watchdog_allowed_mask __read_mostly; 290 291 /* Global variables, exported for sysctl */ 292 unsigned int __read_mostly softlockup_panic = 293 IS_ENABLED(CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC); 294 295 static bool softlockup_initialized __read_mostly; 296 static u64 __read_mostly sample_period; 297 298 /* Timestamp taken after the last successful reschedule. */ 299 static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts); 300 /* Timestamp of the last softlockup report. */ 301 static DEFINE_PER_CPU(unsigned long, watchdog_report_ts); 302 static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer); 303 static DEFINE_PER_CPU(bool, softlockup_touch_sync); 304 static unsigned long soft_lockup_nmi_warn; 305 306 static int __init softlockup_panic_setup(char *str) 307 { 308 softlockup_panic = simple_strtoul(str, NULL, 0); 309 return 1; 310 } 311 __setup("softlockup_panic=", softlockup_panic_setup); 312 313 static int __init nowatchdog_setup(char *str) 314 { 315 watchdog_user_enabled = 0; 316 return 1; 317 } 318 __setup("nowatchdog", nowatchdog_setup); 319 320 static int __init nosoftlockup_setup(char *str) 321 { 322 watchdog_softlockup_user_enabled = 0; 323 return 1; 324 } 325 __setup("nosoftlockup", nosoftlockup_setup); 326 327 static int __init watchdog_thresh_setup(char *str) 328 { 329 get_option(&str, &watchdog_thresh); 330 return 1; 331 } 332 __setup("watchdog_thresh=", watchdog_thresh_setup); 333 334 static void __lockup_detector_cleanup(void); 335 336 /* 337 * Hard-lockup warnings should be triggered after just a few seconds. Soft- 338 * lockups can have false positives under extreme conditions. So we generally 339 * want a higher threshold for soft lockups than for hard lockups. So we couple 340 * the thresholds with a factor: we make the soft threshold twice the amount of 341 * time the hard threshold is. 342 */ 343 static int get_softlockup_thresh(void) 344 { 345 return watchdog_thresh * 2; 346 } 347 348 /* 349 * Returns seconds, approximately. We don't need nanosecond 350 * resolution, and we don't need to waste time with a big divide when 351 * 2^30ns == 1.074s. 352 */ 353 static unsigned long get_timestamp(void) 354 { 355 return running_clock() >> 30LL; /* 2^30 ~= 10^9 */ 356 } 357 358 static void set_sample_period(void) 359 { 360 /* 361 * convert watchdog_thresh from seconds to ns 362 * the divide by 5 is to give hrtimer several chances (two 363 * or three with the current relation between the soft 364 * and hard thresholds) to increment before the 365 * hardlockup detector generates a warning 366 */ 367 sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5); 368 watchdog_update_hrtimer_threshold(sample_period); 369 } 370 371 static void update_report_ts(void) 372 { 373 __this_cpu_write(watchdog_report_ts, get_timestamp()); 374 } 375 376 /* Commands for resetting the watchdog */ 377 static void update_touch_ts(void) 378 { 379 __this_cpu_write(watchdog_touch_ts, get_timestamp()); 380 update_report_ts(); 381 } 382 383 /** 384 * touch_softlockup_watchdog_sched - touch watchdog on scheduler stalls 385 * 386 * Call when the scheduler may have stalled for legitimate reasons 387 * preventing the watchdog task from executing - e.g. the scheduler 388 * entering idle state. This should only be used for scheduler events. 389 * Use touch_softlockup_watchdog() for everything else. 390 */ 391 notrace void touch_softlockup_watchdog_sched(void) 392 { 393 /* 394 * Preemption can be enabled. It doesn't matter which CPU's watchdog 395 * report period gets restarted here, so use the raw_ operation. 396 */ 397 raw_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT); 398 } 399 400 notrace void touch_softlockup_watchdog(void) 401 { 402 touch_softlockup_watchdog_sched(); 403 wq_watchdog_touch(raw_smp_processor_id()); 404 } 405 EXPORT_SYMBOL(touch_softlockup_watchdog); 406 407 void touch_all_softlockup_watchdogs(void) 408 { 409 int cpu; 410 411 /* 412 * watchdog_mutex cannpt be taken here, as this might be called 413 * from (soft)interrupt context, so the access to 414 * watchdog_allowed_cpumask might race with a concurrent update. 415 * 416 * The watchdog time stamp can race against a concurrent real 417 * update as well, the only side effect might be a cycle delay for 418 * the softlockup check. 419 */ 420 for_each_cpu(cpu, &watchdog_allowed_mask) { 421 per_cpu(watchdog_report_ts, cpu) = SOFTLOCKUP_DELAY_REPORT; 422 wq_watchdog_touch(cpu); 423 } 424 } 425 426 void touch_softlockup_watchdog_sync(void) 427 { 428 __this_cpu_write(softlockup_touch_sync, true); 429 __this_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT); 430 } 431 432 static int is_softlockup(unsigned long touch_ts, 433 unsigned long period_ts, 434 unsigned long now) 435 { 436 if ((watchdog_enabled & WATCHDOG_SOFTOCKUP_ENABLED) && watchdog_thresh) { 437 /* Warn about unreasonable delays. */ 438 if (time_after(now, period_ts + get_softlockup_thresh())) 439 return now - touch_ts; 440 } 441 return 0; 442 } 443 444 /* watchdog detector functions */ 445 static DEFINE_PER_CPU(struct completion, softlockup_completion); 446 static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work); 447 448 /* 449 * The watchdog feed function - touches the timestamp. 450 * 451 * It only runs once every sample_period seconds (4 seconds by 452 * default) to reset the softlockup timestamp. If this gets delayed 453 * for more than 2*watchdog_thresh seconds then the debug-printout 454 * triggers in watchdog_timer_fn(). 455 */ 456 static int softlockup_fn(void *data) 457 { 458 update_touch_ts(); 459 complete(this_cpu_ptr(&softlockup_completion)); 460 461 return 0; 462 } 463 464 /* watchdog kicker functions */ 465 static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) 466 { 467 unsigned long touch_ts, period_ts, now; 468 struct pt_regs *regs = get_irq_regs(); 469 int duration; 470 int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace; 471 unsigned long flags; 472 473 if (!watchdog_enabled) 474 return HRTIMER_NORESTART; 475 476 watchdog_hardlockup_kick(); 477 478 /* kick the softlockup detector */ 479 if (completion_done(this_cpu_ptr(&softlockup_completion))) { 480 reinit_completion(this_cpu_ptr(&softlockup_completion)); 481 stop_one_cpu_nowait(smp_processor_id(), 482 softlockup_fn, NULL, 483 this_cpu_ptr(&softlockup_stop_work)); 484 } 485 486 /* .. and repeat */ 487 hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period)); 488 489 /* 490 * Read the current timestamp first. It might become invalid anytime 491 * when a virtual machine is stopped by the host or when the watchog 492 * is touched from NMI. 493 */ 494 now = get_timestamp(); 495 /* 496 * If a virtual machine is stopped by the host it can look to 497 * the watchdog like a soft lockup. This function touches the watchdog. 498 */ 499 kvm_check_and_clear_guest_paused(); 500 /* 501 * The stored timestamp is comparable with @now only when not touched. 502 * It might get touched anytime from NMI. Make sure that is_softlockup() 503 * uses the same (valid) value. 504 */ 505 period_ts = READ_ONCE(*this_cpu_ptr(&watchdog_report_ts)); 506 507 /* Reset the interval when touched by known problematic code. */ 508 if (period_ts == SOFTLOCKUP_DELAY_REPORT) { 509 if (unlikely(__this_cpu_read(softlockup_touch_sync))) { 510 /* 511 * If the time stamp was touched atomically 512 * make sure the scheduler tick is up to date. 513 */ 514 __this_cpu_write(softlockup_touch_sync, false); 515 sched_clock_tick(); 516 } 517 518 update_report_ts(); 519 return HRTIMER_RESTART; 520 } 521 522 /* Check for a softlockup. */ 523 touch_ts = __this_cpu_read(watchdog_touch_ts); 524 duration = is_softlockup(touch_ts, period_ts, now); 525 if (unlikely(duration)) { 526 /* 527 * Prevent multiple soft-lockup reports if one cpu is already 528 * engaged in dumping all cpu back traces. 529 */ 530 if (softlockup_all_cpu_backtrace) { 531 if (test_and_set_bit_lock(0, &soft_lockup_nmi_warn)) 532 return HRTIMER_RESTART; 533 } 534 535 /* Start period for the next softlockup warning. */ 536 update_report_ts(); 537 538 printk_cpu_sync_get_irqsave(flags); 539 pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n", 540 smp_processor_id(), duration, 541 current->comm, task_pid_nr(current)); 542 print_modules(); 543 print_irqtrace_events(current); 544 if (regs) 545 show_regs(regs); 546 else 547 dump_stack(); 548 printk_cpu_sync_put_irqrestore(flags); 549 550 if (softlockup_all_cpu_backtrace) { 551 trigger_allbutcpu_cpu_backtrace(smp_processor_id()); 552 if (!softlockup_panic) 553 clear_bit_unlock(0, &soft_lockup_nmi_warn); 554 } 555 556 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); 557 if (softlockup_panic) 558 panic("softlockup: hung tasks"); 559 } 560 561 return HRTIMER_RESTART; 562 } 563 564 static void watchdog_enable(unsigned int cpu) 565 { 566 struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer); 567 struct completion *done = this_cpu_ptr(&softlockup_completion); 568 569 WARN_ON_ONCE(cpu != smp_processor_id()); 570 571 init_completion(done); 572 complete(done); 573 574 /* 575 * Start the timer first to prevent the hardlockup watchdog triggering 576 * before the timer has a chance to fire. 577 */ 578 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); 579 hrtimer->function = watchdog_timer_fn; 580 hrtimer_start(hrtimer, ns_to_ktime(sample_period), 581 HRTIMER_MODE_REL_PINNED_HARD); 582 583 /* Initialize timestamp */ 584 update_touch_ts(); 585 /* Enable the hardlockup detector */ 586 if (watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED) 587 watchdog_hardlockup_enable(cpu); 588 } 589 590 static void watchdog_disable(unsigned int cpu) 591 { 592 struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer); 593 594 WARN_ON_ONCE(cpu != smp_processor_id()); 595 596 /* 597 * Disable the hardlockup detector first. That prevents that a large 598 * delay between disabling the timer and disabling the hardlockup 599 * detector causes a false positive. 600 */ 601 watchdog_hardlockup_disable(cpu); 602 hrtimer_cancel(hrtimer); 603 wait_for_completion(this_cpu_ptr(&softlockup_completion)); 604 } 605 606 static int softlockup_stop_fn(void *data) 607 { 608 watchdog_disable(smp_processor_id()); 609 return 0; 610 } 611 612 static void softlockup_stop_all(void) 613 { 614 int cpu; 615 616 if (!softlockup_initialized) 617 return; 618 619 for_each_cpu(cpu, &watchdog_allowed_mask) 620 smp_call_on_cpu(cpu, softlockup_stop_fn, NULL, false); 621 622 cpumask_clear(&watchdog_allowed_mask); 623 } 624 625 static int softlockup_start_fn(void *data) 626 { 627 watchdog_enable(smp_processor_id()); 628 return 0; 629 } 630 631 static void softlockup_start_all(void) 632 { 633 int cpu; 634 635 cpumask_copy(&watchdog_allowed_mask, &watchdog_cpumask); 636 for_each_cpu(cpu, &watchdog_allowed_mask) 637 smp_call_on_cpu(cpu, softlockup_start_fn, NULL, false); 638 } 639 640 int lockup_detector_online_cpu(unsigned int cpu) 641 { 642 if (cpumask_test_cpu(cpu, &watchdog_allowed_mask)) 643 watchdog_enable(cpu); 644 return 0; 645 } 646 647 int lockup_detector_offline_cpu(unsigned int cpu) 648 { 649 if (cpumask_test_cpu(cpu, &watchdog_allowed_mask)) 650 watchdog_disable(cpu); 651 return 0; 652 } 653 654 static void __lockup_detector_reconfigure(void) 655 { 656 cpus_read_lock(); 657 watchdog_hardlockup_stop(); 658 659 softlockup_stop_all(); 660 set_sample_period(); 661 lockup_detector_update_enable(); 662 if (watchdog_enabled && watchdog_thresh) 663 softlockup_start_all(); 664 665 watchdog_hardlockup_start(); 666 cpus_read_unlock(); 667 /* 668 * Must be called outside the cpus locked section to prevent 669 * recursive locking in the perf code. 670 */ 671 __lockup_detector_cleanup(); 672 } 673 674 void lockup_detector_reconfigure(void) 675 { 676 mutex_lock(&watchdog_mutex); 677 __lockup_detector_reconfigure(); 678 mutex_unlock(&watchdog_mutex); 679 } 680 681 /* 682 * Create the watchdog infrastructure and configure the detector(s). 683 */ 684 static __init void lockup_detector_setup(void) 685 { 686 /* 687 * If sysctl is off and watchdog got disabled on the command line, 688 * nothing to do here. 689 */ 690 lockup_detector_update_enable(); 691 692 if (!IS_ENABLED(CONFIG_SYSCTL) && 693 !(watchdog_enabled && watchdog_thresh)) 694 return; 695 696 mutex_lock(&watchdog_mutex); 697 __lockup_detector_reconfigure(); 698 softlockup_initialized = true; 699 mutex_unlock(&watchdog_mutex); 700 } 701 702 #else /* CONFIG_SOFTLOCKUP_DETECTOR */ 703 static void __lockup_detector_reconfigure(void) 704 { 705 cpus_read_lock(); 706 watchdog_hardlockup_stop(); 707 lockup_detector_update_enable(); 708 watchdog_hardlockup_start(); 709 cpus_read_unlock(); 710 } 711 void lockup_detector_reconfigure(void) 712 { 713 __lockup_detector_reconfigure(); 714 } 715 static inline void lockup_detector_setup(void) 716 { 717 __lockup_detector_reconfigure(); 718 } 719 #endif /* !CONFIG_SOFTLOCKUP_DETECTOR */ 720 721 static void __lockup_detector_cleanup(void) 722 { 723 lockdep_assert_held(&watchdog_mutex); 724 hardlockup_detector_perf_cleanup(); 725 } 726 727 /** 728 * lockup_detector_cleanup - Cleanup after cpu hotplug or sysctl changes 729 * 730 * Caller must not hold the cpu hotplug rwsem. 731 */ 732 void lockup_detector_cleanup(void) 733 { 734 mutex_lock(&watchdog_mutex); 735 __lockup_detector_cleanup(); 736 mutex_unlock(&watchdog_mutex); 737 } 738 739 /** 740 * lockup_detector_soft_poweroff - Interface to stop lockup detector(s) 741 * 742 * Special interface for parisc. It prevents lockup detector warnings from 743 * the default pm_poweroff() function which busy loops forever. 744 */ 745 void lockup_detector_soft_poweroff(void) 746 { 747 watchdog_enabled = 0; 748 } 749 750 #ifdef CONFIG_SYSCTL 751 752 /* Propagate any changes to the watchdog infrastructure */ 753 static void proc_watchdog_update(void) 754 { 755 /* Remove impossible cpus to keep sysctl output clean. */ 756 cpumask_and(&watchdog_cpumask, &watchdog_cpumask, cpu_possible_mask); 757 __lockup_detector_reconfigure(); 758 } 759 760 /* 761 * common function for watchdog, nmi_watchdog and soft_watchdog parameter 762 * 763 * caller | table->data points to | 'which' 764 * -------------------|----------------------------------|------------------------------- 765 * proc_watchdog | watchdog_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED | 766 * | | WATCHDOG_SOFTOCKUP_ENABLED 767 * -------------------|----------------------------------|------------------------------- 768 * proc_nmi_watchdog | watchdog_hardlockup_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED 769 * -------------------|----------------------------------|------------------------------- 770 * proc_soft_watchdog | watchdog_softlockup_user_enabled | WATCHDOG_SOFTOCKUP_ENABLED 771 */ 772 static int proc_watchdog_common(int which, struct ctl_table *table, int write, 773 void *buffer, size_t *lenp, loff_t *ppos) 774 { 775 int err, old, *param = table->data; 776 777 mutex_lock(&watchdog_mutex); 778 779 if (!write) { 780 /* 781 * On read synchronize the userspace interface. This is a 782 * racy snapshot. 783 */ 784 *param = (watchdog_enabled & which) != 0; 785 err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 786 } else { 787 old = READ_ONCE(*param); 788 err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 789 if (!err && old != READ_ONCE(*param)) 790 proc_watchdog_update(); 791 } 792 mutex_unlock(&watchdog_mutex); 793 return err; 794 } 795 796 /* 797 * /proc/sys/kernel/watchdog 798 */ 799 int proc_watchdog(struct ctl_table *table, int write, 800 void *buffer, size_t *lenp, loff_t *ppos) 801 { 802 return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED | 803 WATCHDOG_SOFTOCKUP_ENABLED, 804 table, write, buffer, lenp, ppos); 805 } 806 807 /* 808 * /proc/sys/kernel/nmi_watchdog 809 */ 810 int proc_nmi_watchdog(struct ctl_table *table, int write, 811 void *buffer, size_t *lenp, loff_t *ppos) 812 { 813 if (!watchdog_hardlockup_available && write) 814 return -ENOTSUPP; 815 return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED, 816 table, write, buffer, lenp, ppos); 817 } 818 819 /* 820 * /proc/sys/kernel/soft_watchdog 821 */ 822 int proc_soft_watchdog(struct ctl_table *table, int write, 823 void *buffer, size_t *lenp, loff_t *ppos) 824 { 825 return proc_watchdog_common(WATCHDOG_SOFTOCKUP_ENABLED, 826 table, write, buffer, lenp, ppos); 827 } 828 829 /* 830 * /proc/sys/kernel/watchdog_thresh 831 */ 832 int proc_watchdog_thresh(struct ctl_table *table, int write, 833 void *buffer, size_t *lenp, loff_t *ppos) 834 { 835 int err, old; 836 837 mutex_lock(&watchdog_mutex); 838 839 old = READ_ONCE(watchdog_thresh); 840 err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 841 842 if (!err && write && old != READ_ONCE(watchdog_thresh)) 843 proc_watchdog_update(); 844 845 mutex_unlock(&watchdog_mutex); 846 return err; 847 } 848 849 /* 850 * The cpumask is the mask of possible cpus that the watchdog can run 851 * on, not the mask of cpus it is actually running on. This allows the 852 * user to specify a mask that will include cpus that have not yet 853 * been brought online, if desired. 854 */ 855 int proc_watchdog_cpumask(struct ctl_table *table, int write, 856 void *buffer, size_t *lenp, loff_t *ppos) 857 { 858 int err; 859 860 mutex_lock(&watchdog_mutex); 861 862 err = proc_do_large_bitmap(table, write, buffer, lenp, ppos); 863 if (!err && write) 864 proc_watchdog_update(); 865 866 mutex_unlock(&watchdog_mutex); 867 return err; 868 } 869 870 static const int sixty = 60; 871 872 static struct ctl_table watchdog_sysctls[] = { 873 { 874 .procname = "watchdog", 875 .data = &watchdog_user_enabled, 876 .maxlen = sizeof(int), 877 .mode = 0644, 878 .proc_handler = proc_watchdog, 879 .extra1 = SYSCTL_ZERO, 880 .extra2 = SYSCTL_ONE, 881 }, 882 { 883 .procname = "watchdog_thresh", 884 .data = &watchdog_thresh, 885 .maxlen = sizeof(int), 886 .mode = 0644, 887 .proc_handler = proc_watchdog_thresh, 888 .extra1 = SYSCTL_ZERO, 889 .extra2 = (void *)&sixty, 890 }, 891 { 892 .procname = "watchdog_cpumask", 893 .data = &watchdog_cpumask_bits, 894 .maxlen = NR_CPUS, 895 .mode = 0644, 896 .proc_handler = proc_watchdog_cpumask, 897 }, 898 #ifdef CONFIG_SOFTLOCKUP_DETECTOR 899 { 900 .procname = "soft_watchdog", 901 .data = &watchdog_softlockup_user_enabled, 902 .maxlen = sizeof(int), 903 .mode = 0644, 904 .proc_handler = proc_soft_watchdog, 905 .extra1 = SYSCTL_ZERO, 906 .extra2 = SYSCTL_ONE, 907 }, 908 { 909 .procname = "softlockup_panic", 910 .data = &softlockup_panic, 911 .maxlen = sizeof(int), 912 .mode = 0644, 913 .proc_handler = proc_dointvec_minmax, 914 .extra1 = SYSCTL_ZERO, 915 .extra2 = SYSCTL_ONE, 916 }, 917 #ifdef CONFIG_SMP 918 { 919 .procname = "softlockup_all_cpu_backtrace", 920 .data = &sysctl_softlockup_all_cpu_backtrace, 921 .maxlen = sizeof(int), 922 .mode = 0644, 923 .proc_handler = proc_dointvec_minmax, 924 .extra1 = SYSCTL_ZERO, 925 .extra2 = SYSCTL_ONE, 926 }, 927 #endif /* CONFIG_SMP */ 928 #endif 929 #ifdef CONFIG_HARDLOCKUP_DETECTOR 930 { 931 .procname = "hardlockup_panic", 932 .data = &hardlockup_panic, 933 .maxlen = sizeof(int), 934 .mode = 0644, 935 .proc_handler = proc_dointvec_minmax, 936 .extra1 = SYSCTL_ZERO, 937 .extra2 = SYSCTL_ONE, 938 }, 939 #ifdef CONFIG_SMP 940 { 941 .procname = "hardlockup_all_cpu_backtrace", 942 .data = &sysctl_hardlockup_all_cpu_backtrace, 943 .maxlen = sizeof(int), 944 .mode = 0644, 945 .proc_handler = proc_dointvec_minmax, 946 .extra1 = SYSCTL_ZERO, 947 .extra2 = SYSCTL_ONE, 948 }, 949 #endif /* CONFIG_SMP */ 950 #endif 951 {} 952 }; 953 954 static struct ctl_table watchdog_hardlockup_sysctl[] = { 955 { 956 .procname = "nmi_watchdog", 957 .data = &watchdog_hardlockup_user_enabled, 958 .maxlen = sizeof(int), 959 .mode = 0444, 960 .proc_handler = proc_nmi_watchdog, 961 .extra1 = SYSCTL_ZERO, 962 .extra2 = SYSCTL_ONE, 963 }, 964 {} 965 }; 966 967 static void __init watchdog_sysctl_init(void) 968 { 969 register_sysctl_init("kernel", watchdog_sysctls); 970 971 if (watchdog_hardlockup_available) 972 watchdog_hardlockup_sysctl[0].mode = 0644; 973 register_sysctl_init("kernel", watchdog_hardlockup_sysctl); 974 } 975 976 #else 977 #define watchdog_sysctl_init() do { } while (0) 978 #endif /* CONFIG_SYSCTL */ 979 980 static void __init lockup_detector_delay_init(struct work_struct *work); 981 static bool allow_lockup_detector_init_retry __initdata; 982 983 static struct work_struct detector_work __initdata = 984 __WORK_INITIALIZER(detector_work, lockup_detector_delay_init); 985 986 static void __init lockup_detector_delay_init(struct work_struct *work) 987 { 988 int ret; 989 990 ret = watchdog_hardlockup_probe(); 991 if (ret) { 992 pr_info("Delayed init of the lockup detector failed: %d\n", ret); 993 pr_info("Hard watchdog permanently disabled\n"); 994 return; 995 } 996 997 allow_lockup_detector_init_retry = false; 998 999 watchdog_hardlockup_available = true; 1000 lockup_detector_setup(); 1001 } 1002 1003 /* 1004 * lockup_detector_retry_init - retry init lockup detector if possible. 1005 * 1006 * Retry hardlockup detector init. It is useful when it requires some 1007 * functionality that has to be initialized later on a particular 1008 * platform. 1009 */ 1010 void __init lockup_detector_retry_init(void) 1011 { 1012 /* Must be called before late init calls */ 1013 if (!allow_lockup_detector_init_retry) 1014 return; 1015 1016 schedule_work(&detector_work); 1017 } 1018 1019 /* 1020 * Ensure that optional delayed hardlockup init is proceed before 1021 * the init code and memory is freed. 1022 */ 1023 static int __init lockup_detector_check(void) 1024 { 1025 /* Prevent any later retry. */ 1026 allow_lockup_detector_init_retry = false; 1027 1028 /* Make sure no work is pending. */ 1029 flush_work(&detector_work); 1030 1031 watchdog_sysctl_init(); 1032 1033 return 0; 1034 1035 } 1036 late_initcall_sync(lockup_detector_check); 1037 1038 void __init lockup_detector_init(void) 1039 { 1040 if (tick_nohz_full_enabled()) 1041 pr_info("Disabling watchdog on nohz_full cores by default\n"); 1042 1043 cpumask_copy(&watchdog_cpumask, 1044 housekeeping_cpumask(HK_TYPE_TIMER)); 1045 1046 if (!watchdog_hardlockup_probe()) 1047 watchdog_hardlockup_available = true; 1048 else 1049 allow_lockup_detector_init_retry = true; 1050 1051 lockup_detector_setup(); 1052 } 1053