1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/kernel.h> 34 #include <linux/module.h> 35 #include <linux/random.h> 36 #include <linux/vmalloc.h> 37 #include <linux/hardirq.h> 38 #include <linux/mlx5/driver.h> 39 #include "mlx5_core.h" 40 #include "lib/eq.h" 41 #include "lib/mlx5.h" 42 #include "lib/pci_vsc.h" 43 #include "diag/fw_tracer.h" 44 45 enum { 46 MLX5_HEALTH_POLL_INTERVAL = 2 * HZ, 47 MAX_MISSES = 3, 48 }; 49 50 enum { 51 MLX5_HEALTH_SYNDR_FW_ERR = 0x1, 52 MLX5_HEALTH_SYNDR_IRISC_ERR = 0x7, 53 MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR = 0x8, 54 MLX5_HEALTH_SYNDR_CRC_ERR = 0x9, 55 MLX5_HEALTH_SYNDR_FETCH_PCI_ERR = 0xa, 56 MLX5_HEALTH_SYNDR_HW_FTL_ERR = 0xb, 57 MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR = 0xc, 58 MLX5_HEALTH_SYNDR_EQ_ERR = 0xd, 59 MLX5_HEALTH_SYNDR_EQ_INV = 0xe, 60 MLX5_HEALTH_SYNDR_FFSER_ERR = 0xf, 61 MLX5_HEALTH_SYNDR_HIGH_TEMP = 0x10 62 }; 63 64 enum { 65 MLX5_DROP_NEW_HEALTH_WORK, 66 }; 67 68 enum { 69 MLX5_SENSOR_NO_ERR = 0, 70 MLX5_SENSOR_PCI_COMM_ERR = 1, 71 MLX5_SENSOR_PCI_ERR = 2, 72 MLX5_SENSOR_NIC_DISABLED = 3, 73 MLX5_SENSOR_NIC_SW_RESET = 4, 74 MLX5_SENSOR_FW_SYND_RFR = 5, 75 }; 76 77 u8 mlx5_get_nic_state(struct mlx5_core_dev *dev) 78 { 79 return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 7; 80 } 81 82 void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state) 83 { 84 u32 cur_cmdq_addr_l_sz; 85 86 cur_cmdq_addr_l_sz = ioread32be(&dev->iseg->cmdq_addr_l_sz); 87 iowrite32be((cur_cmdq_addr_l_sz & 0xFFFFF000) | 88 state << MLX5_NIC_IFC_OFFSET, 89 &dev->iseg->cmdq_addr_l_sz); 90 } 91 92 static bool sensor_pci_not_working(struct mlx5_core_dev *dev) 93 { 94 struct mlx5_core_health *health = &dev->priv.health; 95 struct health_buffer __iomem *h = health->health; 96 97 /* Offline PCI reads return 0xffffffff */ 98 return (ioread32be(&h->fw_ver) == 0xffffffff); 99 } 100 101 static bool sensor_fw_synd_rfr(struct mlx5_core_dev *dev) 102 { 103 struct mlx5_core_health *health = &dev->priv.health; 104 struct health_buffer __iomem *h = health->health; 105 u32 rfr = ioread32be(&h->rfr) >> MLX5_RFR_OFFSET; 106 u8 synd = ioread8(&h->synd); 107 108 if (rfr && synd) 109 mlx5_core_dbg(dev, "FW requests reset, synd: %d\n", synd); 110 return rfr && synd; 111 } 112 113 static u32 check_fatal_sensors(struct mlx5_core_dev *dev) 114 { 115 if (sensor_pci_not_working(dev)) 116 return MLX5_SENSOR_PCI_COMM_ERR; 117 if (pci_channel_offline(dev->pdev)) 118 return MLX5_SENSOR_PCI_ERR; 119 if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) 120 return MLX5_SENSOR_NIC_DISABLED; 121 if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_SW_RESET) 122 return MLX5_SENSOR_NIC_SW_RESET; 123 if (sensor_fw_synd_rfr(dev)) 124 return MLX5_SENSOR_FW_SYND_RFR; 125 126 return MLX5_SENSOR_NO_ERR; 127 } 128 129 static int lock_sem_sw_reset(struct mlx5_core_dev *dev, bool lock) 130 { 131 enum mlx5_vsc_state state; 132 int ret; 133 134 if (!mlx5_core_is_pf(dev)) 135 return -EBUSY; 136 137 /* Try to lock GW access, this stage doesn't return 138 * EBUSY because locked GW does not mean that other PF 139 * already started the reset. 140 */ 141 ret = mlx5_vsc_gw_lock(dev); 142 if (ret == -EBUSY) 143 return -EINVAL; 144 if (ret) 145 return ret; 146 147 state = lock ? MLX5_VSC_LOCK : MLX5_VSC_UNLOCK; 148 /* At this stage, if the return status == EBUSY, then we know 149 * for sure that another PF started the reset, so don't allow 150 * another reset. 151 */ 152 ret = mlx5_vsc_sem_set_space(dev, MLX5_SEMAPHORE_SW_RESET, state); 153 if (ret) 154 mlx5_core_warn(dev, "Failed to lock SW reset semaphore\n"); 155 156 /* Unlock GW access */ 157 mlx5_vsc_gw_unlock(dev); 158 159 return ret; 160 } 161 162 static bool reset_fw_if_needed(struct mlx5_core_dev *dev) 163 { 164 bool supported = (ioread32be(&dev->iseg->initializing) >> 165 MLX5_FW_RESET_SUPPORTED_OFFSET) & 1; 166 u32 fatal_error; 167 168 if (!supported) 169 return false; 170 171 /* The reset only needs to be issued by one PF. The health buffer is 172 * shared between all functions, and will be cleared during a reset. 173 * Check again to avoid a redundant 2nd reset. If the fatal erros was 174 * PCI related a reset won't help. 175 */ 176 fatal_error = check_fatal_sensors(dev); 177 if (fatal_error == MLX5_SENSOR_PCI_COMM_ERR || 178 fatal_error == MLX5_SENSOR_NIC_DISABLED || 179 fatal_error == MLX5_SENSOR_NIC_SW_RESET) { 180 mlx5_core_warn(dev, "Not issuing FW reset. Either it's already done or won't help."); 181 return false; 182 } 183 184 mlx5_core_warn(dev, "Issuing FW Reset\n"); 185 /* Write the NIC interface field to initiate the reset, the command 186 * interface address also resides here, don't overwrite it. 187 */ 188 mlx5_set_nic_state(dev, MLX5_NIC_IFC_SW_RESET); 189 190 return true; 191 } 192 193 void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force) 194 { 195 bool err_detected = false; 196 197 /* Mark the device as fatal in order to abort FW commands */ 198 if ((check_fatal_sensors(dev) || force) && 199 dev->state == MLX5_DEVICE_STATE_UP) { 200 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 201 err_detected = true; 202 } 203 mutex_lock(&dev->intf_state_mutex); 204 if (!err_detected && dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 205 goto unlock;/* a previous error is still being handled */ 206 if (dev->state == MLX5_DEVICE_STATE_UNINITIALIZED) { 207 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 208 goto unlock; 209 } 210 211 if (check_fatal_sensors(dev) || force) { /* protected state setting */ 212 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 213 mlx5_cmd_flush(dev); 214 } 215 216 mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1); 217 unlock: 218 mutex_unlock(&dev->intf_state_mutex); 219 } 220 221 #define MLX5_CRDUMP_WAIT_MS 60000 222 #define MLX5_FW_RESET_WAIT_MS 1000 223 void mlx5_error_sw_reset(struct mlx5_core_dev *dev) 224 { 225 unsigned long end, delay_ms = MLX5_FW_RESET_WAIT_MS; 226 int lock = -EBUSY; 227 228 mutex_lock(&dev->intf_state_mutex); 229 if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) 230 goto unlock; 231 232 mlx5_core_err(dev, "start\n"); 233 234 if (check_fatal_sensors(dev) == MLX5_SENSOR_FW_SYND_RFR) { 235 /* Get cr-dump and reset FW semaphore */ 236 lock = lock_sem_sw_reset(dev, true); 237 238 if (lock == -EBUSY) { 239 delay_ms = MLX5_CRDUMP_WAIT_MS; 240 goto recover_from_sw_reset; 241 } 242 /* Execute SW reset */ 243 reset_fw_if_needed(dev); 244 } 245 246 recover_from_sw_reset: 247 /* Recover from SW reset */ 248 end = jiffies + msecs_to_jiffies(delay_ms); 249 do { 250 if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) 251 break; 252 253 msleep(20); 254 } while (!time_after(jiffies, end)); 255 256 if (mlx5_get_nic_state(dev) != MLX5_NIC_IFC_DISABLED) { 257 dev_err(&dev->pdev->dev, "NIC IFC still %d after %lums.\n", 258 mlx5_get_nic_state(dev), delay_ms); 259 } 260 261 /* Release FW semaphore if you are the lock owner */ 262 if (!lock) 263 lock_sem_sw_reset(dev, false); 264 265 mlx5_core_err(dev, "end\n"); 266 267 unlock: 268 mutex_unlock(&dev->intf_state_mutex); 269 } 270 271 static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) 272 { 273 u8 nic_interface = mlx5_get_nic_state(dev); 274 275 switch (nic_interface) { 276 case MLX5_NIC_IFC_FULL: 277 mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n"); 278 break; 279 280 case MLX5_NIC_IFC_DISABLED: 281 mlx5_core_warn(dev, "starting teardown\n"); 282 break; 283 284 case MLX5_NIC_IFC_NO_DRAM_NIC: 285 mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n"); 286 break; 287 288 case MLX5_NIC_IFC_SW_RESET: 289 /* The IFC mode field is 3 bits, so it will read 0x7 in 2 cases: 290 * 1. PCI has been disabled (ie. PCI-AER, PF driver unloaded 291 * and this is a VF), this is not recoverable by SW reset. 292 * Logging of this is handled elsewhere. 293 * 2. FW reset has been issued by another function, driver can 294 * be reloaded to recover after the mode switches to 295 * MLX5_NIC_IFC_DISABLED. 296 */ 297 if (dev->priv.health.fatal_error != MLX5_SENSOR_PCI_COMM_ERR) 298 mlx5_core_warn(dev, "NIC SW reset in progress\n"); 299 break; 300 301 default: 302 mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n", 303 nic_interface); 304 } 305 306 mlx5_disable_device(dev); 307 } 308 309 /* How much time to wait until health resetting the driver (in msecs) */ 310 #define MLX5_RECOVERY_WAIT_MSECS 60000 311 static int mlx5_health_try_recover(struct mlx5_core_dev *dev) 312 { 313 unsigned long end; 314 315 mlx5_core_warn(dev, "handling bad device here\n"); 316 mlx5_handle_bad_state(dev); 317 end = jiffies + msecs_to_jiffies(MLX5_RECOVERY_WAIT_MSECS); 318 while (sensor_pci_not_working(dev)) { 319 if (time_after(jiffies, end)) { 320 mlx5_core_err(dev, 321 "health recovery flow aborted, PCI reads still not working\n"); 322 return -EIO; 323 } 324 msleep(100); 325 } 326 327 mlx5_core_err(dev, "starting health recovery flow\n"); 328 mlx5_recover_device(dev); 329 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state) || 330 check_fatal_sensors(dev)) { 331 mlx5_core_err(dev, "health recovery failed\n"); 332 return -EIO; 333 } 334 return 0; 335 } 336 337 static const char *hsynd_str(u8 synd) 338 { 339 switch (synd) { 340 case MLX5_HEALTH_SYNDR_FW_ERR: 341 return "firmware internal error"; 342 case MLX5_HEALTH_SYNDR_IRISC_ERR: 343 return "irisc not responding"; 344 case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR: 345 return "unrecoverable hardware error"; 346 case MLX5_HEALTH_SYNDR_CRC_ERR: 347 return "firmware CRC error"; 348 case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR: 349 return "ICM fetch PCI error"; 350 case MLX5_HEALTH_SYNDR_HW_FTL_ERR: 351 return "HW fatal error\n"; 352 case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR: 353 return "async EQ buffer overrun"; 354 case MLX5_HEALTH_SYNDR_EQ_ERR: 355 return "EQ error"; 356 case MLX5_HEALTH_SYNDR_EQ_INV: 357 return "Invalid EQ referenced"; 358 case MLX5_HEALTH_SYNDR_FFSER_ERR: 359 return "FFSER error"; 360 case MLX5_HEALTH_SYNDR_HIGH_TEMP: 361 return "High temperature"; 362 default: 363 return "unrecognized error"; 364 } 365 } 366 367 static void print_health_info(struct mlx5_core_dev *dev) 368 { 369 struct mlx5_core_health *health = &dev->priv.health; 370 struct health_buffer __iomem *h = health->health; 371 char fw_str[18]; 372 u32 fw; 373 int i; 374 375 /* If the syndrome is 0, the device is OK and no need to print buffer */ 376 if (!ioread8(&h->synd)) 377 return; 378 379 for (i = 0; i < ARRAY_SIZE(h->assert_var); i++) 380 mlx5_core_err(dev, "assert_var[%d] 0x%08x\n", i, 381 ioread32be(h->assert_var + i)); 382 383 mlx5_core_err(dev, "assert_exit_ptr 0x%08x\n", 384 ioread32be(&h->assert_exit_ptr)); 385 mlx5_core_err(dev, "assert_callra 0x%08x\n", 386 ioread32be(&h->assert_callra)); 387 sprintf(fw_str, "%d.%d.%d", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev)); 388 mlx5_core_err(dev, "fw_ver %s\n", fw_str); 389 mlx5_core_err(dev, "hw_id 0x%08x\n", ioread32be(&h->hw_id)); 390 mlx5_core_err(dev, "irisc_index %d\n", ioread8(&h->irisc_index)); 391 mlx5_core_err(dev, "synd 0x%x: %s\n", ioread8(&h->synd), 392 hsynd_str(ioread8(&h->synd))); 393 mlx5_core_err(dev, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd)); 394 fw = ioread32be(&h->fw_ver); 395 mlx5_core_err(dev, "raw fw_ver 0x%08x\n", fw); 396 } 397 398 static int 399 mlx5_fw_reporter_diagnose(struct devlink_health_reporter *reporter, 400 struct devlink_fmsg *fmsg, 401 struct netlink_ext_ack *extack) 402 { 403 struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter); 404 struct mlx5_core_health *health = &dev->priv.health; 405 struct health_buffer __iomem *h = health->health; 406 u8 synd; 407 int err; 408 409 synd = ioread8(&h->synd); 410 err = devlink_fmsg_u8_pair_put(fmsg, "Syndrome", synd); 411 if (err || !synd) 412 return err; 413 return devlink_fmsg_string_pair_put(fmsg, "Description", hsynd_str(synd)); 414 } 415 416 struct mlx5_fw_reporter_ctx { 417 u8 err_synd; 418 int miss_counter; 419 }; 420 421 static int 422 mlx5_fw_reporter_ctx_pairs_put(struct devlink_fmsg *fmsg, 423 struct mlx5_fw_reporter_ctx *fw_reporter_ctx) 424 { 425 int err; 426 427 err = devlink_fmsg_u8_pair_put(fmsg, "syndrome", 428 fw_reporter_ctx->err_synd); 429 if (err) 430 return err; 431 err = devlink_fmsg_u32_pair_put(fmsg, "fw_miss_counter", 432 fw_reporter_ctx->miss_counter); 433 if (err) 434 return err; 435 return 0; 436 } 437 438 static int 439 mlx5_fw_reporter_heath_buffer_data_put(struct mlx5_core_dev *dev, 440 struct devlink_fmsg *fmsg) 441 { 442 struct mlx5_core_health *health = &dev->priv.health; 443 struct health_buffer __iomem *h = health->health; 444 int err; 445 int i; 446 447 if (!ioread8(&h->synd)) 448 return 0; 449 450 err = devlink_fmsg_pair_nest_start(fmsg, "health buffer"); 451 if (err) 452 return err; 453 err = devlink_fmsg_obj_nest_start(fmsg); 454 if (err) 455 return err; 456 err = devlink_fmsg_arr_pair_nest_start(fmsg, "assert_var"); 457 if (err) 458 return err; 459 460 for (i = 0; i < ARRAY_SIZE(h->assert_var); i++) { 461 err = devlink_fmsg_u32_put(fmsg, ioread32be(h->assert_var + i)); 462 if (err) 463 return err; 464 } 465 err = devlink_fmsg_arr_pair_nest_end(fmsg); 466 if (err) 467 return err; 468 err = devlink_fmsg_u32_pair_put(fmsg, "assert_exit_ptr", 469 ioread32be(&h->assert_exit_ptr)); 470 if (err) 471 return err; 472 err = devlink_fmsg_u32_pair_put(fmsg, "assert_callra", 473 ioread32be(&h->assert_callra)); 474 if (err) 475 return err; 476 err = devlink_fmsg_u32_pair_put(fmsg, "hw_id", ioread32be(&h->hw_id)); 477 if (err) 478 return err; 479 err = devlink_fmsg_u8_pair_put(fmsg, "irisc_index", 480 ioread8(&h->irisc_index)); 481 if (err) 482 return err; 483 err = devlink_fmsg_u8_pair_put(fmsg, "synd", ioread8(&h->synd)); 484 if (err) 485 return err; 486 err = devlink_fmsg_u32_pair_put(fmsg, "ext_synd", 487 ioread16be(&h->ext_synd)); 488 if (err) 489 return err; 490 err = devlink_fmsg_u32_pair_put(fmsg, "raw_fw_ver", 491 ioread32be(&h->fw_ver)); 492 if (err) 493 return err; 494 err = devlink_fmsg_obj_nest_end(fmsg); 495 if (err) 496 return err; 497 return devlink_fmsg_pair_nest_end(fmsg); 498 } 499 500 static int 501 mlx5_fw_reporter_dump(struct devlink_health_reporter *reporter, 502 struct devlink_fmsg *fmsg, void *priv_ctx, 503 struct netlink_ext_ack *extack) 504 { 505 struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter); 506 int err; 507 508 err = mlx5_fw_tracer_trigger_core_dump_general(dev); 509 if (err) 510 return err; 511 512 if (priv_ctx) { 513 struct mlx5_fw_reporter_ctx *fw_reporter_ctx = priv_ctx; 514 515 err = mlx5_fw_reporter_ctx_pairs_put(fmsg, fw_reporter_ctx); 516 if (err) 517 return err; 518 } 519 520 err = mlx5_fw_reporter_heath_buffer_data_put(dev, fmsg); 521 if (err) 522 return err; 523 return mlx5_fw_tracer_get_saved_traces_objects(dev->tracer, fmsg); 524 } 525 526 static void mlx5_fw_reporter_err_work(struct work_struct *work) 527 { 528 struct mlx5_fw_reporter_ctx fw_reporter_ctx; 529 struct mlx5_core_health *health; 530 531 health = container_of(work, struct mlx5_core_health, report_work); 532 533 if (IS_ERR_OR_NULL(health->fw_reporter)) 534 return; 535 536 fw_reporter_ctx.err_synd = health->synd; 537 fw_reporter_ctx.miss_counter = health->miss_counter; 538 if (fw_reporter_ctx.err_synd) { 539 devlink_health_report(health->fw_reporter, 540 "FW syndrom reported", &fw_reporter_ctx); 541 return; 542 } 543 if (fw_reporter_ctx.miss_counter) 544 devlink_health_report(health->fw_reporter, 545 "FW miss counter reported", 546 &fw_reporter_ctx); 547 } 548 549 static const struct devlink_health_reporter_ops mlx5_fw_reporter_ops = { 550 .name = "fw", 551 .diagnose = mlx5_fw_reporter_diagnose, 552 .dump = mlx5_fw_reporter_dump, 553 }; 554 555 static int 556 mlx5_fw_fatal_reporter_recover(struct devlink_health_reporter *reporter, 557 void *priv_ctx, 558 struct netlink_ext_ack *extack) 559 { 560 struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter); 561 562 return mlx5_health_try_recover(dev); 563 } 564 565 static int 566 mlx5_fw_fatal_reporter_dump(struct devlink_health_reporter *reporter, 567 struct devlink_fmsg *fmsg, void *priv_ctx, 568 struct netlink_ext_ack *extack) 569 { 570 struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter); 571 u32 crdump_size = dev->priv.health.crdump_size; 572 u32 *cr_data; 573 int err; 574 575 if (!mlx5_core_is_pf(dev)) 576 return -EPERM; 577 578 cr_data = kvmalloc(crdump_size, GFP_KERNEL); 579 if (!cr_data) 580 return -ENOMEM; 581 err = mlx5_crdump_collect(dev, cr_data); 582 if (err) 583 goto free_data; 584 585 if (priv_ctx) { 586 struct mlx5_fw_reporter_ctx *fw_reporter_ctx = priv_ctx; 587 588 err = mlx5_fw_reporter_ctx_pairs_put(fmsg, fw_reporter_ctx); 589 if (err) 590 goto free_data; 591 } 592 593 err = devlink_fmsg_binary_pair_put(fmsg, "crdump_data", cr_data, crdump_size); 594 595 free_data: 596 kvfree(cr_data); 597 return err; 598 } 599 600 static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work) 601 { 602 struct mlx5_fw_reporter_ctx fw_reporter_ctx; 603 struct mlx5_core_health *health; 604 struct mlx5_core_dev *dev; 605 struct mlx5_priv *priv; 606 607 health = container_of(work, struct mlx5_core_health, fatal_report_work); 608 priv = container_of(health, struct mlx5_priv, health); 609 dev = container_of(priv, struct mlx5_core_dev, priv); 610 611 mlx5_enter_error_state(dev, false); 612 if (IS_ERR_OR_NULL(health->fw_fatal_reporter)) { 613 if (mlx5_health_try_recover(dev)) 614 mlx5_core_err(dev, "health recovery failed\n"); 615 return; 616 } 617 fw_reporter_ctx.err_synd = health->synd; 618 fw_reporter_ctx.miss_counter = health->miss_counter; 619 devlink_health_report(health->fw_fatal_reporter, 620 "FW fatal error reported", &fw_reporter_ctx); 621 } 622 623 static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = { 624 .name = "fw_fatal", 625 .recover = mlx5_fw_fatal_reporter_recover, 626 .dump = mlx5_fw_fatal_reporter_dump, 627 }; 628 629 #define MLX5_REPORTER_FW_GRACEFUL_PERIOD 1200000 630 static void mlx5_fw_reporters_create(struct mlx5_core_dev *dev) 631 { 632 struct mlx5_core_health *health = &dev->priv.health; 633 struct devlink *devlink = priv_to_devlink(dev); 634 635 health->fw_reporter = 636 devlink_health_reporter_create(devlink, &mlx5_fw_reporter_ops, 637 0, dev); 638 if (IS_ERR(health->fw_reporter)) 639 mlx5_core_warn(dev, "Failed to create fw reporter, err = %ld\n", 640 PTR_ERR(health->fw_reporter)); 641 642 health->fw_fatal_reporter = 643 devlink_health_reporter_create(devlink, 644 &mlx5_fw_fatal_reporter_ops, 645 MLX5_REPORTER_FW_GRACEFUL_PERIOD, 646 dev); 647 if (IS_ERR(health->fw_fatal_reporter)) 648 mlx5_core_warn(dev, "Failed to create fw fatal reporter, err = %ld\n", 649 PTR_ERR(health->fw_fatal_reporter)); 650 } 651 652 static void mlx5_fw_reporters_destroy(struct mlx5_core_dev *dev) 653 { 654 struct mlx5_core_health *health = &dev->priv.health; 655 656 if (!IS_ERR_OR_NULL(health->fw_reporter)) 657 devlink_health_reporter_destroy(health->fw_reporter); 658 659 if (!IS_ERR_OR_NULL(health->fw_fatal_reporter)) 660 devlink_health_reporter_destroy(health->fw_fatal_reporter); 661 } 662 663 static unsigned long get_next_poll_jiffies(void) 664 { 665 unsigned long next; 666 667 get_random_bytes(&next, sizeof(next)); 668 next %= HZ; 669 next += jiffies + MLX5_HEALTH_POLL_INTERVAL; 670 671 return next; 672 } 673 674 void mlx5_trigger_health_work(struct mlx5_core_dev *dev) 675 { 676 struct mlx5_core_health *health = &dev->priv.health; 677 unsigned long flags; 678 679 spin_lock_irqsave(&health->wq_lock, flags); 680 if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) 681 queue_work(health->wq, &health->fatal_report_work); 682 else 683 mlx5_core_err(dev, "new health works are not permitted at this stage\n"); 684 spin_unlock_irqrestore(&health->wq_lock, flags); 685 } 686 687 static void poll_health(struct timer_list *t) 688 { 689 struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer); 690 struct mlx5_core_health *health = &dev->priv.health; 691 struct health_buffer __iomem *h = health->health; 692 u32 fatal_error; 693 u8 prev_synd; 694 u32 count; 695 696 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 697 goto out; 698 699 fatal_error = check_fatal_sensors(dev); 700 701 if (fatal_error && !health->fatal_error) { 702 mlx5_core_err(dev, "Fatal error %u detected\n", fatal_error); 703 dev->priv.health.fatal_error = fatal_error; 704 print_health_info(dev); 705 mlx5_trigger_health_work(dev); 706 goto out; 707 } 708 709 count = ioread32be(health->health_counter); 710 if (count == health->prev) 711 ++health->miss_counter; 712 else 713 health->miss_counter = 0; 714 715 health->prev = count; 716 if (health->miss_counter == MAX_MISSES) { 717 mlx5_core_err(dev, "device's health compromised - reached miss count\n"); 718 print_health_info(dev); 719 queue_work(health->wq, &health->report_work); 720 } 721 722 prev_synd = health->synd; 723 health->synd = ioread8(&h->synd); 724 if (health->synd && health->synd != prev_synd) 725 queue_work(health->wq, &health->report_work); 726 727 out: 728 mod_timer(&health->timer, get_next_poll_jiffies()); 729 } 730 731 void mlx5_start_health_poll(struct mlx5_core_dev *dev) 732 { 733 struct mlx5_core_health *health = &dev->priv.health; 734 735 timer_setup(&health->timer, poll_health, 0); 736 health->fatal_error = MLX5_SENSOR_NO_ERR; 737 clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); 738 health->health = &dev->iseg->health; 739 health->health_counter = &dev->iseg->health_counter; 740 741 health->timer.expires = round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL); 742 add_timer(&health->timer); 743 } 744 745 void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health) 746 { 747 struct mlx5_core_health *health = &dev->priv.health; 748 unsigned long flags; 749 750 if (disable_health) { 751 spin_lock_irqsave(&health->wq_lock, flags); 752 set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); 753 spin_unlock_irqrestore(&health->wq_lock, flags); 754 } 755 756 del_timer_sync(&health->timer); 757 } 758 759 void mlx5_drain_health_wq(struct mlx5_core_dev *dev) 760 { 761 struct mlx5_core_health *health = &dev->priv.health; 762 unsigned long flags; 763 764 spin_lock_irqsave(&health->wq_lock, flags); 765 set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); 766 spin_unlock_irqrestore(&health->wq_lock, flags); 767 cancel_work_sync(&health->report_work); 768 cancel_work_sync(&health->fatal_report_work); 769 } 770 771 void mlx5_health_flush(struct mlx5_core_dev *dev) 772 { 773 struct mlx5_core_health *health = &dev->priv.health; 774 775 flush_workqueue(health->wq); 776 } 777 778 void mlx5_health_cleanup(struct mlx5_core_dev *dev) 779 { 780 struct mlx5_core_health *health = &dev->priv.health; 781 782 destroy_workqueue(health->wq); 783 mlx5_fw_reporters_destroy(dev); 784 } 785 786 int mlx5_health_init(struct mlx5_core_dev *dev) 787 { 788 struct mlx5_core_health *health; 789 char *name; 790 791 mlx5_fw_reporters_create(dev); 792 793 health = &dev->priv.health; 794 name = kmalloc(64, GFP_KERNEL); 795 if (!name) 796 goto out_err; 797 798 strcpy(name, "mlx5_health"); 799 strcat(name, dev_name(dev->device)); 800 health->wq = create_singlethread_workqueue(name); 801 kfree(name); 802 if (!health->wq) 803 goto out_err; 804 spin_lock_init(&health->wq_lock); 805 INIT_WORK(&health->fatal_report_work, mlx5_fw_fatal_reporter_err_work); 806 INIT_WORK(&health->report_work, mlx5_fw_reporter_err_work); 807 808 return 0; 809 810 out_err: 811 mlx5_fw_reporters_destroy(dev); 812 return -ENOMEM; 813 } 814