1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 #include <fm/fmd_api.h> 26 #include <fm/libtopo.h> 27 #include <fm/topo_hc.h> 28 #include <fm/topo_mod.h> 29 #include <fm/topo_method.h> 30 31 #include <sys/fm/protocol.h> 32 #include <sys/systeminfo.h> 33 34 #include <string.h> 35 36 #define ST_EREPORT_CLASS "ereport.sensor.failure" 37 38 typedef struct sensor_fault { 39 struct sensor_fault *sf_next; 40 char *sf_fru; 41 uint32_t sf_num_fails; 42 boolean_t sf_last_faulted; 43 boolean_t sf_faulted; 44 boolean_t sf_unknown; 45 } sensor_fault_t; 46 47 typedef struct sensor_transport { 48 fmd_hdl_t *st_hdl; 49 fmd_xprt_t *st_xprt; 50 hrtime_t st_interval; 51 id_t st_timer; 52 sensor_fault_t *st_faults; 53 boolean_t st_first; 54 /* 55 * The number of consecutive sensor readings indicating failure that 56 * we'll tolerate before sending an ereport. 57 */ 58 uint32_t st_tolerance; 59 } sensor_transport_t; 60 61 typedef struct st_stats { 62 fmd_stat_t st_bad_fmri; 63 fmd_stat_t st_topo_errs; 64 fmd_stat_t st_repairs; 65 } st_stats_t; 66 67 st_stats_t st_stats = { 68 { "bad_fmri", FMD_TYPE_UINT64, "bad or missing resource/FRU FMRI" }, 69 { "topo_errors", FMD_TYPE_UINT64, "errors walking topology" }, 70 { "repairs", FMD_TYPE_UINT64, "auto repairs" } 71 }; 72 73 static int st_check_component_complaints; 74 static int have_complained; 75 76 static int 77 st_check_component(topo_hdl_t *thp, tnode_t *node, void *arg) 78 { 79 sensor_transport_t *stp = arg; 80 fmd_hdl_t *hdl = stp->st_hdl; 81 const char *name = topo_node_name(node); 82 nvlist_t *nvl, *props, *rsrc, *fru; 83 char *fmri; 84 int err, ret; 85 int32_t last_source, source = -1; 86 boolean_t nonrecov, faulted, predictive, source_diff; 87 nvpair_t *nvp; 88 uint64_t ena; 89 nvlist_t *event; 90 sensor_fault_t *sfp, **current; 91 92 if (strcmp(name, FAN) != 0 && strcmp(name, PSU) != 0) 93 return (0); 94 95 if (topo_node_resource(node, &rsrc, NULL) != 0) { 96 st_stats.st_bad_fmri.fmds_value.ui64++; 97 return (0); 98 } 99 100 /* 101 * If the resource isn't present, don't bother invoking the sensor 102 * failure method. It may be that the sensors aren't part of the same 103 * physical FRU and will report failure if the FRU is no longer there. 104 */ 105 if ((ret = topo_fmri_present(thp, rsrc, &err)) < 0) { 106 fmd_hdl_debug(hdl, "topo_fmri_present() failed for %s=%d", 107 name, topo_node_instance(node)); 108 nvlist_free(rsrc); 109 return (0); 110 } 111 112 if (!ret) { 113 fmd_hdl_debug(hdl, "%s=%d is not present, ignoring", 114 name, topo_node_instance(node)); 115 nvlist_free(rsrc); 116 return (0); 117 } 118 119 if (topo_method_invoke(node, TOPO_METH_SENSOR_FAILURE, 120 TOPO_METH_SENSOR_FAILURE_VERSION, NULL, &nvl, &err) != 0) { 121 if (err == ETOPO_METHOD_NOTSUP) { 122 st_check_component_complaints++; 123 if (!have_complained) { 124 fmd_hdl_debug(hdl, "Method %s not supported " 125 "on %s=%d", TOPO_METH_SENSOR_FAILURE, name, 126 topo_node_instance(node)); 127 } 128 nvlist_free(rsrc); 129 return (0); 130 } 131 nvl = NULL; 132 } 133 134 if (topo_node_fru(node, &fru, NULL, NULL) != 0) { 135 st_stats.st_bad_fmri.fmds_value.ui64++; 136 nvlist_free(nvl); 137 nvlist_free(rsrc); 138 return (0); 139 } 140 141 if (topo_fmri_nvl2str(thp, fru, &fmri, &err) != 0) { 142 st_stats.st_bad_fmri.fmds_value.ui64++; 143 nvlist_free(nvl); 144 nvlist_free(fru); 145 nvlist_free(rsrc); 146 return (0); 147 } 148 149 nvlist_free(fru); 150 151 faulted = nonrecov = source_diff = B_FALSE; 152 predictive = B_TRUE; 153 if (nvl != NULL) { 154 nvp = NULL; 155 while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) { 156 if (nvpair_value_nvlist(nvp, &props) != 0) 157 continue; 158 159 faulted = B_TRUE; 160 161 /* 162 * We need some simple rules to handle the case where 163 * there are multiple facility nodes that indicate 164 * a problem with this FRU, but disagree on the values 165 * of nonrecov, predictive or source: 166 * 167 * 1) nonrecov will be set to true if one or more 168 * facility nodes indicates true. Otherwise it will 169 * default to false 170 * 171 * 2) predictive will default to false and remain false 172 * if one or more facility nodes indicate false. 173 * 174 * 3) source will be set to unknown unless all facility 175 * nodes agree on the source 176 */ 177 if (nonrecov == B_FALSE) 178 if (nvlist_lookup_boolean_value(props, 179 "nonrecov", &nonrecov) != 0) 180 nonrecov = B_FALSE; 181 if (predictive == B_TRUE) 182 if (nvlist_lookup_boolean_value(props, 183 "predictive", &predictive) != 0) 184 predictive = B_FALSE; 185 186 last_source = source; 187 if (nvlist_lookup_uint32(props, "source", 188 (uint32_t *)&source) != 0) 189 source = TOPO_SENSOR_ERRSRC_UNKNOWN; 190 if (last_source != -1 && last_source != source) 191 source_diff = B_TRUE; 192 } 193 if (source_diff) 194 source = TOPO_SENSOR_ERRSRC_UNKNOWN; 195 } 196 197 /* 198 * See if we know about this fru. 199 */ 200 for (current = &stp->st_faults; *current != NULL; 201 current = &(*current)->sf_next) { 202 if (topo_fmri_strcmp(thp, fmri, 203 (*current)->sf_fru)) 204 break; 205 } 206 207 sfp = *current; 208 if (sfp == NULL) { 209 /* 210 * We add this FRU to our list under two circumstances: 211 * 212 * 1. This FRU is faulted and needs to be remembered to 213 * avoid duplicate ereports. 214 * 215 * 2. This is the initial pass, and we want to repair the 216 * FRU in case it was repaired while we were offline. 217 */ 218 if (stp->st_first || faulted) { 219 sfp = fmd_hdl_zalloc(hdl, sizeof (sensor_fault_t), 220 FMD_SLEEP); 221 sfp->sf_fru = fmd_hdl_strdup(hdl, fmri, FMD_SLEEP); 222 sfp->sf_next = stp->st_faults; 223 stp->st_faults = sfp; 224 } else { 225 goto out; 226 } 227 } 228 229 if (faulted) 230 sfp->sf_num_fails++; 231 232 if (nvl == NULL) 233 sfp->sf_unknown = B_TRUE; 234 235 if (faulted) { 236 /* 237 * Construct and post the ereport. 238 * 239 * XXFM we only post one ereport per fru. It should be possible 240 * to uniquely identify faulty resources instead and post one 241 * per resource, even if they share the same FRU. 242 */ 243 if (!sfp->sf_last_faulted && 244 (sfp->sf_num_fails > stp->st_tolerance)) { 245 ena = fmd_event_ena_create(hdl); 246 event = fmd_nvl_alloc(hdl, FMD_SLEEP); 247 248 (void) nvlist_add_string(event, "type", name); 249 (void) nvlist_add_boolean_value(event, "nonrecov", 250 nonrecov); 251 (void) nvlist_add_boolean_value(event, "predictive", 252 predictive); 253 (void) nvlist_add_uint32(event, "source", 254 (uint32_t)source); 255 (void) nvlist_add_nvlist(event, "details", nvl); 256 (void) nvlist_add_string(event, FM_CLASS, 257 ST_EREPORT_CLASS); 258 (void) nvlist_add_uint8(event, FM_VERSION, 259 FM_EREPORT_VERSION); 260 (void) nvlist_add_uint64(event, FM_EREPORT_ENA, ena); 261 (void) nvlist_add_nvlist(event, FM_EREPORT_DETECTOR, 262 rsrc); 263 264 fmd_xprt_post(hdl, stp->st_xprt, event, 0); 265 fmd_hdl_debug(hdl, "posted ereport: %s", 266 ST_EREPORT_CLASS); 267 } 268 269 sfp->sf_faulted = B_TRUE; 270 } 271 272 out: 273 topo_hdl_strfree(thp, fmri); 274 nvlist_free(rsrc); 275 nvlist_free(nvl); 276 return (0); 277 } 278 279 int st_timeout_verbose = 0; 280 281 /*ARGSUSED*/ 282 static void 283 st_timeout(fmd_hdl_t *hdl, id_t id, void *data) 284 { 285 sensor_transport_t *stp; 286 sensor_fault_t *sfp, **current; 287 topo_hdl_t *thp; 288 topo_walk_t *twp; 289 int err; 290 291 if (st_timeout_verbose) 292 fmd_hdl_debug(hdl, "timeout: checking topology"); 293 294 stp = fmd_hdl_getspecific(hdl); 295 thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION); 296 297 if ((twp = topo_walk_init(thp, FM_FMRI_SCHEME_HC, st_check_component, 298 stp, &err)) == NULL) { 299 fmd_hdl_topo_rele(hdl, thp); 300 fmd_hdl_error(hdl, "failed to walk topology: %s\n", 301 topo_strerror(err)); 302 st_stats.st_topo_errs.fmds_value.ui64++; 303 return; 304 } 305 306 if (st_check_component_complaints) 307 have_complained++; 308 309 /* 310 * Initialize values in our internal FRU list for this iteration of 311 * sensor reads. Keep track of whether the FRU was faulted in the 312 * previous pass so we don't send multiple ereports for the same 313 * problem. 314 */ 315 for (sfp = stp->st_faults; sfp != NULL; sfp = sfp->sf_next) { 316 sfp->sf_unknown = B_FALSE; 317 if (sfp->sf_num_fails > stp->st_tolerance) 318 sfp->sf_last_faulted = sfp->sf_faulted; 319 sfp->sf_faulted = B_FALSE; 320 } 321 322 if (topo_walk_step(twp, TOPO_WALK_CHILD) == TOPO_WALK_ERR) { 323 topo_walk_fini(twp); 324 fmd_hdl_topo_rele(hdl, thp); 325 fmd_hdl_error(hdl, "failed to walk topology\n"); 326 st_stats.st_topo_errs.fmds_value.ui64++; 327 return; 328 } 329 330 /* 331 * Remove any faults that weren't seen in the last pass. 332 */ 333 for (current = &stp->st_faults; *current != NULL; ) { 334 sfp = *current; 335 if (!sfp->sf_faulted && !sfp->sf_unknown) { 336 fmd_hdl_debug(hdl, "repairing %s", sfp->sf_fru); 337 fmd_repair_fru(hdl, sfp->sf_fru); 338 st_stats.st_repairs.fmds_value.ui64++; 339 *current = sfp->sf_next; 340 fmd_hdl_strfree(hdl, sfp->sf_fru); 341 fmd_hdl_free(hdl, sfp, sizeof (sensor_fault_t)); 342 } else { 343 current = &sfp->sf_next; 344 } 345 } 346 347 stp->st_first = B_FALSE; 348 topo_walk_fini(twp); 349 fmd_hdl_topo_rele(hdl, thp); 350 351 stp->st_timer = fmd_timer_install(hdl, NULL, NULL, stp->st_interval); 352 } 353 354 static const fmd_prop_t fmd_props[] = { 355 { "interval", FMD_TYPE_TIME, "1min" }, 356 { "tolerance", FMD_TYPE_UINT32, "1" }, 357 { NULL, 0, NULL } 358 }; 359 360 static const fmd_hdl_ops_t fmd_ops = { 361 NULL, /* fmdo_recv */ 362 st_timeout, /* fmdo_timeout */ 363 NULL, /* fmdo_close */ 364 NULL, /* fmdo_stats */ 365 NULL, /* fmdo_gc */ 366 NULL, /* fmdo_send */ 367 NULL /* fmdo_topo */ 368 }; 369 370 static const fmd_hdl_info_t fmd_info = { 371 "Sensor Transport Agent", "1.1", &fmd_ops, fmd_props 372 }; 373 374 void 375 _fmd_init(fmd_hdl_t *hdl) 376 { 377 sensor_transport_t *stp; 378 char buf[SYS_NMLN]; 379 380 /* 381 * The sensor-transport module is currently only supported on x86 382 * platforms. So to avoid unnecessarily wasting cpu cycles on sparc 383 * walking the hc scheme tree every 60 seconds, we'll bail out before 384 * registering the handle. 385 */ 386 if ((sysinfo(SI_ARCHITECTURE, buf, sizeof (buf)) == -1) || 387 (strcmp(buf, "i386") != 0)) 388 return; 389 390 if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) 391 return; 392 393 (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, 394 sizeof (st_stats) / sizeof (fmd_stat_t), 395 (fmd_stat_t *)&st_stats); 396 397 stp = fmd_hdl_zalloc(hdl, sizeof (sensor_transport_t), FMD_SLEEP); 398 stp->st_interval = fmd_prop_get_int64(hdl, "interval"); 399 stp->st_tolerance = fmd_prop_get_int32(hdl, "tolerance"); 400 401 fmd_hdl_setspecific(hdl, stp); 402 403 stp->st_xprt = fmd_xprt_open(hdl, FMD_XPRT_RDONLY, NULL, NULL); 404 stp->st_hdl = hdl; 405 stp->st_first = B_TRUE; 406 407 /* kick off the first asynchronous discovery */ 408 stp->st_timer = fmd_timer_install(hdl, NULL, NULL, 0); 409 } 410 411 void 412 _fmd_fini(fmd_hdl_t *hdl) 413 { 414 sensor_transport_t *stp; 415 sensor_fault_t *sfp; 416 417 stp = fmd_hdl_getspecific(hdl); 418 if (stp != NULL) { 419 fmd_xprt_close(hdl, stp->st_xprt); 420 421 while ((sfp = stp->st_faults) != NULL) { 422 stp->st_faults = sfp->sf_next; 423 424 fmd_hdl_strfree(hdl, sfp->sf_fru); 425 fmd_hdl_free(hdl, sfp, sizeof (sensor_fault_t)); 426 } 427 428 fmd_hdl_free(hdl, stp, sizeof (sensor_transport_t)); 429 } 430 } 431