root/usr/src/cmd/fm/modules/common/sensor-transport/sensor_transport.c
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2018, Joyent, Inc.
 */

#include <fm/fmd_api.h>
#include <fm/libtopo.h>
#include <fm/topo_hc.h>
#include <fm/topo_mod.h>
#include <fm/topo_method.h>

#include <sys/fm/protocol.h>
#include <sys/systeminfo.h>

#include <string.h>

#define ST_EREPORT_CLASS        "ereport.sensor.failure"

typedef struct sensor_fault {
        struct sensor_fault     *sf_next;
        char                    *sf_fru;
        uint32_t                sf_num_fails;
        boolean_t               sf_last_faulted;
        boolean_t               sf_faulted;
        boolean_t               sf_unknown;
} sensor_fault_t;

typedef struct sensor_transport {
        fmd_hdl_t       *st_hdl;
        fmd_xprt_t      *st_xprt;
        hrtime_t        st_interval;
        id_t            st_timer;
        sensor_fault_t  *st_faults;
        boolean_t       st_first;
        /*
         * The number of consecutive sensor readings indicating failure that
         * we'll tolerate before sending an ereport.
         */
        uint32_t        st_tolerance;
        nvlist_t        *st_spoofs;
} sensor_transport_t;

typedef struct st_stats {
        fmd_stat_t st_bad_fmri;
        fmd_stat_t st_topo_errs;
        fmd_stat_t st_repairs;
} st_stats_t;

st_stats_t st_stats = {
        { "bad_fmri", FMD_TYPE_UINT64, "bad or missing resource/FRU FMRI" },
        { "topo_errors", FMD_TYPE_UINT64, "errors walking topology" },
        { "repairs", FMD_TYPE_UINT64, "auto repairs" }
};

static int st_check_component_complaints;
static int have_complained;
static char *spoof_prop = NULL;

static int
st_check_component(topo_hdl_t *thp, tnode_t *node, void *arg)
{
        sensor_transport_t *stp = arg;
        fmd_hdl_t *hdl = stp->st_hdl;
        const char *name = topo_node_name(node);
        nvlist_t *nvl, *props, *rsrc, *fru;
        char *fmri;
        int err, ret;
        int32_t last_source, source = -1;
        boolean_t nonrecov, faulted, predictive, source_diff, injected;
        nvpair_t *nvp;
        uint64_t ena;
        nvlist_t *event;
        sensor_fault_t *sfp, **current;

        if (strcmp(name, FAN) != 0 && strcmp(name, PSU) != 0)
                return (0);

        if (topo_node_resource(node, &rsrc, NULL) != 0) {
                st_stats.st_bad_fmri.fmds_value.ui64++;
                return (0);
        }

        /*
         * If the resource isn't present, don't bother invoking the sensor
         * failure method.  It may be that the sensors aren't part of the same
         * physical FRU and will report failure if the FRU is no longer there.
         */
        if ((ret = topo_fmri_present(thp, rsrc, &err)) < 0) {
                fmd_hdl_debug(hdl, "topo_fmri_present() failed for %s=%d",
                    name, topo_node_instance(node));
                nvlist_free(rsrc);
                return (0);
        }

        if (!ret) {
                fmd_hdl_debug(hdl, "%s=%d is not present, ignoring",
                    name, topo_node_instance(node));
                nvlist_free(rsrc);
                return (0);
        }

        if (topo_method_invoke(node, TOPO_METH_SENSOR_FAILURE,
            TOPO_METH_SENSOR_FAILURE_VERSION, stp->st_spoofs, &nvl, &err) !=
            0) {
                if (err == ETOPO_METHOD_NOTSUP) {
                        st_check_component_complaints++;
                        if (!have_complained) {
                                fmd_hdl_debug(hdl, "Method %s not supported "
                                    "on %s=%d", TOPO_METH_SENSOR_FAILURE, name,
                                    topo_node_instance(node));
                        }
                        nvlist_free(rsrc);
                        return (0);
                }
                nvl = NULL;
        }

        if (topo_node_fru(node, &fru, NULL, &err) != 0) {
                st_stats.st_bad_fmri.fmds_value.ui64++;
                nvlist_free(nvl);
                nvlist_free(rsrc);
                return (0);
        }

        if (topo_fmri_nvl2str(thp, fru, &fmri, &err) != 0) {
                st_stats.st_bad_fmri.fmds_value.ui64++;
                nvlist_free(nvl);
                nvlist_free(fru);
                nvlist_free(rsrc);
                return (0);
        }

        nvlist_free(fru);

        faulted = nonrecov = source_diff = injected = B_FALSE;
        predictive = B_TRUE;
        if (nvl != NULL)  {
                nvp = NULL;
                while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) {
                        if (nvpair_value_nvlist(nvp, &props) != 0)
                                continue;

                        faulted = B_TRUE;

                        /*
                         * We need some simple rules to handle the case where
                         * there are multiple facility nodes that indicate
                         * a problem with this FRU, but disagree on the values
                         * of nonrecov, predictive or source:
                         *
                         * 1) nonrecov will be set to true if one or more
                         *   facility nodes indicates true.  Otherwise it will
                         *   default to false
                         *
                         * 2) predictive will default to false and remain false
                         *    if one or more facility nodes indicate false.
                         *
                         * 3) source will be set to unknown unless all facility
                         *    nodes agree on the source
                         *
                         * 4) injected defaults to false, but will be set to
                         *    true if any of the sensor states were injected.
                         */
                        if (nonrecov == B_FALSE)
                                if (nvlist_lookup_boolean_value(props,
                                    "nonrecov", &nonrecov) != 0)
                                        nonrecov = B_FALSE;
                        if (predictive == B_TRUE)
                                if (nvlist_lookup_boolean_value(props,
                                    "predictive", &predictive) != 0)
                                        predictive = B_FALSE;
                        (void) nvlist_lookup_boolean_value(props,
                            "injected", &injected);

                        last_source = source;
                        if (nvlist_lookup_uint32(props, "source",
                            (uint32_t *)&source) != 0)
                                source = TOPO_SENSOR_ERRSRC_UNKNOWN;
                        if (last_source != -1 && last_source != source)
                                source_diff = B_TRUE;
                }
                if (source_diff)
                        source = TOPO_SENSOR_ERRSRC_UNKNOWN;
        }

        /*
         * See if we know about this fru.
         */
        for (current = &stp->st_faults; *current != NULL;
            current = &(*current)->sf_next) {
                if (topo_fmri_strcmp(thp, fmri,
                    (*current)->sf_fru))
                        break;
        }

        sfp = *current;
        if (sfp == NULL) {
                /*
                 * We add this FRU to our list under two circumstances:
                 *
                 *      1. This FRU is faulted and needs to be remembered to
                 *         avoid duplicate ereports.
                 *
                 *      2. This is the initial pass, and we want to repair the
                 *         FRU in case it was repaired while we were offline.
                 */
                if (stp->st_first || faulted) {
                        sfp = fmd_hdl_zalloc(hdl, sizeof (sensor_fault_t),
                            FMD_SLEEP);
                        sfp->sf_fru = fmd_hdl_strdup(hdl, fmri, FMD_SLEEP);
                        sfp->sf_next = stp->st_faults;
                        stp->st_faults = sfp;
                } else {
                        goto out;
                }
        }

        if (faulted)
                sfp->sf_num_fails++;

        if (nvl == NULL)
                sfp->sf_unknown = B_TRUE;

        if (faulted) {
                /*
                 * Construct and post the ereport.
                 *
                 * XXFM we only post one ereport per fru.  It should be possible
                 * to uniquely identify faulty resources instead and post one
                 * per resource, even if they share the same FRU.
                 */
                if (!sfp->sf_last_faulted &&
                    (sfp->sf_num_fails > stp->st_tolerance)) {
                        ena = fmd_event_ena_create(hdl);
                        event = fmd_nvl_alloc(hdl, FMD_SLEEP);

                        (void) nvlist_add_string(event, "type", name);
                        (void) nvlist_add_boolean_value(event, "nonrecov",
                            nonrecov);
                        (void) nvlist_add_boolean_value(event, "predictive",
                            predictive);
                        (void) nvlist_add_uint32(event, "source",
                            (uint32_t)source);
                        (void) nvlist_add_nvlist(event, "details", nvl);
                        (void) nvlist_add_string(event, FM_CLASS,
                            ST_EREPORT_CLASS);
                        (void) nvlist_add_uint8(event, FM_VERSION,
                            FM_EREPORT_VERSION);
                        (void) nvlist_add_uint64(event, FM_EREPORT_ENA, ena);
                        (void) nvlist_add_nvlist(event, FM_EREPORT_DETECTOR,
                            rsrc);
                        (void) nvlist_add_boolean_value(event, "__injected",
                            injected);
                        fmd_xprt_post(hdl, stp->st_xprt, event, 0);
                        fmd_hdl_debug(hdl, "posted ereport: %s",
                            ST_EREPORT_CLASS);
                }

                sfp->sf_faulted = B_TRUE;
        }

out:
        topo_hdl_strfree(thp, fmri);
        nvlist_free(rsrc);
        nvlist_free(nvl);
        return (0);
}

int st_timeout_verbose = 0;

/*ARGSUSED*/
static void
st_timeout(fmd_hdl_t *hdl, id_t id, void *data)
{
        sensor_transport_t *stp;
        sensor_fault_t *sfp, **current;
        topo_hdl_t *thp;
        topo_walk_t *twp;
        int err;

        if (st_timeout_verbose)
                fmd_hdl_debug(hdl, "timeout: checking topology");

        stp = fmd_hdl_getspecific(hdl);
        thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION);

        if ((twp = topo_walk_init(thp, FM_FMRI_SCHEME_HC, st_check_component,
            stp, &err)) == NULL) {
                fmd_hdl_topo_rele(hdl, thp);
                fmd_hdl_error(hdl, "failed to walk topology: %s\n",
                    topo_strerror(err));
                st_stats.st_topo_errs.fmds_value.ui64++;
                return;
        }

        if (st_check_component_complaints)
                have_complained++;

        /*
         * Initialize values in our internal FRU list for this iteration of
         * sensor reads.  Keep track of whether the FRU was faulted in the
         * previous pass so we don't send multiple ereports for the same
         * problem.
         */
        for (sfp = stp->st_faults; sfp != NULL; sfp = sfp->sf_next) {
                sfp->sf_unknown = B_FALSE;
                if (sfp->sf_num_fails > stp->st_tolerance)
                        sfp->sf_last_faulted = sfp->sf_faulted;
                sfp->sf_faulted = B_FALSE;
        }

        if (topo_walk_step(twp, TOPO_WALK_CHILD) == TOPO_WALK_ERR) {
                topo_walk_fini(twp);
                fmd_hdl_topo_rele(hdl, thp);
                fmd_hdl_error(hdl, "failed to walk topology\n");
                st_stats.st_topo_errs.fmds_value.ui64++;
                return;
        }

        /*
         * Remove any faults that weren't seen in the last pass.
         */
        for (current = &stp->st_faults; *current != NULL; ) {
                sfp = *current;
                if (!sfp->sf_faulted && !sfp->sf_unknown) {
                        fmd_hdl_debug(hdl, "repairing %s", sfp->sf_fru);
                        fmd_repair_fru(hdl, sfp->sf_fru);
                        st_stats.st_repairs.fmds_value.ui64++;
                        *current = sfp->sf_next;
                        fmd_hdl_strfree(hdl, sfp->sf_fru);
                        fmd_hdl_free(hdl, sfp, sizeof (sensor_fault_t));
                } else {
                        current = &sfp->sf_next;
                }
        }

        stp->st_first = B_FALSE;
        topo_walk_fini(twp);
        fmd_hdl_topo_rele(hdl, thp);

        stp->st_timer = fmd_timer_install(hdl, NULL, NULL, stp->st_interval);
}

/*
 * Parse the value of the spoof-sensor-state module property and store the
 * result in an nvlist of nvlists.  The format of the value is 3-tuple,
 * delimited by colons, as follows:
 *
 * FMRIPATTERN:SENSORNAME:SENSORSTATE;...
 *
 * where FMRIPATTERN can be a string with wildcards that matches the FMRI
 * of a node associated with the target sensor facility.
 *
 * where SENSORNAME is the node name of the target sensor facility
 *
 * where SENSORSTATE is the desired sensor state value to spoof.
 *
 * Multiple tuples can be specifed, delimited by semicolons.
 *
 * If any errors are encountered while parsing the value, all parsing is
 * ceased and an ereport will be generated indicating a failure to parse
 * the value.
 */
/*ARGSUSED*/
static int
parse_spoof_param(fmd_hdl_t *hdl, char *param, sensor_transport_t *stp)
{
        char *sensor, *last_sensor, *field, *last_field;
        nvlist_t *spoof;

        if (nvlist_alloc(&stp->st_spoofs, NV_UNIQUE_NAME, 0) != 0) {
                return (-1);
        }

        sensor = strtok_r(param, ";", &last_sensor);
        while (sensor != NULL) {
                if (nvlist_alloc(&spoof, NV_UNIQUE_NAME, 0) != 0)
                        goto err;

                if ((field = strtok_r(sensor, ":", &last_field)) == NULL ||
                    nvlist_add_string(spoof, ST_SPOOF_FMRI, field) != 0)
                        goto err;

                if ((field = strtok_r(NULL, ":", &last_field)) == NULL ||
                    nvlist_add_string(spoof, ST_SPOOF_SENSOR, field) != 0)
                        goto err;

                if ((field = strtok_r(NULL, ":", &last_field)) == NULL ||
                    nvlist_add_uint32(spoof, ST_SPOOF_STATE,
                    strtol(field, NULL, 0)) != 0)
                        goto err;

                if (nvlist_add_nvlist(stp->st_spoofs, sensor, spoof) != 0)
                        goto err;

                spoof = NULL;
                sensor = strtok_r(NULL, ";", &last_sensor);
        }

        return (0);
err:
        nvlist_free(spoof);
        nvlist_free(stp->st_spoofs);
        stp->st_spoofs = NULL;
        return (-1);
}

static const fmd_prop_t fmd_props[] = {
        { "interval", FMD_TYPE_TIME, "1min" },
        { "tolerance", FMD_TYPE_UINT32, "1" },
        { "spoof_sensor_state", FMD_TYPE_STRING, NULL },
        { NULL, 0, NULL }
};

static const fmd_hdl_ops_t fmd_ops = {
        NULL,                   /* fmdo_recv */
        st_timeout,             /* fmdo_timeout */
        NULL,                   /* fmdo_close */
        NULL,                   /* fmdo_stats */
        NULL,                   /* fmdo_gc */
        NULL,                   /* fmdo_send */
        NULL                    /* fmdo_topo */
};

static const fmd_hdl_info_t fmd_info = {
        "Sensor Transport Agent", "1.1", &fmd_ops, fmd_props
};

void
_fmd_init(fmd_hdl_t *hdl)
{
        sensor_transport_t *stp;
        char buf[SYS_NMLN];

        /*
         * The sensor-transport module is currently only supported on x86
         * platforms.  So to avoid unnecessarily wasting cpu cycles on sparc
         * walking the hc scheme tree every 60 seconds, we'll bail out before
         * registering the handle.
         */
        if ((sysinfo(SI_ARCHITECTURE, buf, sizeof (buf)) == -1) ||
            (strcmp(buf, "i386") != 0))
                return;

        if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0)
                return;

        (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC,
            sizeof (st_stats) / sizeof (fmd_stat_t),
            (fmd_stat_t *)&st_stats);

        stp = fmd_hdl_zalloc(hdl, sizeof (sensor_transport_t), FMD_SLEEP);
        stp->st_interval = fmd_prop_get_int64(hdl, "interval");
        stp->st_tolerance = fmd_prop_get_int32(hdl, "tolerance");
        spoof_prop = fmd_prop_get_string(hdl, "spoof_sensor_state");

        if (spoof_prop != NULL && parse_spoof_param(hdl, spoof_prop, stp) != 0)
                fmd_hdl_error(hdl, "Error parsing config file");

        fmd_hdl_setspecific(hdl, stp);

        stp->st_xprt = fmd_xprt_open(hdl, FMD_XPRT_RDONLY, NULL, NULL);
        stp->st_hdl = hdl;
        stp->st_first = B_TRUE;

        /* kick off the first asynchronous discovery */
        stp->st_timer = fmd_timer_install(hdl, NULL, NULL, 0);
}

void
_fmd_fini(fmd_hdl_t *hdl)
{
        sensor_transport_t *stp;
        sensor_fault_t *sfp;

        stp = fmd_hdl_getspecific(hdl);
        if (stp != NULL) {
                fmd_xprt_close(hdl, stp->st_xprt);

                while ((sfp = stp->st_faults) != NULL) {
                        stp->st_faults = sfp->sf_next;

                        fmd_hdl_strfree(hdl, sfp->sf_fru);
                        fmd_hdl_free(hdl, sfp, sizeof (sensor_fault_t));
                }
                nvlist_free(stp->st_spoofs);
                fmd_hdl_free(hdl, stp, sizeof (sensor_transport_t));
        }
        fmd_prop_free_string(hdl, spoof_prop);
}