/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * Panic software-diagnosis subsidiary
 *
 * We model a system panic as a defect diagnosis in FMA. When a system
 * panicks, savecore publishes events which we subscribe to here.
 *
 * Our driving events are all raised by savecore, run either from
 * startup of the dumpadm service or interactively at the command line.
 * The following describes the logic for the handling of these events.
 *
 * On reboot after panic we will run savecore as part of the dumpadm
 * service startup; we run savecore even if savecore is otherwise
 * disabled (ie dumpadm -n in effect) - we run savecore -c to check for
 * a valid dump and raise the initial event.
 *
 * If savecore (or savecore -c) observes a valid dump pending on the
 * device, it raises a "dump_pending_on_device" event provided this
 * was not an FMA-initiated panic (for those we will replay ereports
 * from the dump device as usual and make a diagnosis from those; we do
 * not need to open a case for the panic).  We subscribe to the
 * "dump_pending_on_device" event and use that to open a case;  we
 * open a case requesting the same case uuid as the panic dump image
 * has for the OS instance uuid - if that fails because of a duplicate
 * uuid then we have already opened a case for this panic so no need
 * to open another.
 *
 * Included in the "dump_pending_on_device" event is an indication of
 * whether or not dumpadm is enabled.  If not (dumpadm -n in effect)
 * then we do not expect any further events regarding this panic
 * until such time as the admin runs savecore manually (if ever).
 * So in this case we solve the case immediately after open.  If/when
 * subsequent events arrive when savecore is run manually, we will toss
 * them.
 *
 * If dumpadm is enabled then savecore, run from dumpadm service startup,
 * will attempt to process the dump - either to copy it off the dump
 * device (if saving compressed) or to uncompress it off the dump device.
 * If this succeeds savecore raises a "dump_available" event which
 * includes information on the directory it was saved in, the instance
 * number, image uuid, compressed form or not, and whether the dump
 * was complete (as per the dumphdr).  If the savecore fails for
 * some reason then it exits and raises a "savecore_failure" event.
 * These two events are raised even for FMA-initiated panics.
 *
 * We subscribe to both the "dump_available" and "savecore_failed" events,
 * and in the handling thereof we will close the case opened earlier (if
 * this is not an FMA-initiated panic).  On receipt of the initial
 * "dump_available" event we also arm a timer for +10 minutes if
 * dumpadm is enabled - if no "dump_available" or "savecore_failed" arrives
 * in that time we will solve the case on timeout.
 *
 * When the timer fires we check whether the initial event for each panic
 * case was received more than 30 minutes ago; if it was we solve the case
 * with what we have.  If we're still within the waiting period we rearm
 * for a further 10 minutes.  The timer is shared by all cases that we
 * create, which is why the fire interval is shorter than the maximum time
 * we are prepared to wait.
 */

#include <strings.h>
#include <sys/panic.h>
#include <alloca.h>
#include <zone.h>

#include "../../common/sw.h"
#include "panic.h"

#define MAX_STRING_LEN 160

static id_t myid;

static id_t mytimerid;

/*
 * Our serialization structure type.
 */
#define SWDE_PANIC_CASEDATA_VERS        1

typedef struct swde_panic_casedata {
        uint32_t scd_vers;              /* must be first member */
        uint64_t scd_receive_time;      /* when we first knew of this panic */
        size_t scd_nvlbufsz;            /* size of following buffer */
                                        /* packed attr nvlist follows */
} swde_panic_casedata_t;

static struct {
        fmd_stat_t swde_panic_diagnosed;
        fmd_stat_t swde_panic_badclass;
        fmd_stat_t swde_panic_noattr;
        fmd_stat_t swde_panic_unexpected_fm_panic;
        fmd_stat_t swde_panic_badattr;
        fmd_stat_t swde_panic_badfmri;
        fmd_stat_t swde_panic_noinstance;
        fmd_stat_t swde_panic_nouuid;
        fmd_stat_t swde_panic_dupuuid;
        fmd_stat_t swde_panic_nocase;
        fmd_stat_t swde_panic_notime;
        fmd_stat_t swde_panic_nopanicstr;
        fmd_stat_t swde_panic_nodumpdir;
        fmd_stat_t swde_panic_nostack;
        fmd_stat_t swde_panic_incomplete;
        fmd_stat_t swde_panic_failed;
        fmd_stat_t swde_panic_basecasedata;
        fmd_stat_t swde_panic_failsrlz;
} swde_panic_stats = {
        { "swde_panic_diagnosed", FMD_TYPE_UINT64,
            "panic defects published" },
        { "swde_panic_badclass", FMD_TYPE_UINT64,
            "incorrect event class received" },
        { "swde_panic_noattr", FMD_TYPE_UINT64,
            "malformed event - missing attr nvlist" },
        { "swde_panic_unexpected_fm_panic", FMD_TYPE_UINT64,
            "dump available for an fm_panic()" },
        { "swde_panic_badattr", FMD_TYPE_UINT64,
            "malformed event - invalid attr list" },
        { "swde_panic_badfmri", FMD_TYPE_UINT64,
            "malformed event - fmri2str fails" },
        { "swde_panic_noinstance", FMD_TYPE_UINT64,
            "malformed event - no instance number" },
        { "swde_panic_nouuid", FMD_TYPE_UINT64,
            "malformed event - missing uuid" },
        { "swde_panic_dupuuid", FMD_TYPE_UINT64,
            "duplicate events received" },
        { "swde_panic_nocase", FMD_TYPE_UINT64,
            "case missing for uuid" },
        { "swde_panic_notime", FMD_TYPE_UINT64,
            "missing crash dump time" },
        { "swde_panic_nopanicstr", FMD_TYPE_UINT64,
            "missing panic string" },
        { "swde_panic_nodumpdir", FMD_TYPE_UINT64,
            "missing crashdump save directory" },
        { "swde_panic_nostack", FMD_TYPE_UINT64,
            "missing panic stack" },
        { "swde_panic_incomplete", FMD_TYPE_UINT64,
            "missing panic incomplete" },
        { "swde_panic_failed", FMD_TYPE_UINT64,
            "missing panic failed" },
        { "swde_panic_badcasedata", FMD_TYPE_UINT64,
            "bad case data during timeout" },
        { "swde_panic_failsrlz", FMD_TYPE_UINT64,
            "failures to serialize case data" },
};

#define BUMPSTAT(stat)          swde_panic_stats.stat.fmds_value.ui64++

static nvlist_t *
panic_sw_fmri(fmd_hdl_t *hdl, char *object)
{
        nvlist_t *fmri;
        nvlist_t *sw_obj;
        int err = 0;

        fmri = fmd_nvl_alloc(hdl, FMD_SLEEP);
        err |= nvlist_add_uint8(fmri, FM_VERSION, FM_SW_SCHEME_VERSION);
        err |= nvlist_add_string(fmri, FM_FMRI_SCHEME, FM_FMRI_SCHEME_SW);

        sw_obj = fmd_nvl_alloc(hdl, FMD_SLEEP);
        err |= nvlist_add_string(sw_obj, FM_FMRI_SW_OBJ_PATH, object);
        err |= nvlist_add_nvlist(fmri, FM_FMRI_SW_OBJ, sw_obj);
        nvlist_free(sw_obj);
        if (!err)
                return (fmri);
        else
                return (0);
}

static const char *dumpfiles[2] = { "unix.%lld", "vmcore.%lld" };
static const char *dumpfiles_comp[2] = { "vmdump.%lld", NULL};

static void
swde_panic_solve(fmd_hdl_t *hdl, fmd_case_t *cp,
    nvlist_t *attr, fmd_event_t *ep, boolean_t savecore_success)
{
        char *dumpdir, *path, *uuid;
        nvlist_t *defect, *rsrc;
        nvpair_t *nvp;
        int i;

        /*
         * Attribute members to include in event-specific defect
         * payload.  Some attributes will not be present for some
         * cases - e.g., if we timed out and solved the case without
         * a "dump_available" report.
         */
        const char *toadd[] = {
                "os-instance-uuid",     /* same as case uuid */
                "panicstr",             /* for initial classification work */
                "panicstack",           /* for initial classification work */
                "crashtime",            /* in epoch time */
                "panic-time",           /* Formatted crash time */
        };

        if (ep != NULL)
                fmd_case_add_ereport(hdl, cp, ep);
        /*
         * As a temporary solution we create and fmri in the sw scheme
         * in panic_sw_fmri. This should become a generic fmri constructor
         *
         * We need to user a resource FMRI which will have a sufficiently
         * unique string representation such that fmd will not see
         * repeated panic diagnoses (all using the same defect class)
         * as duplicates and discard later cases.  We can't actually diagnose
         * the panic to anything specific (e.g., a path to a module and
         * function/line etc therein).  We could pick on a generic
         * representative such as /kernel/genunix but that could lead
         * to misunderstanding.  So we choose a path based on <dumpdir>
         * and the OS instance UUID - "<dumpdir>/.<os-instance-uuid>".
         * There's no file at that path (*) but no matter.  We can't use
         * <dumpdir>/vmdump.N or similar because if savecore is disabled
         * or failed we don't have any file or instance number.
         *
         * (*) Some day it would seem tidier to keep all files to do
         * with a single crash (unix/vmcore/vmdump, analysis output etc)
         * in a distinct directory, and <dumpdir>/.<uuid> seems like a good
         * choice.  For compatability we'd symlink into it.  So that is
         * another reason for this choice - some day it may exist!
         */
        (void) nvlist_lookup_string(attr, "dumpdir", &dumpdir);
        (void) nvlist_lookup_string(attr, "os-instance-uuid", &uuid);
        path = alloca(strlen(dumpdir) + 1 + 1 + 36 + 1);
        /* LINTED: E_SEC_SPRINTF_UNBOUNDED_COPY */
        (void) sprintf(path, "%s/.%s", dumpdir, uuid);
        rsrc = panic_sw_fmri(hdl, path);

        defect = fmd_nvl_create_defect(hdl, SW_SUNOS_PANIC_DEFECT,
            100, rsrc, NULL, rsrc);
        nvlist_free(rsrc);

        (void) nvlist_add_boolean_value(defect, "savecore-succcess",
            savecore_success);

        if (savecore_success) {
                boolean_t compressed;
                int64_t instance;
                const char **pathfmts;
                char buf[2][32];
                int files = 0;
                char *arr[2];
                int i;

                (void) nvlist_lookup_int64(attr, "instance", &instance);
                (void) nvlist_lookup_boolean_value(attr, "compressed",
                    &compressed);

                pathfmts = compressed ? &dumpfiles_comp[0] : &dumpfiles[0];

                for (i = 0; i < 2; i++) {
                        if (pathfmts[i] == NULL) {
                                arr[i] = NULL;
                                continue;
                        }

                        (void) snprintf(buf[i], 32, pathfmts[i], instance);
                        arr[i] = buf[i];
                        files++;
                }

                (void) nvlist_add_string(defect, "dump-dir", dumpdir);
                (void) nvlist_add_string_array(defect, "dump-files", arr,
                    files);
        } else {
                char *rsn;

                if (nvlist_lookup_string(attr, "failure-reason", &rsn) == 0)
                        (void) nvlist_add_string(defect, "failure-reason", rsn);
        }

        /*
         * Not all attributes will necessarily be available - eg if
         * dumpadm was not enabled there'll be no instance and dumpdir.
         */
        for (i = 0; i < sizeof (toadd) / sizeof (toadd[0]); i++) {
                if (nvlist_lookup_nvpair(attr, toadd[i], &nvp) == 0)
                        (void) nvlist_add_nvpair(defect, nvp);
        }

        fmd_case_add_suspect(hdl, cp, defect);
        fmd_case_solve(hdl, cp);

        /*
         * Close the case.  Do no free casedata - framework does that for us
         * on closure callback.
         */
        fmd_case_close(hdl, cp);
        BUMPSTAT(swde_panic_diagnosed);
}

/*ARGSUSED*/
static void
swde_panic_timeout(fmd_hdl_t *hdl, id_t timerid, void *data)
{
        fmd_case_t *cp = swde_case_first(hdl, myid);
        swde_panic_casedata_t *cdp;
        time_t now = time(NULL);
        nvlist_t *attr;
        int remain = 0;
        uint32_t vers;

        while (cp != NULL) {
                cdp = swde_case_data(hdl, cp, &vers);
                if (vers != SWDE_PANIC_CASEDATA_VERS)
                        fmd_hdl_abort(hdl, "case data version confused\n");

                if (now > cdp->scd_receive_time + 30 * 60) {
                        if (nvlist_unpack((char *)cdp + sizeof (*cdp),
                            cdp->scd_nvlbufsz, &attr, 0) == 0) {
                                swde_panic_solve(hdl, cp, attr, NULL, B_FALSE);
                                nvlist_free(attr);
                        } else {
                                BUMPSTAT(swde_panic_basecasedata);
                                fmd_case_close(hdl, cp);
                        }
                } else {
                        remain++;
                }


                cp = swde_case_next(hdl, cp);
        }

        if (remain) {
                mytimerid = sw_timer_install(hdl, myid, NULL, NULL,
                    10ULL * NANOSEC * 60);
        }
}

/*
 * Our verify entry point is called for each of our open cases during
 * module load.  We must return 0 for the case to be closed by our caller,
 * or 1 to keep it (or if we have already closed it during this call).
 */
static int
swde_panic_vrfy(fmd_hdl_t *hdl, fmd_case_t *cp)
{
        swde_panic_casedata_t *cdp;
        time_t now = time(NULL);
        nvlist_t *attr;
        uint32_t vers;

        cdp = swde_case_data(hdl, cp, &vers);

        if (vers != SWDE_PANIC_CASEDATA_VERS)
                return (0);     /* case will be closed */

        if (now > cdp->scd_receive_time + 30 * 60) {
                if (nvlist_unpack((char *)cdp + sizeof (*cdp),
                    cdp->scd_nvlbufsz, &attr, 0) == 0) {
                        swde_panic_solve(hdl, cp, attr, NULL, B_FALSE);
                        nvlist_free(attr);
                        return (1);     /* case already closed */
                } else {
                        return (0);     /* close case */
                }
        }

        if (mytimerid != 0)
                mytimerid = sw_timer_install(hdl, myid,
                    NULL, NULL, 10ULL * NANOSEC * 60);

        return (1);     /* retain case */
}

/*
 * Handler for ireport.os.sunos.panic.dump_pending_on_device.
 *
 * A future RFE should try adding a means of avoiding diagnosing repeated
 * defects on panic loops, which would just add to the mayhem and potentially
 * log lots of calls through ASR.  Panics with similar enough panic
 * strings and/or stacks should not diagnose to new defects with some
 * period of time, for example.
 */

/*ARGSUSED*/
void
swde_panic_detected(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
    const char *class, void *arg)
{
        boolean_t fm_panic, expect_savecore;
        swde_panic_casedata_t *cdp;
        nvlist_t *attr;
        fmd_case_t *cp;
        char *fmribuf;
        char *uuid;
        size_t sz;

        fmd_hdl_debug(hdl, "swde_panic_detected\n");

        if (nvlist_lookup_nvlist(nvl, FM_IREPORT_ATTRIBUTES, &attr) != 0) {
                BUMPSTAT(swde_panic_noattr);
                return;
        }

        if (nvlist_lookup_string(attr, "os-instance-uuid", &uuid) != 0) {
                BUMPSTAT(swde_panic_nouuid);
                return;
        }

        fmd_hdl_debug(hdl, "swde_panic_detected: OS instance %s\n", uuid);

        if (nvlist_lookup_boolean_value(attr, "fm-panic", &fm_panic) != 0 ||
            fm_panic == B_TRUE) {
                BUMPSTAT(swde_panic_unexpected_fm_panic);
                return;
        }

        /*
         * Prepare serialization data to be associated with a new
         * case.  Our serialization data consists of a swde_panic_casedata_t
         * structure followed by a packed nvlist of the attributes of
         * the initial event.
         */
        if (nvlist_size(attr, &sz, NV_ENCODE_NATIVE) != 0) {
                BUMPSTAT(swde_panic_failsrlz);
                return;
        }

        cdp = fmd_hdl_zalloc(hdl, sizeof (*cdp) + sz, FMD_SLEEP);
        fmribuf = (char *)cdp + sizeof (*cdp);
        cdp->scd_vers = SWDE_PANIC_CASEDATA_VERS;
        cdp->scd_receive_time = time(NULL);
        cdp->scd_nvlbufsz = sz;

        /*
         * Open a case with UUID matching the the panicking kernel, add this
         * event to the case.
         */
        if ((cp = swde_case_open(hdl, myid, uuid, SWDE_PANIC_CASEDATA_VERS,
            cdp, sizeof (*cdp) + sz)) == NULL) {
                BUMPSTAT(swde_panic_dupuuid);
                fmd_hdl_debug(hdl, "swde_case_open returned NULL - dup?\n");
                fmd_hdl_free(hdl, cdp, sizeof (*cdp) + sz);
                return;
        }

        fmd_case_setprincipal(hdl, cp, ep);

        if (nvlist_lookup_boolean_value(attr, "will-attempt-savecore",
            &expect_savecore) != 0 || expect_savecore == B_FALSE) {
                fmd_hdl_debug(hdl, "savecore not being attempted - "
                    "solve now\n");
                swde_panic_solve(hdl, cp, attr, ep, B_FALSE);
                return;
        }

        /*
         * We expect to see either a "dump_available" or a "savecore_failed"
         * event before too long.  In case that never shows up, for whatever
         * reason, we want to be able to solve the case anyway.
         */
        fmd_case_add_ereport(hdl, cp, ep);
        (void) nvlist_pack(attr, &fmribuf, &sz, NV_ENCODE_NATIVE, 0);
        swde_case_data_write(hdl, cp);

        if (mytimerid == 0) {
                mytimerid = sw_timer_install(hdl, myid, NULL, ep,
                    10ULL * NANOSEC * 60);
                fmd_hdl_debug(hdl, "armed timer\n");
        } else {
                fmd_hdl_debug(hdl, "timer already armed\n");
        }
}

/*
 * savecore has now run and saved a crash dump to the filesystem. It is
 * either a compressed dump (vmdump.n) or uncompressed {unix.n, vmcore.n}
 * Savecore has raised an ireport to say the dump is there.
 */

/*ARGSUSED*/
void
swde_panic_savecore_done(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
    const char *class, void *arg)
{
        boolean_t savecore_success = (arg != NULL);
        boolean_t fm_panic;
        nvlist_t *attr;
        fmd_case_t *cp;
        char *uuid;

        fmd_hdl_debug(hdl, "savecore_done (%s)\n", savecore_success ?
            "success" : "fail");

        if (nvlist_lookup_nvlist(nvl, FM_IREPORT_ATTRIBUTES, &attr) != 0) {
                BUMPSTAT(swde_panic_noattr);
                return;
        }

        if (nvlist_lookup_boolean_value(attr, "fm-panic", &fm_panic) != 0 ||
            fm_panic == B_TRUE) {
                return;         /* not expected, but just in case */
        }

        if (nvlist_lookup_string(attr, "os-instance-uuid", &uuid) != 0) {
                BUMPSTAT(swde_panic_nouuid);
                return;
        }

        /*
         * Find the case related to the panicking kernel; our cases have
         * the same uuid as the crashed OS image.
         */
        cp = fmd_case_uulookup(hdl, uuid);
        if (!cp) {
                /* Unable to find the case. */
                fmd_hdl_debug(hdl, "savecore_done: can't find case for "
                    "image %s\n", uuid);
                BUMPSTAT(swde_panic_nocase);
                return;
        }

        fmd_hdl_debug(hdl, "savecore_done: solving case %s\n", uuid);
        swde_panic_solve(hdl, cp, attr, ep, savecore_success);
}

const struct sw_disp swde_panic_disp[] = {
        { SW_SUNOS_PANIC_DETECTED, swde_panic_detected, NULL },
        { SW_SUNOS_PANIC_AVAIL, swde_panic_savecore_done, (void *)1 },
        { SW_SUNOS_PANIC_FAILURE, swde_panic_savecore_done, NULL },
        /*
         * Something has to subscribe to every fault
         * or defect diagnosed in fmd.  We do that here, but throw it away.
         */
        { SW_SUNOS_PANIC_DEFECT, NULL, NULL },
        { NULL, NULL, NULL }
};

/*ARGSUSED*/
int
swde_panic_init(fmd_hdl_t *hdl, id_t id, const struct sw_disp **dpp,
    int *nelemp)
{
        myid = id;

        if (getzoneid() != GLOBAL_ZONEID)
                return (SW_SUB_INIT_FAIL_VOLUNTARY);

        (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC,
            sizeof (swde_panic_stats) / sizeof (fmd_stat_t),
            (fmd_stat_t *)&swde_panic_stats);

        fmd_hdl_subscribe(hdl, SW_SUNOS_PANIC_DETECTED);
        fmd_hdl_subscribe(hdl, SW_SUNOS_PANIC_FAILURE);
        fmd_hdl_subscribe(hdl, SW_SUNOS_PANIC_AVAIL);

        *dpp = &swde_panic_disp[0];
        *nelemp = sizeof (swde_panic_disp) / sizeof (swde_panic_disp[0]);
        return (SW_SUB_INIT_SUCCESS);
}

void
swde_panic_fini(fmd_hdl_t *hdl)
{
        if (mytimerid)
                sw_timer_remove(hdl, myid, mytimerid);
}

const struct sw_subinfo panic_diag_info = {
        "panic diagnosis",              /* swsub_name */
        SW_CASE_PANIC,                  /* swsub_casetype */
        swde_panic_init,                /* swsub_init */
        swde_panic_fini,                /* swsub_fini */
        swde_panic_timeout,             /* swsub_timeout */
        NULL,                           /* swsub_case_close */
        swde_panic_vrfy,                /* swsub_case_vrfy */
};