/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2012 Milan Jurik. All rights reserved.
 */

#include <sys/types.h>
#include <sys/socket.h>
#include <sys/list.h>
#include <sys/stropts.h>
#include <sys/siginfo.h>
#include <sys/wait.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <stdlib.h>
#include <stdio.h>
#include <strings.h>
#include <stddef.h>
#include <unistd.h>
#include <libilb.h>
#include <port.h>
#include <time.h>
#include <signal.h>
#include <assert.h>
#include <errno.h>
#include <spawn.h>
#include <fcntl.h>
#include <limits.h>
#include "libilb_impl.h"
#include "ilbd.h"

/* Global list of HC objects */
list_t ilbd_hc_list;

/* Timer queue for all hc related timers. */
static iu_tq_t *ilbd_hc_timer_q;

/* Indicate whether the timer needs to be updated */
static boolean_t hc_timer_restarted;

static void ilbd_hc_probe_timer(iu_tq_t *, void *);
static ilb_status_t ilbd_hc_restart_timer(ilbd_hc_t *, ilbd_hc_srv_t *);
static boolean_t ilbd_run_probe(ilbd_hc_srv_t *);

#define MAX(a, b)       ((a) > (b) ? (a) : (b))

/*
 * Number of arguments passed to a probe.  argc[0] is the path name of
 * the probe.
 */
#define HC_PROBE_ARGC   8

/*
 * Max number of characters to be read from the output of a probe.  It
 * is long enough to read in a 64 bit integer.
 */
#define HC_MAX_PROBE_OUTPUT     24

void
i_ilbd_setup_hc_list(void)
{
        list_create(&ilbd_hc_list, sizeof (ilbd_hc_t),
            offsetof(ilbd_hc_t, ihc_link));
}

/*
 * Given a hc object name, return a pointer to hc object if found.
 */
ilbd_hc_t *
ilbd_get_hc(const char *name)
{
        ilbd_hc_t *hc;

        for (hc = list_head(&ilbd_hc_list); hc != NULL;
            hc = list_next(&ilbd_hc_list, hc)) {
                if (strcasecmp(hc->ihc_name, name) == 0)
                        return (hc);
        }
        return (NULL);
}

/*
 * Generates an audit record for create-healthcheck,
 * delete-healtcheck subcommands.
 */
static void
ilbd_audit_hc_event(const char *audit_hcname,
    const ilb_hc_info_t *audit_hcinfo, ilbd_cmd_t cmd,
    ilb_status_t rc, ucred_t *ucredp)
{
        adt_session_data_t      *ah;
        adt_event_data_t        *event;
        au_event_t      flag;
        int     audit_error;

        if ((ucredp == NULL) && (cmd == ILBD_CREATE_HC))  {
                /*
                 * we came here from the path where ilbd incorporates
                 * the configuration that is listed in SCF:
                 * i_ilbd_read_config->ilbd_walk_hc_pgs->
                 *   ->ilbd_scf_instance_walk_pg->ilbd_create_hc
                 * We skip auditing in that case
                 */
                logdebug("ilbd_audit_hc_event: skipping auditing");
                return;
        }

        if (adt_start_session(&ah, NULL, 0) != 0) {
                logerr("ilbd_audit_hc_event: adt_start_session failed");
                exit(EXIT_FAILURE);
        }
        if (adt_set_from_ucred(ah, ucredp, ADT_NEW) != 0) {
                (void) adt_end_session(ah);
                logerr("ilbd_audit_rule_event: adt_set_from_ucred failed");
                exit(EXIT_FAILURE);
        }
        if (cmd == ILBD_CREATE_HC)
                flag = ADT_ilb_create_healthcheck;
        else if (cmd == ILBD_DESTROY_HC)
                flag = ADT_ilb_delete_healthcheck;

        if ((event = adt_alloc_event(ah, flag)) == NULL) {
                logerr("ilbd_audit_hc_event: adt_alloc_event failed");
                exit(EXIT_FAILURE);
        }
        (void) memset((char *)event, 0, sizeof (adt_event_data_t));

        switch (cmd) {
        case ILBD_CREATE_HC:
                event->adt_ilb_create_healthcheck.auth_used =
                    NET_ILB_CONFIG_AUTH;
                event->adt_ilb_create_healthcheck.hc_test =
                    (char *)audit_hcinfo->hci_test;
                event->adt_ilb_create_healthcheck.hc_name =
                    (char *)audit_hcinfo->hci_name;

                /*
                 * If the value 0 is stored, the default values are
                 * set in the kernel. User land does not know about them
                 * So if the user does not specify them, audit record
                 * will show them as 0
                 */
                event->adt_ilb_create_healthcheck.hc_timeout =
                    audit_hcinfo->hci_timeout;
                event->adt_ilb_create_healthcheck.hc_count =
                    audit_hcinfo->hci_count;
                event->adt_ilb_create_healthcheck.hc_interval =
                    audit_hcinfo->hci_interval;
                break;
        case ILBD_DESTROY_HC:
                event->adt_ilb_delete_healthcheck.auth_used =
                    NET_ILB_CONFIG_AUTH;
                event->adt_ilb_delete_healthcheck.hc_name =
                    (char *)audit_hcname;
                break;
        }

        /* Fill in success/failure */
        if (rc == ILB_STATUS_OK) {
                if (adt_put_event(event, ADT_SUCCESS, ADT_SUCCESS) != 0) {
                        logerr("ilbd_audit_hc_event: adt_put_event failed");
                        exit(EXIT_FAILURE);
                }
        } else {
                audit_error = ilberror2auditerror(rc);
                if (adt_put_event(event, ADT_FAILURE, audit_error) != 0) {
                        logerr("ilbd_audit_hc_event: adt_put_event failed");
                        exit(EXIT_FAILURE);
                }
        }
        adt_free_event(event);
        (void) adt_end_session(ah);
}

/*
 * Given the ilb_hc_info_t passed in (from the libilb), create a hc object
 * in ilbd.  The parameter ev_port is not used, refer to comments of
 * ilbd_create_sg() in ilbd_sg.c
 */
/* ARGSUSED */
ilb_status_t
ilbd_create_hc(const ilb_hc_info_t *hc_info, int ev_port,
    const struct passwd *ps, ucred_t *ucredp)
{
        ilbd_hc_t *hc;
        ilb_status_t ret = ILB_STATUS_OK;

        /*
         * ps == NULL is from the daemon when it starts and load configuration
         * ps != NULL is from client.
         */
        if (ps != NULL) {
                ret = ilbd_check_client_config_auth(ps);
                if (ret != ILB_STATUS_OK) {
                        ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
                            ret, ucredp);
                        return (ret);
                }
        }

        if (hc_info->hci_name[0] == '\0') {
                logdebug("ilbd_create_hc: missing healthcheck info");
                ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
                    ILB_STATUS_ENOHCINFO, ucredp);
                return (ILB_STATUS_ENOHCINFO);
        }

        hc = ilbd_get_hc(hc_info->hci_name);
        if (hc != NULL) {
                logdebug("ilbd_create_hc: healthcheck name %s already"
                    " exists", hc_info->hci_name);
                ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
                    ILB_STATUS_EEXIST, ucredp);
                return (ILB_STATUS_EEXIST);
        }

        /*
         * Sanity check on user supplied probe.  The given path name
         * must be a full path name (starts with '/') and is
         * executable.
         */
        if (strcasecmp(hc_info->hci_test, ILB_HC_STR_TCP) != 0 &&
            strcasecmp(hc_info->hci_test, ILB_HC_STR_UDP) != 0 &&
            strcasecmp(hc_info->hci_test, ILB_HC_STR_PING) != 0 &&
            (hc_info->hci_test[0] != '/' ||
            access(hc_info->hci_test, X_OK) == -1)) {
                if (errno == ENOENT) {
                        logdebug("ilbd_create_hc: user script %s doesn't "
                            "exist", hc_info->hci_test);
                        ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
                            ILB_STATUS_ENOENT, ucredp);
                        return (ILB_STATUS_ENOENT);
                } else {
                        logdebug("ilbd_create_hc: user script %s is "
                            "invalid", hc_info->hci_test);
                        ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
                            ILB_STATUS_EINVAL, ucredp);
                        return (ILB_STATUS_EINVAL);
                }
        }

        /* Create and add the hc object */
        hc = calloc(1, sizeof (ilbd_hc_t));
        if (hc == NULL) {
                ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
                    ILB_STATUS_ENOMEM, ucredp);
                return (ILB_STATUS_ENOMEM);
        }
        (void) memcpy(&hc->ihc_info, hc_info, sizeof (ilb_hc_info_t));
        if (strcasecmp(hc->ihc_test, ILB_HC_STR_TCP) == 0)
                hc->ihc_test_type = ILBD_HC_TCP;
        else if (strcasecmp(hc->ihc_test, ILB_HC_STR_UDP) == 0)
                hc->ihc_test_type = ILBD_HC_UDP;
        else if (strcasecmp(hc->ihc_test, ILB_HC_STR_PING) == 0)
                hc->ihc_test_type = ILBD_HC_PING;
        else
                hc->ihc_test_type = ILBD_HC_USER;
        list_create(&hc->ihc_rules, sizeof (ilbd_hc_rule_t),
            offsetof(ilbd_hc_rule_t, hcr_link));

        /* Update SCF */
        if (ps != NULL) {
                if ((ret = ilbd_create_pg(ILBD_SCF_HC, (void *)hc)) !=
                    ILB_STATUS_OK) {
                        ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
                            ret, ucredp);
                        list_destroy(&hc->ihc_rules);
                        free(hc);
                        return (ret);
                }
        }

        /* Everything is fine, now add it to the global list. */
        list_insert_tail(&ilbd_hc_list, hc);
        ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC, ret, ucredp);
        return (ret);
}

/*
 * Given a name of a hc object, destroy it.
 */
ilb_status_t
ilbd_destroy_hc(const char *hc_name, const struct passwd *ps,
    ucred_t *ucredp)
{
        ilb_status_t ret;
        ilbd_hc_t *hc;

        /*
         * No need to check ps == NULL, daemon won't call any destroy func
         * at start up.
         */
        ret = ilbd_check_client_config_auth(ps);
        if (ret != ILB_STATUS_OK) {
                ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
                    ret, ucredp);
                return (ret);
        }

        hc = ilbd_get_hc(hc_name);
        if (hc == NULL) {
                logdebug("ilbd_destroy_hc: healthcheck %s does not exist",
                    hc_name);
                ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
                    ILB_STATUS_ENOENT, ucredp);
                return (ILB_STATUS_ENOENT);
        }

        /* If hc is in use, cannot delete it */
        if (hc->ihc_rule_cnt > 0) {
                logdebug("ilbd_destroy_hc: healthcheck %s is associated"
                    " with a rule - cannot remove", hc_name);
                ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
                    ILB_STATUS_INUSE, ucredp);
                return (ILB_STATUS_INUSE);
        }

        if ((ret = ilbd_destroy_pg(ILBD_SCF_HC, hc_name)) !=
            ILB_STATUS_OK) {
                logdebug("ilbd_destroy_hc: cannot destroy healthcheck %s "
                    "property group", hc_name);
                ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
                    ret, ucredp);
                return (ret);
        }

        list_remove(&ilbd_hc_list, hc);
        list_destroy(&hc->ihc_rules);
        free(hc);
        ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC, ret, ucredp);
        return (ret);
}

/*
 * Given a hc object name, return its information.  Used by libilb to
 * get hc info.
 */
ilb_status_t
ilbd_get_hc_info(const char *hc_name, uint32_t *rbuf, size_t *rbufsz)
{
        ilbd_hc_t       *hc;
        ilb_hc_info_t   *hc_info;
        ilb_comm_t      *ic = (ilb_comm_t *)rbuf;

        hc = ilbd_get_hc(hc_name);
        if (hc == NULL) {
                logdebug("%s: healthcheck %s does not exist", __func__,
                    hc_name);
                return (ILB_STATUS_ENOENT);
        }
        ilbd_reply_ok(rbuf, rbufsz);
        hc_info = (ilb_hc_info_t *)&ic->ic_data;

        (void) strlcpy(hc_info->hci_name, hc->ihc_name, sizeof (hc->ihc_name));
        (void) strlcpy(hc_info->hci_test, hc->ihc_test, sizeof (hc->ihc_test));
        hc_info->hci_timeout = hc->ihc_timeout;
        hc_info->hci_count = hc->ihc_count;
        hc_info->hci_interval = hc->ihc_interval;
        hc_info->hci_def_ping = hc->ihc_def_ping;

        *rbufsz += sizeof (ilb_hc_info_t);

        return (ILB_STATUS_OK);
}

static void
ilbd_hc_copy_srvs(uint32_t *rbuf, size_t *rbufsz, ilbd_hc_rule_t *hc_rule,
    const char *rulename)
{
        ilbd_hc_srv_t           *tmp_srv;
        ilb_hc_srv_t            *dst_srv;
        ilb_hc_rule_srv_t       *srvs;
        size_t                  tmp_rbufsz;
        int                     i;

        tmp_rbufsz = *rbufsz;
        /* Set up the reply buffer.  rbufsz will be set to the new size. */
        ilbd_reply_ok(rbuf, rbufsz);

        /* Calculate how much space is left for holding server info. */
        *rbufsz += sizeof (ilb_hc_rule_srv_t);
        tmp_rbufsz -= *rbufsz;

        srvs = (ilb_hc_rule_srv_t *)&((ilb_comm_t *)rbuf)->ic_data;

        tmp_srv = list_head(&hc_rule->hcr_servers);
        for (i = 0; tmp_srv != NULL && tmp_rbufsz >= sizeof (*dst_srv); i++) {
                dst_srv = &srvs->rs_srvs[i];

                (void) strlcpy(dst_srv->hcs_rule_name, rulename, ILB_NAMESZ);
                (void) strlcpy(dst_srv->hcs_ID, tmp_srv->shc_sg_srv->sgs_srvID,
                    ILB_NAMESZ);
                (void) strlcpy(dst_srv->hcs_hc_name,
                    tmp_srv->shc_hc->ihc_name, ILB_NAMESZ);
                dst_srv->hcs_IP = tmp_srv->shc_sg_srv->sgs_addr;
                dst_srv->hcs_fail_cnt = tmp_srv->shc_fail_cnt;
                dst_srv->hcs_status = tmp_srv->shc_status;
                dst_srv->hcs_rtt = tmp_srv->shc_rtt;
                dst_srv->hcs_lasttime = tmp_srv->shc_lasttime;
                dst_srv->hcs_nexttime = tmp_srv->shc_nexttime;

                tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv);
                tmp_rbufsz -= sizeof (*dst_srv);
        }
        srvs->rs_num_srvs = i;
        *rbufsz += i * sizeof (*dst_srv);
}

/*
 * Given a rule name, return the hc status of its servers.
 */
ilb_status_t
ilbd_get_hc_srvs(const char *rulename, uint32_t *rbuf, size_t *rbufsz)
{
        ilbd_hc_t       *hc;
        ilbd_hc_rule_t  *hc_rule;

        for (hc = list_head(&ilbd_hc_list); hc != NULL;
            hc = list_next(&ilbd_hc_list, hc)) {
                for (hc_rule = list_head(&hc->ihc_rules); hc_rule != NULL;
                    hc_rule = list_next(&hc->ihc_rules, hc_rule)) {
                        if (strcasecmp(hc_rule->hcr_rule->irl_name,
                            rulename) != 0) {
                                continue;
                        }
                        ilbd_hc_copy_srvs(rbuf, rbufsz, hc_rule, rulename);
                        return (ILB_STATUS_OK);
                }
        }
        return (ILB_STATUS_RULE_NO_HC);
}

/*
 * Initialize the hc timer and associate the notification of timeout to
 * the given event port.
 */
void
ilbd_hc_timer_init(int ev_port, ilbd_timer_event_obj_t *ev_obj)
{
        struct sigevent sigev;
        port_notify_t notify;

        if ((ilbd_hc_timer_q = iu_tq_create()) == NULL) {
                logerr("%s: cannot create hc timer queue", __func__);
                exit(EXIT_FAILURE);
        }
        hc_timer_restarted = B_FALSE;

        ev_obj->ev = ILBD_EVENT_TIMER;
        ev_obj->timerid = -1;

        notify.portnfy_port = ev_port;
        notify.portnfy_user = ev_obj;
        sigev.sigev_notify = SIGEV_PORT;
        sigev.sigev_value.sival_ptr = &notify;
        if (timer_create(CLOCK_REALTIME, &sigev, &ev_obj->timerid) == -1) {
                logerr("%s: cannot create timer", __func__);
                exit(EXIT_FAILURE);
        }
}

/*
 * HC timeout handler.
 */
void
ilbd_hc_timeout(void)
{
        (void) iu_expire_timers(ilbd_hc_timer_q);
        hc_timer_restarted = B_TRUE;
}

/*
 * Set up the timer to fire at the earliest timeout.
 */
void
ilbd_hc_timer_update(ilbd_timer_event_obj_t *ev_obj)
{
        itimerspec_t itimeout;
        int timeout;

        /*
         * There is no change on the timer list, so no need to set up the
         * timer again.
         */
        if (!hc_timer_restarted)
                return;

restart:
        if ((timeout = iu_earliest_timer(ilbd_hc_timer_q)) == INFTIM) {
                hc_timer_restarted = B_FALSE;
                return;
        } else if (timeout == 0) {
                /*
                 * Handle the timeout immediately.  After that (clearing all
                 * the expired timers), check to  see if there are still
                 * timers running.  If yes, start them.
                 */
                (void) iu_expire_timers(ilbd_hc_timer_q);
                goto restart;
        }

        itimeout.it_value.tv_sec = timeout / MILLISEC + 1;
        itimeout.it_value.tv_nsec = 0;
        itimeout.it_interval.tv_sec = 0;
        itimeout.it_interval.tv_nsec = 0;

        /*
         * Failure to set a timeout is "OK" since hopefully there will be
         * other events and timer_settime() will be called again.  So
         * we will only miss some timeouts.  But in the worst case, no event
         * will happen and ilbd will get stuck...
         */
        if (timer_settime(ev_obj->timerid, 0, &itimeout, NULL) == -1)
                logerr("%s: cannot set timer", __func__);
        hc_timer_restarted = B_FALSE;
}

/*
 * Kill the probe process of a server.
 */
static void
ilbd_hc_kill_probe(ilbd_hc_srv_t *srv)
{
        /*
         * First dissociate the fd from the event port.  It should not
         * fail.
         */
        if (port_dissociate(srv->shc_ev_port, PORT_SOURCE_FD,
            srv->shc_child_fd) != 0) {
                logdebug("%s: port_dissociate: %s", __func__, strerror(errno));
        }
        (void) close(srv->shc_child_fd);
        free(srv->shc_ev);
        srv->shc_ev = NULL;

        /* Then kill the probe process. */
        if (kill(srv->shc_child_pid, SIGKILL) != 0) {
                logerr("%s: rule %s server %s: %s", __func__,
                    srv->shc_hc_rule->hcr_rule->irl_name,
                    srv->shc_sg_srv->sgs_srvID, strerror(errno));
        }
        /* Should not fail... */
        if (waitpid(srv->shc_child_pid, NULL, 0) != srv->shc_child_pid) {
                logdebug("%s: waitpid: rule %s server %s", __func__,
                    srv->shc_hc_rule->hcr_rule->irl_name,
                    srv->shc_sg_srv->sgs_srvID);
        }
        srv->shc_child_pid = 0;
}

/*
 * Disable the server, either because the server is dead or because a timer
 * cannot be started for this server.  Note that this only affects the
 * transient configuration, meaning only in memory.  The persistent
 * configuration is not affected.
 */
static void
ilbd_mark_server_disabled(ilbd_hc_srv_t *srv)
{
        srv->shc_status = ILB_HCS_DISABLED;

        /* Disable the server in kernel. */
        if (ilbd_k_Xable_server(&srv->shc_sg_srv->sgs_addr,
            srv->shc_hc_rule->hcr_rule->irl_name,
            stat_declare_srv_dead) != ILB_STATUS_OK) {
                logerr("%s: cannot disable server in kernel: rule %s "
                    "server %s", __func__,
                    srv->shc_hc_rule->hcr_rule->irl_name,
                    srv->shc_sg_srv->sgs_srvID);
        }
}

/*
 * A probe fails, set the state of the server.
 */
static void
ilbd_set_fail_state(ilbd_hc_srv_t *srv)
{
        if (++srv->shc_fail_cnt < srv->shc_hc->ihc_count) {
                /* Probe again */
                ilbd_hc_probe_timer(ilbd_hc_timer_q, srv);
                return;
        }

        logdebug("%s: rule %s server %s fails %u", __func__,
            srv->shc_hc_rule->hcr_rule->irl_name, srv->shc_sg_srv->sgs_srvID,
            srv->shc_fail_cnt);

        /*
         * If this is a ping test, mark the server as
         * unreachable instead of dead.
         */
        if (srv->shc_hc->ihc_test_type == ILBD_HC_PING ||
            srv->shc_state == ilbd_hc_def_pinging) {
                srv->shc_status = ILB_HCS_UNREACH;
        } else {
                srv->shc_status = ILB_HCS_DEAD;
        }

        /* Disable the server in kernel. */
        if (ilbd_k_Xable_server(&srv->shc_sg_srv->sgs_addr,
            srv->shc_hc_rule->hcr_rule->irl_name, stat_declare_srv_dead) !=
            ILB_STATUS_OK) {
                logerr("%s: cannot disable server in kernel: rule %s "
                    "server %s", __func__,
                    srv->shc_hc_rule->hcr_rule->irl_name,
                    srv->shc_sg_srv->sgs_srvID);
        }

        /* Still keep probing in case the server is alive again. */
        if (ilbd_hc_restart_timer(srv->shc_hc, srv) != ILB_STATUS_OK) {
                /* Only thing to do is to disable the server... */
                logerr("%s: cannot restart timer: rule %s server %s", __func__,
                    srv->shc_hc_rule->hcr_rule->irl_name,
                    srv->shc_sg_srv->sgs_srvID);
                srv->shc_status = ILB_HCS_DISABLED;
        }
}

/*
 * A probe process has not returned for the ihc_timeout period, we should
 * kill it.  This function is the handler of this.
 */
/* ARGSUSED */
static void
ilbd_hc_kill_timer(iu_tq_t *tq, void *arg)
{
        ilbd_hc_srv_t *srv = (ilbd_hc_srv_t *)arg;

        ilbd_hc_kill_probe(srv);
        ilbd_set_fail_state(srv);
}

/*
 * Probe timeout handler.  Send out the appropriate probe.
 */
/* ARGSUSED */
static void
ilbd_hc_probe_timer(iu_tq_t *tq, void *arg)
{
        ilbd_hc_srv_t *srv = (ilbd_hc_srv_t *)arg;

        /*
         * If starting the probe fails, just pretend that the timeout has
         * extended.
         */
        if (!ilbd_run_probe(srv)) {
                /*
                 * If we cannot restart the timer, the only thing we can do
                 * is to disable this server.  Hopefully the sys admin will
                 * notice this and enable this server again later.
                 */
                if (ilbd_hc_restart_timer(srv->shc_hc, srv) != ILB_STATUS_OK) {
                        logerr("%s: cannot restart timer: rule %s server %s, "
                            "disabling it", __func__,
                            srv->shc_hc_rule->hcr_rule->irl_name,
                            srv->shc_sg_srv->sgs_srvID);
                        ilbd_mark_server_disabled(srv);
                }
                return;
        }

        /*
         * Similar to above, if kill timer cannot be started, disable the
         * server.
         */
        if ((srv->shc_tid = iu_schedule_timer(ilbd_hc_timer_q,
            srv->shc_hc->ihc_timeout, ilbd_hc_kill_timer, srv)) == -1) {
                logerr("%s: cannot start kill timer: rule %s server %s, "
                    "disabling it", __func__,
                    srv->shc_hc_rule->hcr_rule->irl_name,
                    srv->shc_sg_srv->sgs_srvID);
                ilbd_mark_server_disabled(srv);
        }
        hc_timer_restarted = B_TRUE;
}

/* Restart the periodic timer for a given server. */
static ilb_status_t
ilbd_hc_restart_timer(ilbd_hc_t *hc, ilbd_hc_srv_t *srv)
{
        int timeout;

        /* Don't allow the timeout interval to be less than 1s */
        timeout = MAX((hc->ihc_interval >> 1) + (gethrtime() %
            (hc->ihc_interval + 1)), 1);

        /*
         * If the probe is actually a ping probe, there is no need to
         * do default pinging.  Just skip the step.
         */
        if (hc->ihc_def_ping && hc->ihc_test_type != ILBD_HC_PING)
                srv->shc_state = ilbd_hc_def_pinging;
        else
                srv->shc_state = ilbd_hc_probing;
        srv->shc_tid = iu_schedule_timer(ilbd_hc_timer_q, timeout,
            ilbd_hc_probe_timer, srv);

        if (srv->shc_tid == -1)
                return (ILB_STATUS_TIMER);
        srv->shc_lasttime = time(NULL);
        srv->shc_nexttime = time(NULL) + timeout;

        hc_timer_restarted = B_TRUE;
        return (ILB_STATUS_OK);
}

/* Helper routine to associate a server with its hc object. */
static ilb_status_t
ilbd_hc_srv_add(ilbd_hc_t *hc, ilbd_hc_rule_t *hc_rule,
    const ilb_sg_srv_t *srv, int ev_port)
{
        ilbd_hc_srv_t *new_srv;
        ilb_status_t ret;

        if ((new_srv = calloc(1, sizeof (ilbd_hc_srv_t))) == NULL)
                return (ILB_STATUS_ENOMEM);
        new_srv->shc_hc = hc;
        new_srv->shc_hc_rule = hc_rule;
        new_srv->shc_sg_srv = srv;
        new_srv->shc_ev_port = ev_port;
        new_srv->shc_tid = -1;
        new_srv->shc_nexttime = time(NULL);
        new_srv->shc_lasttime = new_srv->shc_nexttime;

        if ((hc_rule->hcr_rule->irl_flags & ILB_FLAGS_RULE_ENABLED) &&
            ILB_IS_SRV_ENABLED(srv->sgs_flags)) {
                new_srv->shc_status = ILB_HCS_UNINIT;
                ret = ilbd_hc_restart_timer(hc, new_srv);
                if (ret != ILB_STATUS_OK) {
                        free(new_srv);
                        return (ret);
                }
        } else {
                new_srv->shc_status = ILB_HCS_DISABLED;
        }

        list_insert_tail(&hc_rule->hcr_servers, new_srv);
        return (ILB_STATUS_OK);
}

/* Handy macro to cancel a server's timer. */
#define HC_CANCEL_TIMER(srv)                                            \
{                                                                       \
        void *arg;                                                      \
        int ret;                                                        \
        if ((srv)->shc_tid != -1) {                                     \
                ret = iu_cancel_timer(ilbd_hc_timer_q, (srv)->shc_tid, &arg); \
                (srv)->shc_tid = -1;                                    \
                assert(ret == 1);                                       \
                assert(arg == (srv));                                   \
        }                                                               \
        hc_timer_restarted = B_TRUE;                                    \
}

/* Helper routine to dissociate a server from its hc object. */
static ilb_status_t
ilbd_hc_srv_rem(ilbd_hc_rule_t *hc_rule, const ilb_sg_srv_t *srv)
{
        ilbd_hc_srv_t *tmp_srv;

        for (tmp_srv = list_head(&hc_rule->hcr_servers); tmp_srv != NULL;
            tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv)) {
                if (tmp_srv->shc_sg_srv == srv) {
                        list_remove(&hc_rule->hcr_servers, tmp_srv);
                        HC_CANCEL_TIMER(tmp_srv);
                        if (tmp_srv->shc_child_pid != 0)
                                ilbd_hc_kill_probe(tmp_srv);
                        free(tmp_srv);
                        return (ILB_STATUS_OK);
                }
        }
        return (ILB_STATUS_ENOENT);
}

/* Helper routine to dissociate all servers of a rule from its hc object. */
static void
ilbd_hc_srv_rem_all(ilbd_hc_rule_t *hc_rule)
{
        ilbd_hc_srv_t *srv;

        while ((srv = list_remove_head(&hc_rule->hcr_servers)) != NULL) {
                HC_CANCEL_TIMER(srv);
                if (srv->shc_child_pid != 0)
                        ilbd_hc_kill_probe(srv);
                free(srv);
        }
}

/* Associate a rule with its hc object. */
ilb_status_t
ilbd_hc_associate_rule(const ilbd_rule_t *rule, int ev_port)
{
        ilbd_hc_t       *hc;
        ilbd_hc_rule_t  *hc_rule;
        ilb_status_t    ret;
        ilbd_sg_t       *sg;
        ilbd_srv_t      *ilbd_srv;

        /* The rule is assumed to be initialized appropriately. */
        if ((hc = ilbd_get_hc(rule->irl_hcname)) == NULL) {
                logdebug("ilbd_hc_associate_rule: healthcheck %s does not "
                    "exist", rule->irl_hcname);
                return (ILB_STATUS_ENOHCINFO);
        }
        if ((hc->ihc_test_type == ILBD_HC_TCP &&
            rule->irl_proto != IPPROTO_TCP) ||
            (hc->ihc_test_type == ILBD_HC_UDP &&
            rule->irl_proto != IPPROTO_UDP)) {
                return (ILB_STATUS_RULE_HC_MISMATCH);
        }
        if ((hc_rule = calloc(1, sizeof (ilbd_hc_rule_t))) == NULL) {
                logdebug("ilbd_hc_associate_rule: out of memory");
                return (ILB_STATUS_ENOMEM);
        }

        hc_rule->hcr_rule = rule;
        list_create(&hc_rule->hcr_servers, sizeof (ilbd_hc_srv_t),
            offsetof(ilbd_hc_srv_t, shc_srv_link));

        /* Add all the servers. */
        sg = rule->irl_sg;
        for (ilbd_srv = list_head(&sg->isg_srvlist); ilbd_srv != NULL;
            ilbd_srv = list_next(&sg->isg_srvlist, ilbd_srv)) {
                if ((ret = ilbd_hc_srv_add(hc, hc_rule, &ilbd_srv->isv_srv,
                    ev_port)) != ILB_STATUS_OK) {
                        /* Remove all previously added servers */
                        ilbd_hc_srv_rem_all(hc_rule);
                        list_destroy(&hc_rule->hcr_servers);
                        free(hc_rule);
                        return (ret);
                }
        }
        list_insert_tail(&hc->ihc_rules, hc_rule);
        hc->ihc_rule_cnt++;

        return (ILB_STATUS_OK);
}

/* Dissociate a rule from its hc object. */
ilb_status_t
ilbd_hc_dissociate_rule(const ilbd_rule_t *rule)
{
        ilbd_hc_t       *hc;
        ilbd_hc_rule_t  *hc_rule;

        /* The rule is assumed to be initialized appropriately. */
        if ((hc = ilbd_get_hc(rule->irl_hcname)) == NULL) {
                logdebug("ilbd_hc_dissociate_rule: healthcheck %s does not "
                    "exist", rule->irl_hcname);
                return (ILB_STATUS_ENOENT);
        }
        for (hc_rule = list_head(&hc->ihc_rules); hc_rule != NULL;
            hc_rule = list_next(&hc->ihc_rules, hc_rule)) {
                if (hc_rule->hcr_rule == rule)
                        break;
        }
        if (hc_rule == NULL) {
                logdebug("ilbd_hc_dissociate_rule: rule %s is not associated "
                    "with healtcheck %s", rule->irl_hcname, hc->ihc_name);
                return (ILB_STATUS_ENOENT);
        }
        ilbd_hc_srv_rem_all(hc_rule);
        list_remove(&hc->ihc_rules, hc_rule);
        hc->ihc_rule_cnt--;
        list_destroy(&hc_rule->hcr_servers);
        free(hc_rule);
        return (ILB_STATUS_OK);
}

/*
 * Given a hc object name and a rule, check to see if the rule is associated
 * with the hc object.  If it is, the hc object is returned in **hc and the
 * ilbd_hc_rule_t is returned in **hc_rule.
 */
static boolean_t
ilbd_hc_check_rule(const char *hc_name, const ilbd_rule_t *rule,
    ilbd_hc_t **hc, ilbd_hc_rule_t **hc_rule)
{
        ilbd_hc_t       *tmp_hc;
        ilbd_hc_rule_t  *tmp_hc_rule;

        if ((tmp_hc = ilbd_get_hc(hc_name)) == NULL)
                return (B_FALSE);
        for (tmp_hc_rule = list_head(&tmp_hc->ihc_rules); tmp_hc_rule != NULL;
            tmp_hc_rule = list_next(&tmp_hc->ihc_rules, tmp_hc_rule)) {
                if (tmp_hc_rule->hcr_rule == rule) {
                        *hc = tmp_hc;
                        *hc_rule = tmp_hc_rule;
                        return (B_TRUE);
                }
        }
        return (B_FALSE);
}

/* Associate a server with its hc object. */
ilb_status_t
ilbd_hc_add_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv,
    int ev_port)
{
        ilbd_hc_t       *hc;
        ilbd_hc_rule_t  *hc_rule;

        if (!ilbd_hc_check_rule(rule->irl_hcname, rule, &hc, &hc_rule))
                return (ILB_STATUS_ENOENT);
        return (ilbd_hc_srv_add(hc, hc_rule, srv, ev_port));
}

/* Dissociate a server from its hc object. */
ilb_status_t
ilbd_hc_del_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv)
{
        ilbd_hc_t       *hc;
        ilbd_hc_rule_t  *hc_rule;

        if (!ilbd_hc_check_rule(rule->irl_hcname, rule, &hc, &hc_rule))
                return (ILB_STATUS_ENOENT);
        return (ilbd_hc_srv_rem(hc_rule, srv));
}

/* Helper routine to enable/disable a server's hc probe. */
static ilb_status_t
ilbd_hc_toggle_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv,
    boolean_t enable)
{
        ilbd_hc_t       *hc;
        ilbd_hc_rule_t  *hc_rule;
        ilbd_hc_srv_t   *tmp_srv;
        ilb_status_t    ret;

        if (!ilbd_hc_check_rule(rule->irl_hcname, rule, &hc, &hc_rule))
                return (ILB_STATUS_ENOENT);
        for (tmp_srv = list_head(&hc_rule->hcr_servers); tmp_srv != NULL;
            tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv)) {
                if (tmp_srv->shc_sg_srv != srv) {
                        continue;
                }
                if (enable) {
                        if (tmp_srv->shc_status == ILB_HCS_DISABLED) {
                                ret = ilbd_hc_restart_timer(hc, tmp_srv);
                                if (ret != ILB_STATUS_OK) {
                                        logerr("%s: cannot start timers for "
                                            "rule %s server %s", __func__,
                                            rule->irl_name,
                                            tmp_srv->shc_sg_srv->sgs_srvID);
                                        return (ret);
                                }
                                /* Start from fresh... */
                                tmp_srv->shc_status = ILB_HCS_UNINIT;
                                tmp_srv->shc_rtt = 0;
                                tmp_srv->shc_fail_cnt = 0;
                        }
                } else {
                        if (tmp_srv->shc_status != ILB_HCS_DISABLED) {
                                tmp_srv->shc_status = ILB_HCS_DISABLED;
                                HC_CANCEL_TIMER(tmp_srv);
                                if (tmp_srv->shc_child_pid != 0)
                                        ilbd_hc_kill_probe(tmp_srv);
                        }
                }
                return (ILB_STATUS_OK);
        }
        return (ILB_STATUS_ENOENT);
}

ilb_status_t
ilbd_hc_enable_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv)
{
        return (ilbd_hc_toggle_server(rule, srv, B_TRUE));
}

ilb_status_t
ilbd_hc_disable_server(const ilbd_rule_t *rule, const ilb_sg_srv_t *srv)
{
        return (ilbd_hc_toggle_server(rule, srv, B_FALSE));
}

/*
 * Helper routine to enable/disable a rule's hc probe (including all its
 * servers).
 */
static ilb_status_t
ilbd_hc_toggle_rule(const ilbd_rule_t *rule, boolean_t enable)
{
        ilbd_hc_t       *hc;
        ilbd_hc_rule_t  *hc_rule;
        ilbd_hc_srv_t   *tmp_srv;
        int             ret;

        if (!ilbd_hc_check_rule(rule->irl_hcname, rule, &hc, &hc_rule))
                return (ILB_STATUS_ENOENT);

        for (tmp_srv = list_head(&hc_rule->hcr_servers); tmp_srv != NULL;
            tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv)) {
                if (enable) {
                        /*
                         * If the server is disabled in the rule, do not
                         * restart its timer.
                         */
                        if (tmp_srv->shc_status == ILB_HCS_DISABLED &&
                            ILB_IS_SRV_ENABLED(
                            tmp_srv->shc_sg_srv->sgs_flags)) {
                                ret = ilbd_hc_restart_timer(hc, tmp_srv);
                                if (ret != ILB_STATUS_OK) {
                                        logerr("%s: cannot start timers for "
                                            "rule %s server %s", __func__,
                                            rule->irl_name,
                                            tmp_srv->shc_sg_srv->sgs_srvID);
                                        goto rollback;
                                } else {
                                        /* Start from fresh... */
                                        tmp_srv->shc_status = ILB_HCS_UNINIT;
                                        tmp_srv->shc_rtt = 0;
                                        tmp_srv->shc_fail_cnt = 0;
                                }
                        }
                } else {
                        if (tmp_srv->shc_status != ILB_HCS_DISABLED) {
                                HC_CANCEL_TIMER(tmp_srv);
                                tmp_srv->shc_status = ILB_HCS_DISABLED;
                                if (tmp_srv->shc_child_pid != 0)
                                        ilbd_hc_kill_probe(tmp_srv);
                        }
                }
        }
        return (ILB_STATUS_OK);
rollback:
        enable = !enable;
        for (tmp_srv = list_prev(&hc_rule->hcr_servers, tmp_srv);
            tmp_srv != NULL;
            tmp_srv = list_prev(&hc_rule->hcr_servers, tmp_srv)) {
                if (enable) {
                        if (tmp_srv->shc_status == ILB_HCS_DISABLED &&
                            ILB_IS_SRV_ENABLED(
                            tmp_srv->shc_sg_srv->sgs_flags)) {
                                (void) ilbd_hc_restart_timer(hc, tmp_srv);
                                tmp_srv->shc_status = ILB_HCS_UNINIT;
                                tmp_srv->shc_rtt = 0;
                                tmp_srv->shc_fail_cnt = 0;
                        }
                } else {
                        if (tmp_srv->shc_status != ILB_HCS_DISABLED) {
                                HC_CANCEL_TIMER(tmp_srv);
                                tmp_srv->shc_status = ILB_HCS_DISABLED;
                                if (tmp_srv->shc_child_pid != 0)
                                        ilbd_hc_kill_probe(tmp_srv);
                        }
                }
        }
        return (ret);
}

ilb_status_t
ilbd_hc_enable_rule(const ilbd_rule_t *rule)
{
        return (ilbd_hc_toggle_rule(rule, B_TRUE));
}

ilb_status_t
ilbd_hc_disable_rule(const ilbd_rule_t *rule)
{
        return (ilbd_hc_toggle_rule(rule, B_FALSE));
}

static const char *
topo_2_str(ilb_topo_t topo)
{
        switch (topo) {
        case ILB_TOPO_DSR:
                return ("DSR");
        case ILB_TOPO_NAT:
                return ("NAT");
        case ILB_TOPO_HALF_NAT:
                return ("HALF_NAT");
        default:
                /* Should not happen. */
                logerr("%s: unknown topology", __func__);
                break;
        }
        return ("");
}

/*
 * Create the argument list to be passed to a hc probe command.
 * The passed in argv is assumed to have HC_PROBE_ARGC elements.
 */
static boolean_t
create_argv(ilbd_hc_srv_t *srv, char *argv[])
{
        char buf[INET6_ADDRSTRLEN];
        ilbd_rule_t const *rule;
        ilb_sg_srv_t const *sg_srv;
        struct in_addr v4_addr;
        in_port_t port;
        int i;

        rule = srv->shc_hc_rule->hcr_rule;
        sg_srv = srv->shc_sg_srv;

        if (srv->shc_state == ilbd_hc_def_pinging) {
                if ((argv[0] = strdup(ILB_PROBE_PING)) == NULL)
                        return (B_FALSE);
        } else {
                switch (srv->shc_hc->ihc_test_type) {
                case ILBD_HC_USER:
                        if ((argv[0] = strdup(srv->shc_hc->ihc_test)) == NULL)
                                return (B_FALSE);
                        break;
                case ILBD_HC_TCP:
                case ILBD_HC_UDP:
                        if ((argv[0] = strdup(ILB_PROBE_PROTO)) ==
                            NULL) {
                                return (B_FALSE);
                        }
                        break;
                case ILBD_HC_PING:
                        if ((argv[0] = strdup(ILB_PROBE_PING)) == NULL) {
                                return (B_FALSE);
                        }
                        break;
                }
        }

        /*
         * argv[1] is the VIP.
         *
         * Right now, the VIP and the backend server addresses should be
         * in the same IP address family.  Here we don't do that in case
         * this assumption is changed in future.
         */
        if (IN6_IS_ADDR_V4MAPPED(&rule->irl_vip)) {
                IN6_V4MAPPED_TO_INADDR(&rule->irl_vip, &v4_addr);
                if (inet_ntop(AF_INET, &v4_addr, buf, sizeof (buf)) == NULL)
                        goto cleanup;
        } else {
                if (inet_ntop(AF_INET6, &rule->irl_vip, buf,
                    sizeof (buf)) == NULL) {
                        goto cleanup;
                }
        }
        if ((argv[1] = strdup(buf)) == NULL)
                goto cleanup;

        /*
         * argv[2] is the backend server address.
         */
        if (IN6_IS_ADDR_V4MAPPED(&sg_srv->sgs_addr)) {
                IN6_V4MAPPED_TO_INADDR(&sg_srv->sgs_addr, &v4_addr);
                if (inet_ntop(AF_INET, &v4_addr, buf, sizeof (buf)) == NULL)
                        goto cleanup;
        } else {
                if (inet_ntop(AF_INET6, &sg_srv->sgs_addr, buf,
                    sizeof (buf)) == NULL) {
                        goto cleanup;
                }
        }
        if ((argv[2] = strdup(buf)) == NULL)
                goto cleanup;

        /*
         * argv[3] is the transport protocol used in the rule.
         */
        switch (rule->irl_proto) {
        case IPPROTO_TCP:
                argv[3] = strdup("TCP");
                break;
        case IPPROTO_UDP:
                argv[3] = strdup("UDP");
                break;
        default:
                logerr("%s: unknown protocol", __func__);
                goto cleanup;
        }
        if (argv[3] == NULL)
                goto cleanup;

        /*
         * argv[4] is the load balance mode, DSR, NAT, HALF-NAT.
         */
        if ((argv[4] = strdup(topo_2_str(rule->irl_topo))) == NULL)
                goto cleanup;

        /*
         * argv[5] is the port range.  Right now, there should only be 1 port.
         */
        switch (rule->irl_hcpflag) {
        case ILB_HCI_PROBE_FIX:
                port = ntohs(rule->irl_hcport);
                break;
        case ILB_HCI_PROBE_ANY: {
                in_port_t min, max;

                if (ntohs(sg_srv->sgs_minport) == 0) {
                        min = ntohs(rule->irl_minport);
                        max = ntohs(rule->irl_maxport);
                } else {
                        min = ntohs(sg_srv->sgs_minport);
                        max = ntohs(sg_srv->sgs_maxport);
                }
                if (max > min)
                        port = min + gethrtime() % (max - min + 1);
                else
                        port = min;
                break;
        }
        default:
                logerr("%s: unknown HC flag", __func__);
                goto cleanup;
        }
        (void) sprintf(buf, "%d", port);
        if ((argv[5] = strdup(buf)) == NULL)
                goto cleanup;

        /*
         * argv[6] is the probe timeout.
         */
        (void) sprintf(buf, "%d", srv->shc_hc->ihc_timeout);
        if ((argv[6] = strdup(buf)) == NULL)
                goto cleanup;

        argv[7] = NULL;
        return (B_TRUE);

cleanup:
        for (i = 0; i < HC_PROBE_ARGC; i++) {
                if (argv[i] != NULL)
                        free(argv[i]);
        }
        return (B_FALSE);
}

static void
destroy_argv(char *argv[])
{
        int i;

        for (i = 0; argv[i] != NULL; i++)
                free(argv[i]);
}

/* Spawn a process to run the hc probe on the given server. */
static boolean_t
ilbd_run_probe(ilbd_hc_srv_t *srv)
{
        posix_spawn_file_actions_t      fd_actions;
        boolean_t                       init_fd_actions = B_FALSE;
        posix_spawnattr_t               attr;
        boolean_t                       init_attr = B_FALSE;
        sigset_t                        child_sigset;
        int                             fds[2];
        int                             fdflags;
        pid_t                           pid;
        char                            *child_argv[HC_PROBE_ARGC];
        ilbd_hc_probe_event_t           *probe_ev;
        char                            *probe_name;

        bzero(child_argv, HC_PROBE_ARGC * sizeof (char *));
        if ((probe_ev = calloc(1, sizeof (*probe_ev))) == NULL) {
                logdebug("ilbd_run_probe: calloc");
                return (B_FALSE);
        }

        /* Set up a pipe to get output from probe command. */
        if (pipe(fds) < 0) {
                logdebug("ilbd_run_probe: cannot create pipe");
                free(probe_ev);
                return (B_FALSE);
        }
        /* Set our side of the pipe to be non-blocking */
        if ((fdflags = fcntl(fds[0], F_GETFL, 0)) == -1) {
                logdebug("ilbd_run_probe: fcntl(F_GETFL)");
                goto cleanup;
        }
        if (fcntl(fds[0], F_SETFL, fdflags | O_NONBLOCK) == -1) {
                logdebug("ilbd_run_probe: fcntl(F_SETFL)");
                goto cleanup;
        }

        if (posix_spawn_file_actions_init(&fd_actions) != 0) {
                logdebug("ilbd_run_probe: posix_spawn_file_actions_init");
                goto cleanup;
        }
        init_fd_actions = B_TRUE;
        if (posix_spawnattr_init(&attr) != 0) {
                logdebug("ilbd_run_probe: posix_spawnattr_init");
                goto cleanup;
        }
        init_attr = B_TRUE;
        if (posix_spawn_file_actions_addclose(&fd_actions, fds[0]) != 0) {
                logdebug("ilbd_run_probe: posix_spawn_file_actions_addclose");
                goto cleanup;
        }
        if (posix_spawn_file_actions_adddup2(&fd_actions, fds[1],
            STDOUT_FILENO) != 0) {
                logdebug("ilbd_run_probe: posix_spawn_file_actions_dup2");
                goto cleanup;
        }
        if (posix_spawn_file_actions_addclose(&fd_actions, fds[1]) != 0) {
                logdebug("ilbd_run_probe: posix_spawn_file_actions_addclose");
                goto cleanup;
        }

        /* Reset all signal handling of the child to default. */
        (void) sigfillset(&child_sigset);
        if (posix_spawnattr_setsigdefault(&attr, &child_sigset) != 0) {
                logdebug("ilbd_run_probe: posix_spawnattr_setsigdefault");
                goto cleanup;
        }
        /* Don't want SIGCHLD. */
        if (posix_spawnattr_setflags(&attr, POSIX_SPAWN_NOSIGCHLD_NP|
            POSIX_SPAWN_SETSIGDEF) != 0) {
                logdebug("ilbd_run_probe: posix_spawnattr_setflags");
                goto cleanup;
        }

        if (!create_argv(srv, child_argv)) {
                logdebug("ilbd_run_probe: create_argv");
                goto cleanup;
        }

        /*
         * If we are doing default pinging or not using a user supplied
         * probe, we should execute our standard supplied probe.  The
         * supplied probe command handles all types of probes.  And the
         * type used depends on argv[0], as filled in by create_argv().
         */
        if (srv->shc_state == ilbd_hc_def_pinging ||
            srv->shc_hc->ihc_test_type != ILBD_HC_USER) {
                probe_name = ILB_PROBE_PROTO;
        } else {
                probe_name = srv->shc_hc->ihc_test;
        }
        if (posix_spawn(&pid, probe_name, &fd_actions, &attr, child_argv,
            NULL) != 0) {
                logerr("%s: posix_spawn: %s for server %s: %s", __func__,
                    srv->shc_hc->ihc_test, srv->shc_sg_srv->sgs_srvID,
                    strerror(errno));
                goto cleanup;
        }

        (void) close(fds[1]);
        srv->shc_child_pid = pid;
        srv->shc_child_fd = fds[0];
        srv->shc_ev = probe_ev;

        probe_ev->ihp_ev = ILBD_EVENT_PROBE;
        probe_ev->ihp_srv = srv;
        probe_ev->ihp_pid = pid;
        if (port_associate(srv->shc_ev_port, PORT_SOURCE_FD, fds[0],
            POLLRDNORM, probe_ev) != 0) {
                /*
                 * Need to kill the child.  It will free the srv->shc_ev,
                 * which is probe_ev.  So set probe_ev to NULL.
                 */
                ilbd_hc_kill_probe(srv);
                probe_ev = NULL;
                goto cleanup;
        }

        destroy_argv(child_argv);
        (void) posix_spawn_file_actions_destroy(&fd_actions);
        (void) posix_spawnattr_destroy(&attr);
        return (B_TRUE);

cleanup:
        destroy_argv(child_argv);
        if (init_fd_actions == B_TRUE)
                (void) posix_spawn_file_actions_destroy(&fd_actions);
        if (init_attr == B_TRUE)
                (void) posix_spawnattr_destroy(&attr);
        (void) close(fds[0]);
        (void) close(fds[1]);
        if (probe_ev != NULL)
                free(probe_ev);
        return (B_FALSE);
}

/*
 * Called by ild_hc_probe_return() to re-associate the fd to a child to
 * the event port.
 */
static void
reassociate_port(int ev_port, int fd, ilbd_hc_probe_event_t *ev)
{
        if (port_associate(ev_port, PORT_SOURCE_FD, fd,
            POLLRDNORM, ev) != 0) {
                /*
                 * If we cannot reassociate with the port, the only
                 * thing we can do now is to kill the child and
                 * do a blocking wait here...
                 */
                logdebug("%s: port_associate: %s", __func__, strerror(errno));
                if (kill(ev->ihp_pid, SIGKILL) != 0)
                        logerr("%s: kill: %s", __func__, strerror(errno));
                if (waitpid(ev->ihp_pid, NULL, 0) != ev->ihp_pid)
                        logdebug("%s: waitpid: %s", __func__, strerror(errno));
                free(ev);
        }
}

/*
 * To handle a child probe process hanging up.
 */
static void
ilbd_hc_child_hup(int ev_port, int fd, ilbd_hc_probe_event_t *ev)
{
        ilbd_hc_srv_t *srv;
        pid_t ret_pid;
        int ret;

        srv = ev->ihp_srv;

        if (!ev->ihp_done) {
                /* ilbd does not care about this process anymore ... */
                ev->ihp_done = B_TRUE;
                srv->shc_ev = NULL;
                srv->shc_child_pid = 0;
                HC_CANCEL_TIMER(srv);
                ilbd_set_fail_state(srv);
        }
        ret_pid = waitpid(ev->ihp_pid, &ret, WNOHANG);
        switch (ret_pid) {
        case -1:
                logperror("ilbd_hc_child_hup: waitpid");
                /* FALLTHROUGH */
        case 0:
                /* The child has not completed the exit. Wait again. */
                reassociate_port(ev_port, fd, ev);
                break;
        default:
                /* Right now, we just ignore the exit status. */
                if (WIFEXITED(ret))
                        ret = WEXITSTATUS(ret);
                (void) close(fd);
                free(ev);
        }
}

/*
 * To read the output of a child probe process.
 */
static void
ilbd_hc_child_data(int fd, ilbd_hc_probe_event_t *ev)
{
        ilbd_hc_srv_t *srv;
        char buf[HC_MAX_PROBE_OUTPUT];
        int ret;
        int64_t rtt;

        srv = ev->ihp_srv;

        bzero(buf, HC_MAX_PROBE_OUTPUT);
        ret = read(fd, buf, HC_MAX_PROBE_OUTPUT - 1);
        /* Should not happen since event port should have caught this. */
        assert(ret > 0);

        /*
         * We expect the probe command to print out the RTT only.  But
         * the command may misbehave and print out more than what we intend to
         * read in.  So need to do this check below to "flush" out all the
         * output from the command.
         */
        if (!ev->ihp_done) {
                ev->ihp_done = B_TRUE;
                /* We don't need to know about this event anymore. */
                srv->shc_ev = NULL;
                srv->shc_child_pid = 0;
                HC_CANCEL_TIMER(srv);
        } else {
                return;
        }

        rtt = strtoll(buf, NULL, 10);

        /*
         * -1 means the server is dead or the probe somehow fails.  Treat
         * them both as server is dead.
         */
        if (rtt == -1) {
                ilbd_set_fail_state(srv);
                return;
        } else if (rtt > 0) {
                /* If the returned RTT value is not valid, just ignore it. */
                if (rtt > 0 && rtt <= UINT_MAX) {
                        /* Set rtt to be the simple smoothed average. */
                        if (srv->shc_rtt == 0) {
                                srv->shc_rtt = rtt;
                        } else {
                                srv->shc_rtt = 3 * ((srv)->shc_rtt >> 2) +
                                    (rtt >> 2);
                        }
                }

        }

        switch (srv->shc_state) {
        case ilbd_hc_def_pinging:
                srv->shc_state = ilbd_hc_probing;

                /* Ping is OK, now start the probe. */
                ilbd_hc_probe_timer(ilbd_hc_timer_q, srv);
                break;
        case ilbd_hc_probing:
                srv->shc_fail_cnt = 0;

                /* Server is dead before, re-enable it. */
                if (srv->shc_status == ILB_HCS_UNREACH ||
                    srv->shc_status == ILB_HCS_DEAD) {
                        /*
                         * If enabling the server in kernel fails now,
                         * hopefully when the timer fires again later, the
                         * enabling can be done.
                         */
                        if (ilbd_k_Xable_server(&srv->shc_sg_srv->sgs_addr,
                            srv->shc_hc_rule->hcr_rule->irl_name,
                            stat_declare_srv_alive) != ILB_STATUS_OK) {
                                logerr("%s: cannot enable server in kernel: "
                                    " rule %s server %s", __func__,
                                    srv->shc_hc_rule->hcr_rule->irl_name,
                                    srv->shc_sg_srv->sgs_srvID);
                        } else {
                                srv->shc_status = ILB_HCS_ALIVE;
                        }
                } else {
                        srv->shc_status = ILB_HCS_ALIVE;
                }
                if (ilbd_hc_restart_timer(srv->shc_hc, srv) != ILB_STATUS_OK) {
                        logerr("%s: cannot restart timer: rule %s server %s",
                            __func__, srv->shc_hc_rule->hcr_rule->irl_name,
                            srv->shc_sg_srv->sgs_srvID);
                        ilbd_mark_server_disabled(srv);
                }
                break;
        default:
                logdebug("%s: unknown state", __func__);
                break;
        }
}

/*
 * Handle the return event of a child probe fd.
 */
void
ilbd_hc_probe_return(int ev_port, int fd, int port_events,
    ilbd_hc_probe_event_t *ev)
{
        /*
         * Note that there can be more than one events delivered to us at
         * the same time.  So we need to check them individually.
         */
        if (port_events & POLLRDNORM)
                ilbd_hc_child_data(fd, ev);

        if (port_events & (POLLHUP|POLLERR)) {
                ilbd_hc_child_hup(ev_port, fd, ev);
                return;
        }

        /*
         * Re-associate the fd with the port so that when the child
         * exits, we can reap the status.
         */
        reassociate_port(ev_port, fd, ev);
}