/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2020 Joyent, Inc.
 * Copyright 2022 Tintri by DDN, Inc. All rights reserved.
 * Copyright 2024 Oxide Computer Company
 */

/*
 * This file drives topo node enumeration of NVMe controllers.  A single "nvme"
 * node is enumerated for each NVMe controller.   Child "disk" nodes are then
 * enumerated for each active or attached NVMe namespace.
 *
 * nvme nodes are expected to be enumerated under either a "bay" node (for U.2
 * devices) or a "slot" node (for M.2 devices) or a "pciexfn" node (for AIC
 * devices).
 *
 * Enumeration of NVMe controllers on PCIe add-in cards is automatically driven
 * by the pcibus topo module.
 *
 * In order to allow for associating a given NVMe controller with a physical
 * location, enumeration of U.2 and M.2 devices should be driven by a
 * platform-specific topo map which statically sets the following two
 * properties on the parent "bay" or "slot" node:
 *
 * propgroup        property        description
 * ---------        --------        ------------
 * binding          driver          "nvme"
 * binding          parent-device   devpath of parent PCIe device
 *
 * for example:
 *
 * <propgroup name="binding" version="1" name-stability="Private"
 *   data-stability="Private">
 *     <propval name="driver" type="string" value="nvme"/>
 *     <propval name="parent-device" type="string"
 *       value="/pci@0,0/pci8086,6f09@3,1"/>
 * </propgroup>
 * <dependents grouping="children">
 *     <range name="nvme" min="0" max="0">
 *         <enum-method name="disk" version="1"/>
 *     </range>
 * </dependents>
 */
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <strings.h>
#include <stdbool.h>

#include <sys/fm/protocol.h>
#include <fm/topo_hc.h>
#include <fm/topo_mod.h>
#include <topo_ufm.h>

#include <sys/dkio.h>
#include <sys/scsi/generic/inquiry.h>

#include <libnvme.h>
#include "disk.h"
#include "disk_drivers.h"

typedef struct nvme_enum_info {
        topo_mod_t              *nei_mod;
        di_node_t               nei_dinode;
        nvme_t                  *nei_libnvme;
        nvme_ctrl_t             *nei_ctrl;
        nvme_ctrl_info_t        *nei_ctrl_info;
        const nvme_version_t    *nei_vers;
        tnode_t                 *nei_parent;
        tnode_t                 *nei_nvme;
        nvlist_t                *nei_nvme_fmri;
        int                     nei_fd;
} nvme_enum_info_t;

typedef struct devlink_arg {
        topo_mod_t              *dla_mod;
        char                    *dla_logical_disk;
        uint_t                  dla_strsz;
} devlink_arg_t;

static int
devlink_cb(di_devlink_t dl, void *arg)
{
        devlink_arg_t *dlarg = (devlink_arg_t *)arg;
        topo_mod_t *mod = dlarg->dla_mod;
        const char *devpath;
        char *slice, *ctds;

        if ((devpath = di_devlink_path(dl)) == NULL ||
            (dlarg->dla_logical_disk = topo_mod_strdup(mod, devpath)) ==
            NULL) {
                return (DI_WALK_TERMINATE);
        }

        /*
         * We need to keep track of the original string size before we
         * truncate it with a NUL, so that we can free the right number of
         * bytes when we're done, otherwise libumem will complain.
         */
        dlarg->dla_strsz = strlen(dlarg->dla_logical_disk) + 1;

        /* trim the slice off the public name */
        if (((ctds = strrchr(dlarg->dla_logical_disk, '/')) != NULL) &&
            ((slice = strchr(ctds, 's')) != NULL))
                *slice = '\0';

        return (DI_WALK_TERMINATE);
}

static char *
get_logical_disk(topo_mod_t *mod, const char *devpath, uint_t *bufsz)
{
        di_devlink_handle_t devhdl;
        devlink_arg_t dlarg = { 0 };
        char *minorpath = NULL;

        if (asprintf(&minorpath, "%s:a", devpath) < 0) {
                return (NULL);
        }

        if ((devhdl = di_devlink_init(NULL, 0)) == DI_NODE_NIL) {
                topo_mod_dprintf(mod, "%s: di_devlink_init failed", __func__);
                free(minorpath);
                return (NULL);
        }

        dlarg.dla_mod = mod;

        (void) di_devlink_walk(devhdl, "^dsk/", minorpath, DI_PRIMARY_LINK,
            &dlarg, devlink_cb);

        (void) di_devlink_fini(&devhdl);
        free(minorpath);

        *bufsz = dlarg.dla_strsz;
        return (dlarg.dla_logical_disk);
}

static bool
disk_nvme_make_ns_serial(topo_mod_t *mod, nvme_ns_info_t *ns_info, char *buf,
    size_t buflen)
{
        uint8_t nguid[16], eui64[8];
        int ret;

        if (nvme_ns_info_nguid(ns_info, nguid)) {
                ret = snprintf(buf, buflen, "%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X"
                    "%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X",
                    nguid[0], nguid[1], nguid[2], nguid[3], nguid[4],
                    nguid[5], nguid[6], nguid[7], nguid[8], nguid[9],
                    nguid[10], nguid[11], nguid[12], nguid[13], nguid[14],
                    nguid[15]);
        } else if (nvme_ns_info_eui64(ns_info, eui64)) {
                ret = snprintf(buf, buflen,
                    "%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X",
                    eui64[0], eui64[1], eui64[2], eui64[3], eui64[4],
                    eui64[5], eui64[6], eui64[7]);
        } else {
                ret = snprintf(buf, buflen, "%u", nvme_ns_info_nsid(ns_info));
        }

        if ((size_t)ret >= buflen) {
                topo_mod_dprintf(mod, "overflowed serial number for nsid %u: "
                    "needed %zu bytes, got %d", nvme_ns_info_nsid(ns_info),
                    buflen, ret);
                return (false);
        }

        return (true);
}

/*
 * Create the common I/O property group properties that are shared between
 * controllers and namespaces. We assume the property group was already created.
 */
static bool
disk_nvme_common_io(topo_mod_t *mod, tnode_t *tn, di_node_t di)
{
        int err;
        int inst = di_instance(di);
        const char *drv = di_driver_name(di);
        char *path;
        const char *ppaths[1];

        if (inst != -1 && topo_prop_set_uint32(tn, TOPO_PGROUP_IO,
            TOPO_IO_INSTANCE, TOPO_PROP_IMMUTABLE, (uint32_t)inst, &err) != 0) {
                topo_mod_dprintf(mod, "failed to set %s:%s on %s[%" PRIu64 "]: "
                    "%s", TOPO_PGROUP_IO, TOPO_IO_INSTANCE, topo_node_name(tn),
                    topo_node_instance(tn), topo_strerror(err));
                return (false);
        }

        if (drv != NULL && topo_prop_set_string(tn, TOPO_PGROUP_IO,
            TOPO_IO_DRIVER, TOPO_PROP_IMMUTABLE, drv, &err) != 0) {
                topo_mod_dprintf(mod, "failed to set %s:%s on %s[%" PRIu64 "]: "
                    "%s", TOPO_PGROUP_IO, TOPO_IO_DRIVER, topo_node_name(tn),
                    topo_node_instance(tn), topo_strerror(err));
                return (false);
        }

        if (drv != NULL) {
                nvlist_t *fmri = topo_mod_modfmri(mod, FM_MOD_SCHEME_VERSION,
                    drv);
                if (mod != NULL && topo_prop_set_fmri(tn, TOPO_PGROUP_IO,
                    TOPO_IO_MODULE, TOPO_PROP_IMMUTABLE, fmri, &err) != 0) {
                        topo_mod_dprintf(mod, "failed to set %s:%s on %s[%"
                            PRIu64 "]: %s", TOPO_PGROUP_IO, TOPO_IO_MODULE,
                            topo_node_name(tn), topo_node_instance(tn),
                            topo_strerror(err));
                        nvlist_free(fmri);
                        return (false);
                }
                nvlist_free(fmri);
        }

        path = di_devfs_path(di);
        ppaths[0] = path;
        if (path != NULL && topo_prop_set_string(tn, TOPO_PGROUP_IO,
            TOPO_IO_DEV_PATH, TOPO_PROP_IMMUTABLE, path, &err) != 0) {
                topo_mod_dprintf(mod, "failed to set %s:%s on %s[%" PRIu64 "]: "
                    "%s", TOPO_PGROUP_IO, TOPO_IO_DRIVER, topo_node_name(tn),
                    topo_node_instance(tn), topo_strerror(err));
                di_devfs_path_free(path);
                return (false);
        }

        if (path != NULL && topo_prop_set_string_array(tn, TOPO_PGROUP_IO,
            TOPO_IO_PHYS_PATH, TOPO_PROP_IMMUTABLE, ppaths, 1, &err) != 0) {
                topo_mod_dprintf(mod, "failed to set %s:%s on %s[%" PRIu64 "]: "
                    "%s", TOPO_PGROUP_IO, TOPO_IO_PHYS_PATH, topo_node_name(tn),
                    topo_node_instance(tn), topo_strerror(err));
                di_devfs_path_free(path);
                return (false);
        }
        di_devfs_path_free(path);

        return (true);
}

/*
 * Add the various storage and I/O property group items that are appropriate
 * given that we have a devinfo node. The storage property group has already
 * been created, but the I/O property group has not.
 */
static void
disk_nvme_make_ns_di_props(topo_mod_t *mod, tnode_t *tn, di_node_t di)
{
        int err;
        char *devid, *mfg, *model, *rev, *serial, *log, *path;
        uint_t buflen;

        if (di_prop_lookup_strings(DDI_DEV_T_ANY, di, DEVID_PROP_NAME,
            &devid) != 1 ||
            di_prop_lookup_strings(DDI_DEV_T_ANY, di, INQUIRY_VENDOR_ID,
            &mfg) != 1 ||
            di_prop_lookup_strings(DDI_DEV_T_ANY, di, INQUIRY_PRODUCT_ID,
            &model) != 1 ||
            di_prop_lookup_strings(DDI_DEV_T_ANY, di, INQUIRY_REVISION_ID,
            &rev) != 1 ||
            di_prop_lookup_strings(DDI_DEV_T_ANY, di, INQUIRY_SERIAL_NO,
            &serial) != 1) {
                topo_mod_dprintf(mod, "failed to get devinfo props for %s[%"
                    PRIu64 "]", topo_node_name(tn), topo_node_instance(tn));
                return;
        }

        /*
         * Set the basic storage manufacturer information. Yes, this is
         * information really about the NVMe controller and not the namespace.
         * That's how the storage property group basically works here.
         */
        if (topo_prop_set_string(tn, TOPO_PGROUP_STORAGE,
            TOPO_STORAGE_MANUFACTURER, TOPO_PROP_IMMUTABLE, mfg, &err) != 0 ||
            topo_prop_set_string(tn, TOPO_PGROUP_STORAGE,
            TOPO_STORAGE_SERIAL_NUM, TOPO_PROP_IMMUTABLE, serial, &err) != 0 ||
            topo_prop_set_string(tn, TOPO_PGROUP_STORAGE,
            TOPO_STORAGE_FIRMWARE_REV, TOPO_PROP_IMMUTABLE, rev, &err) != 0 ||
            topo_prop_set_string(tn, TOPO_PGROUP_STORAGE,
            TOPO_STORAGE_MODEL, TOPO_PROP_IMMUTABLE, model, &err) != 0) {
                topo_mod_dprintf(mod, "failed to set storage properties on "
                    "%s[%" PRIu64 "]: %s", topo_node_name(tn),
                    topo_node_instance(tn), topo_strerror(err));
                return;
        }

        if (topo_pgroup_create(tn, &io_pgroup, &err) != 0) {
                topo_mod_dprintf(mod, "failed to create I/O property "
                    "group on %s[%" PRIu64 "]: %s",  topo_node_name(tn),
                    topo_node_instance(tn), topo_strerror(err));
        }

        if (!disk_nvme_common_io(mod, tn, di)) {
                return;
        }

        /*
         * The last property that we'd like to attempt to create for a namespace
         * is a mapping back to its corresponding logical disk entry in /dev.
         * The logical disk will be everything past the trailing /, i.e. a
         * cXtXdX value.
         */
        path = di_devfs_path(di);
        if (path == NULL) {
                return;
        }
        log = get_logical_disk(mod, path, &buflen);
        di_devfs_path_free(path);
        if (log == NULL) {
                return;
        }
        path = strrchr(log, '/');
        if (path != NULL && path[1] != '\0' &&
            topo_prop_set_string(tn, TOPO_PGROUP_STORAGE,
            TOPO_STORAGE_LOGICAL_DISK_NAME, TOPO_PROP_IMMUTABLE, path + 1,
            &err) != 0) {
                topo_mod_dprintf(mod, "failed to set %s:%s on %s[%"
                    PRIu64 "]: %s", TOPO_PGROUP_STORAGE,
                    TOPO_STORAGE_LOGICAL_DISK_NAME, topo_node_name(tn),
                    topo_node_instance(tn), topo_strerror(err));
        }
        topo_mod_free(mod, log, buflen);
}

static void
disk_nvme_make_ns(nvme_enum_info_t *nei, nvme_ns_info_t *ns_info)
{
        topo_mod_t *mod = nei->nei_mod;
        nvlist_t *auth = NULL, *fmri = NULL;
        const uint32_t nsid = nvme_ns_info_nsid(ns_info);
        const topo_instance_t inst = nsid - 1;
        char serial[64], capstr[64];
        const nvme_nvm_lba_fmt_t *fmt;
        const char *bd_addr;
        uint64_t cap, blksz, capblks;
        tnode_t *tn;
        int err;

        auth = topo_mod_auth(mod, nei->nei_nvme);
        if (auth == NULL) {
                topo_mod_dprintf(mod, "failed to get auth for nsid %u from "
                    "parent %s[%" PRIu64 "]: %s", nsid,
                    topo_node_name(nei->nei_nvme),
                    topo_node_instance(nei->nei_nvme), topo_mod_errmsg(mod));
                goto done;
        }

        /*
         * We want to construct the FMRI for the namespace. The namespace is a
         * little awkward in terms of things like the model, revision, and
         * serial. While blkdev sets up standard inquiry properties to map these
         * to the parent device which makes sense in the context of trying to
         * use this as a normal block device, it's not really appropriate here.
         * The namespace is not the NVMe controller. We construct the namespace
         * serial number from the preferential ordering of information that
         * we're given of the NGUID, EUI64, and then fall back to the namespace
         * number.
         */
        if (!disk_nvme_make_ns_serial(mod, ns_info, serial, sizeof (serial))) {
                goto done;
        }
        fmri = topo_mod_hcfmri(mod, nei->nei_nvme, FM_HC_SCHEME_VERSION,
            DISK, inst, NULL, auth, NULL, NULL, serial);
        if (fmri == NULL) {
                topo_mod_dprintf(mod, "failed to make fmri for %s[%" PRIu64
                    "] on nsid %u: %s", DISK, inst, nsid, topo_mod_errmsg(mod));
                goto done;
        }

        tn = topo_node_bind(mod, nei->nei_nvme, DISK, inst, fmri);
        if (tn == NULL) {
                topo_mod_dprintf(mod, "failed to bind fmri for %s[%" PRIu64
                    "] on nsid %u: %s", DISK, inst, nsid, topo_mod_errmsg(mod));
                goto done;
        }

        /*
         * Always inherit our parent's FRU. The namespace is just a part of the
         * device in reality.
         */
        if (topo_node_fru_set(tn, NULL, 0, &err) != 0) {
                topo_mod_dprintf(mod, "failed to set FRU for %s[%" PRIu64
                    "] on nsid %u: %s", DISK, inst, nsid, topo_strerror(err));
                goto done;

        }

        /*
         * Our namespace may or may not be attached. From the namespace we will
         * always get the capacity and block information. The rest of it will
         * end up being filled in if we find a devinfo node.
         */
        if (topo_pgroup_create(tn, &storage_pgroup, &err) != 0) {
                topo_mod_dprintf(mod, "failed to create storage property "
                    "group on %s[%" PRIu64 "]: %s", DISK, inst,
                    topo_strerror(err));
        }

        if (!nvme_ns_info_curformat(ns_info, &fmt)) {
                topo_mod_dprintf(mod, "failed to get current namespace "
                    "format: %s", nvme_ns_info_errmsg(ns_info));
                goto done;
        }

        blksz = nvme_nvm_lba_fmt_data_size(fmt);
        if (topo_prop_set_uint64(tn, TOPO_PGROUP_STORAGE,
            TOPO_STORAGE_LOG_BLOCK_SIZE, TOPO_PROP_IMMUTABLE, blksz, &err) !=
            0) {
                topo_mod_dprintf(mod, "failed to create property %s:%s on %s[%"
                    PRIu64 "]: %s", TOPO_PGROUP_STORAGE,
                    TOPO_STORAGE_LOG_BLOCK_SIZE, DISK, inst,
                    topo_strerror(err));
                goto done;
        }

        if (!nvme_ns_info_cap(ns_info, &capblks)) {
                topo_mod_dprintf(mod, "failed to get namespace capacity: %s",
                    nvme_ns_info_errmsg(ns_info));
                goto done;
        }

        cap = blksz * capblks;
        if (snprintf(capstr, sizeof (capstr), "%" PRIu64, cap) >=
            sizeof (capstr)) {
                topo_mod_dprintf(mod, "overflowed capacity calculation on "
                    "nsid %u", nsid);
                goto done;
        }

        /*
         * Finally attempt to find a child node that has a matching name and go
         * from there. Sorry, this does result in node creation being O(n^2),
         * but at least n is usually small today. Note, we may not have a blkdev
         * address because the disk may not be attached.
         */
        if (!nvme_ns_info_bd_addr(ns_info, &bd_addr)) {
                if (nvme_ns_info_err(ns_info) != NVME_INFO_ERR_NS_NO_BLKDEV) {
                        topo_mod_dprintf(mod, "failed to get namespace blkdev "
                            "address: %s", nvme_ns_info_errmsg(ns_info));
                }
                goto done;
        }

        for (di_node_t di = di_child_node(nei->nei_dinode); di != DI_NODE_NIL;
            di = di_sibling_node(di)) {
                const char *addr = di_bus_addr(di);
                if (addr != NULL && strcmp(addr, bd_addr) == 0) {
                        disk_nvme_make_ns_di_props(mod, tn, di);
                }
        }

done:
        nvlist_free(auth);
        nvlist_free(fmri);
}

/*
 * Attempt to make a ufm node, but swallow the error so we can try to get as
 * much of the disk information as possible.
 */
static void
disk_nvme_make_ufm(topo_mod_t *mod, nvme_enum_info_t *nei)
{
        topo_ufm_devinfo_t tud;
        char *path = di_devfs_path(nei->nei_dinode);
        if (path == NULL) {
                return;
        }

        tud.tud_method = TOPO_UFM_M_DEVINFO;
        tud.tud_path = path;
        if (topo_mod_load(mod, TOPO_MOD_UFM, TOPO_VERSION) == NULL) {
                topo_mod_dprintf(mod, "disk enum could not load ufm module");
                di_devfs_path_free(path);
                return;
        }

        (void) topo_mod_enumerate(mod, nei->nei_nvme, TOPO_MOD_UFM, UFM, 0, 0,
            &tud);
        di_devfs_path_free(path);
}

static const topo_pgroup_info_t nvme_pgroup = {
        TOPO_PGROUP_NVME,
        TOPO_STABILITY_PRIVATE,
        TOPO_STABILITY_PRIVATE,
        1
};

static int
make_nvme_node(nvme_enum_info_t *nvme_info)
{
        topo_mod_t *mod = nvme_info->nei_mod;
        nvme_ctrl_info_t *info = nvme_info->nei_ctrl_info;
        nvme_ns_iter_t *iter = NULL;
        nvme_iter_t nret;
        const nvme_ns_disc_t *disc;
        nvlist_t *auth = NULL, *fmri = NULL, *fru;
        tnode_t *nvme;
        char *model = NULL, *serial = NULL, *vers = NULL;
        char *pname = topo_node_name(nvme_info->nei_parent);
        char *label = NULL;
        topo_instance_t pinst = topo_node_instance(nvme_info->nei_parent);
        int err = 0, ret = -1;

        /*
         * Pass the model and serial strings through a function that sanitizes
         * them of any characters that can't be used in an FMRI string. Note, we
         * do not use the firmware revision here because that's not really a
         * device property that should be part of the FMRI (it can be changed at
         * runtime).
         */
        model = topo_mod_clean_str(mod, nvme_ctrl_info_model(info));
        serial = topo_mod_clean_str(mod, nvme_ctrl_info_serial(info));

        auth = topo_mod_auth(mod, nvme_info->nei_parent);
        fmri = topo_mod_hcfmri(mod, nvme_info->nei_parent, FM_HC_SCHEME_VERSION,
            NVME, 0, NULL, auth, model, NULL, serial);

        if (fmri == NULL) {
                /* errno set */
                topo_mod_dprintf(mod, "%s: hcfmri failed for %s=%" PRIu64
                    "/%s=0", __func__, pname, pinst, NVME);
                goto error;
        }

        /*
         * If our parent is a pciexfn node, then we need to create a nvme range
         * underneath it to hold the nvme hierarchy.  For other cases, where
         * enumeration is being driven by a topo map file, this range will have
         * already been statically defined in the XML.
         */
        if (strcmp(pname, PCIEX_FUNCTION) == 0) {
                if (topo_node_range_create(mod, nvme_info->nei_parent, NVME, 0,
                    0) < 0) {
                        /* errno set */
                        topo_mod_dprintf(mod, "%s: error creating %s range",
                            __func__, NVME);
                        goto error;
                }
        }

        /*
         * Create a new topo node to represent the NVMe controller and bind it
         * to the parent node.
         */
        if ((nvme = topo_node_bind(mod, nvme_info->nei_parent, NVME, 0,
            fmri)) == NULL) {
                /* errno set */
                topo_mod_dprintf(mod, "%s: bind failed for %s=%" PRIu64
                    "/%s=0", __func__, pname, pinst, NVME);
                goto error;
        }
        nvme_info->nei_nvme = nvme;
        nvme_info->nei_nvme_fmri = fmri;

        /*
         * If our parent node is a "pciexfn" node then this is a NVMe device on
         * a PCIe AIC, so we inherit our parent's FRU.  Otherwise, we set the
         * FRU to ourself.
         */
        if (strcmp(topo_node_name(nvme_info->nei_parent), PCIEX_FUNCTION) == 0)
                fru = NULL;
        else
                fru = fmri;

        if (topo_node_fru_set(nvme, fru, 0, &err) != 0) {
                topo_mod_dprintf(mod, "%s: failed to set FRU: %s", __func__,
                    topo_strerror(err));
                (void) topo_mod_seterrno(mod, err);
                goto error;
        }

        /*
         * Clone the label from our parent node.  We can't inherit the property
         * because the label prop is mutable on bay nodes and only immutable
         * properties can be inherited.
         */
        if ((topo_node_label(nvme_info->nei_parent, &label, &err) != 0 &&
            err != ETOPO_PROP_NOENT) ||
            topo_node_label_set(nvme, label, &err) != 0) {
                topo_mod_dprintf(mod, "%s: failed to set label: %s",
                    __func__, topo_strerror(err));
                (void) topo_mod_seterrno(mod, err);
                goto error;
        }

        /*
         * Ensure that we have a UFM property set based on our devinfo path.
         * This is a little repetitive if our parent actually did so as well,
         * but given that the majority of such nodes are under bays and slots
         * right now, it's a worthwhile tradeoff.
         */
        disk_nvme_make_ufm(mod, nvme_info);

        if (topo_pgroup_create(nvme, &nvme_pgroup, &err) != 0) {
                topo_mod_dprintf(mod, "%s: failed to create %s pgroup: %s",
                    __func__, TOPO_PGROUP_NVME, topo_strerror(err));
                (void) topo_mod_seterrno(mod, err);
                goto error;
        }

        if (asprintf(&vers, "%u.%u", nvme_info->nei_vers->v_major,
            nvme_info->nei_vers->v_minor) < 0) {
                topo_mod_dprintf(mod, "%s: failed to alloc string", __func__);
                (void) topo_mod_seterrno(mod, EMOD_NOMEM);
                goto error;
        }
        if (topo_prop_set_string(nvme, TOPO_PGROUP_NVME, TOPO_PROP_NVME_VER,
            TOPO_PROP_IMMUTABLE, vers, &err) != 0) {
                topo_mod_dprintf(mod, "%s: failed to set %s/%s property",
                    __func__, TOPO_PGROUP_NVME, TOPO_PROP_NVME_VER);
                (void) topo_mod_seterrno(mod, err);
                goto error;
        }

        if (topo_pgroup_create(nvme, &io_pgroup, &err) != 0) {
                topo_mod_dprintf(mod, "%s: failed to create %s pgroup: %s",
                    __func__, TOPO_PGROUP_IO, topo_strerror(err));
                (void) topo_mod_seterrno(mod, err);
                goto error;
        }

        if (!disk_nvme_common_io(mod, nvme, nvme_info->nei_dinode)) {
                goto error;
        }

        /*
         * Create a child disk node for each namespace.
         */
        if (topo_node_range_create(mod, nvme, DISK, 0,
            nvme_ctrl_info_nns(info) - 1) < 0) {
                /* errno set */
                topo_mod_dprintf(mod, "%s: error creating %s range", __func__,
                    DISK);
                goto error;
        }

        /*
         * Iterate over each namespace to see if it's a candidate for inclusion.
         * Namespaces start at index 1 and not every namespace will be included.
         * We map things such that a disk instance is always namespace - 1 to
         * fit into the above mapping.
         */
        if (!nvme_ns_discover_init(nvme_info->nei_ctrl,
            NVME_NS_DISC_F_NOT_IGNORED, &iter)) {
                topo_mod_dprintf(mod, "failed to initialize namespace "
                    "discovery: %s", nvme_errmsg(nvme_info->nei_libnvme));
                ret = topo_mod_seterrno(mod, EMOD_UNKNOWN);
                goto error;
        }

        for (nret = nvme_ns_discover_step(iter, &disc); nret == NVME_ITER_VALID;
            nret = nvme_ns_discover_step(iter, &disc)) {
                nvme_ns_info_t *ns_info;
                uint32_t nsid = nvme_ns_disc_nsid(disc);

                if (!nvme_ctrl_ns_info_snap(nvme_info->nei_ctrl, nsid,
                    &ns_info)) {
                        topo_mod_dprintf(mod, "failed to get namespace "
                            "information for ns %u: %s", nsid,
                            nvme_errmsg(nvme_info->nei_libnvme));
                        ret = topo_mod_seterrno(mod, EMOD_UNKNOWN);
                        goto error;
                }

                disk_nvme_make_ns(nvme_info, ns_info);
                nvme_ns_info_free(ns_info);
        }

        if (nret == NVME_ITER_ERROR) {
                topo_mod_dprintf(mod, "namespace discovery failed: %s",
                    nvme_errmsg(nvme_info->nei_libnvme));
                ret = topo_mod_seterrno(mod, EMOD_UNKNOWN);
        }
        ret = 0;

error:
        nvme_ns_discover_fini(iter);
        free(vers);
        nvlist_free(auth);
        nvlist_free(fmri);
        topo_mod_strfree(mod, model);
        topo_mod_strfree(mod, serial);
        topo_mod_strfree(mod, label);
        return (ret);
}

/*
 * This function gathers identity information from the NVMe controller and
 * stores it in a struct.  This struct is passed to make_nvme_node(), which
 * does the actual topo node creation.
 */
static int
discover_nvme_ctl(topo_mod_t *mod, tnode_t *pnode, di_node_t dinode)
{
        topo_disk_t *disk = topo_mod_getspecific(mod);
        nvme_enum_info_t nvme_info = { 0 };
        int ret;

        nvme_info.nei_mod = mod;
        nvme_info.nei_dinode = dinode;
        nvme_info.nei_parent = pnode;
        nvme_info.nei_libnvme = disk->td_nvme;

        if (!nvme_ctrl_init(disk->td_nvme, dinode, &nvme_info.nei_ctrl)) {
                topo_mod_dprintf(mod, "failed to initialize nvme_ctrl_t: %s",
                    nvme_errmsg(disk->td_nvme));
                return (topo_mod_seterrno(mod, EMOD_UNKNOWN));
        }

        if (!nvme_ctrl_info_snap(nvme_info.nei_ctrl,
            &nvme_info.nei_ctrl_info)) {
                topo_mod_dprintf(mod, "failed to initialize nvme_ctrl_t: %s",
                    nvme_errmsg(disk->td_nvme));
                ret = topo_mod_seterrno(mod, EMOD_UNKNOWN);
                goto error;
        }

        nvme_info.nei_vers = nvme_ctrl_info_version(nvme_info.nei_ctrl_info);

        if ((ret = make_nvme_node(&nvme_info)) != 0) {
                goto error;
        }

error:
        if (nvme_info.nei_ctrl_info != NULL)
                nvme_ctrl_info_free(nvme_info.nei_ctrl_info);
        if (nvme_info.nei_ctrl != NULL)
                nvme_ctrl_fini(nvme_info.nei_ctrl);
        return (ret);
}

int
disk_nvme_enum_disk(topo_mod_t *mod, tnode_t *pnode)
{
        char *parent = NULL;
        int err;
        di_node_t devtree;
        di_node_t dnode;
        int ret = -1;

        /*
         * Lookup a property containing the devfs path of the parent PCIe
         * device of the NVMe device we're attempting to enumerate.  This
         * property is hard-coded in per-platform topo XML maps that are
         * delivered with the OS.  This hard-coded path allows topo to map a
         * given NVMe controller to a physical location (bay or slot) on the
         * platform, when generating the topo snapshot.
         */
        if (topo_prop_get_string(pnode, TOPO_PGROUP_BINDING,
            TOPO_BINDING_PARENT_DEV, &parent, &err) != 0) {
                topo_mod_dprintf(mod, "parent node was missing nvme binding "
                    "properties\n");
                (void) topo_mod_seterrno(mod, err);
                goto out;
        }
        if ((devtree = topo_mod_devinfo(mod)) == DI_NODE_NIL) {
                topo_mod_dprintf(mod, "failed to get devinfo snapshot");
                (void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
                goto out;
        }

        /*
         * Walk the devinfo tree looking NVMe devices. For each NVMe device,
         * check if the devfs path of the parent matches the one specified in
         * TOPO_BINDING_PARENT_DEV.
         */
        dnode = di_drv_first_node(NVME_DRV, devtree);
        while (dnode != DI_NODE_NIL) {
                char *path;

                if ((path = di_devfs_path(di_parent_node(dnode))) == NULL) {
                        topo_mod_dprintf(mod, "failed to get dev path");
                        (void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
                        goto out;
                }
                if (strcmp(parent, path) == 0) {
                        ret = discover_nvme_ctl(mod, pnode, dnode);
                        di_devfs_path_free(path);
                        goto out;
                }
                di_devfs_path_free(path);
                dnode = di_drv_next_node(dnode);
        }
        ret = 0;

out:
        topo_mod_strfree(mod, parent);
        return (ret);
}