/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/sysmacros.h>
#include <sys/types.h>
#include <sys/exechdr.h>
#include <sys/elf.h>
#include <sys/elf_notes.h>
#include <sys/bootconf.h>
#include <sys/reboot.h>
#include <sys/fcntl.h>
#include <sys/stat.h>
#include <sys/modctl.h>
#include <sys/link.h>
#include <sys/auxv.h>
#include <sys/salib.h>
#include <sys/bootvfs.h>
#include <sys/platnames.h>

#include "util.h"

union {
        struct exec X;
        Elf32_Ehdr Elfhdr;
        Elf64_Ehdr Elfhdr64;
} ex;

#define x ex.X
#define elfhdr ex.Elfhdr
#define elfhdr64 ex.Elfhdr64

typedef int     (*func_t)();

#define FAIL    ((func_t)-1)
#define ALIGN(x, a)     \
        ((a) == 0 ? (uintptr_t)(x) : (((uintptr_t)(x) + (a) - 1) & ~((a) - 1)))

#define __BOOT_NAUXV_IMPL       22

int     use_align = 0;
int     npagesize = 0;
uint_t  icache_flush = 0;
char    *cpulist = NULL;
char    *mmulist = NULL;
extern char *module_path;               /* path for kernel modules */

/*
 * This file gets compiled in LP64 (for sun4u) and ILP32 models.
 * For LP64 compilation, the "client" file we load and run may be LP64 or ILP32,
 * and during bringup, the LP64 clients may have ELF32 headers.
 */
#ifdef  _ELF64_SUPPORT
/*
 * Bootstrap vector for ELF32 LP64 client
 */
Elf32_Boot *elfbootvecELF32_64;
Elf64_Boot *elfbootvecELF64;    /* ELF bootstrap vector for Elf64 LP64 */

#define OK              ((func_t)0)

#define FAIL_READELF64  ((uint64_t)0)
#define FAIL_ILOAD64    ((Elf64_Addr)-1)
#endif  /* _ELF64_SUPPORT */

/*
 * And by an ILP32 client. The non-sun4u/LP64 booters use these.
 * Also, the sun4u booter must create this for ILP32 clients.
 */
Elf32_Boot *elfbootvec;         /* ELF bootstrap vector normal ILP32 */

/*
 * Read in a Unix executable file and return its entry point.
 * Handle the various a.out formats correctly.
 * "fd" is the standalone file descriptor to read from.
 * Print informative little messages if "print" is on.
 * Returns -1 for errors.
 */

#ifdef DEBUG
static int debug = 1;
#else /* DEBUG */
static int debug = 0;
#endif /* DEBUG */

#define dprintf         if (debug) printf

#ifdef  _ELF64_SUPPORT
typedef struct {
        uint_t  a_type;
        union {
                uint64_t a_val;
                uint64_t a_ptr;
                void    (*a_fcn)();     /* XXX - UNUSED? */
        } a_un;
} auxv64_t;

#if defined(__sparcv9)
extern int client_isLP64;
#endif  /* __sparcv9 */

static uint64_t read_elf64(int, int, Elf64_Ehdr *);
static Elf64_Addr iload64(char *, Elf64_Phdr *, Elf64_Phdr *, auxv64_t **);
#endif  /* _ELF64_SUPPORT */

static func_t   read_elf32(int, int, Elf32_Ehdr *);
static func_t   iload32(char *, Elf32_Phdr *, Elf32_Phdr *, auxv32_t **);
static caddr_t  segbrk(caddr_t *, size_t, size_t);
static int      openpath(char *, char *, int);
static char     *getmodpath(char *);
extern void     setup_aux(void);

extern void     *kmem_alloc(size_t, int);
extern void     kmem_free(void *, size_t);
extern int      cons_gets(char *, int);

extern void sync_instruction_memory(caddr_t v, size_t len);

extern int      verbosemode;
extern int      boothowto;
extern int      pagesize;
extern char     filename[];

/*
 * repeat reads (forever) until size of request is satisfied
 * (Thus, you don't want to use this cases where short reads are ok)
 */
ssize_t
xread(int fd, char *p, size_t nbytes)
{
        size_t bytesread = 0;
        int errorcount = 0;
        ssize_t i;

        while (bytesread < nbytes) {
                i = read(fd, p, nbytes - bytesread);
                if (i < 0) {
                        ++errorcount;
                        if (verbosemode)
                                printf("read error (0x%x times)\n", errorcount);
                        continue;
                }
                bytesread += i;
                p += i;
        }
        return (bytesread);
}

func_t
readfile(int fd, int print)
{
#ifdef  _ELF64_SUPPORT
        uint64_t elf64_go2;
#endif  /* _ELF64_SUPPORT */

        ssize_t i;
        int shared = 0;

        if (verbosemode) {
                dprintf("fd = %x\n", fd);
        }

        i = xread(fd, (char *)&elfhdr, sizeof (Elf64_Ehdr));
        if (x.a_magic == ZMAGIC || x.a_magic == NMAGIC)
                shared = 1;
        if (i != sizeof (Elf64_Ehdr)) {
                printf("Error reading ELF header.\n");
                return (FAIL);
        }
        if (!shared && x.a_magic != OMAGIC) {
                if (*(int *)&elfhdr.e_ident == *(int *)(ELFMAG)) {
                        if (verbosemode) {
                                int is64 = (elfhdr.e_ident[EI_CLASS] ==
                                    ELFCLASS64);

                                dprintf("calling readelf, elfheader is:\n");
                                dprintf("e_ident\t0x%x, 0x%x, 0x%x, 0x%x\n",
                                    *(int *)&elfhdr.e_ident[0],
                                    *(int *)&elfhdr.e_ident[4],
                                    *(int *)&elfhdr.e_ident[8],
                                    *(int *)&elfhdr.e_ident[12]);
                                dprintf("e_machine\t0x%x\n", elfhdr.e_machine);

                                dprintf("e_entry\t\t0x%llx\n", (is64 ?
                                    elfhdr64.e_entry :
                                    (u_longlong_t)elfhdr.e_entry));
                                dprintf("e_shoff\t\t0x%llx\n", (is64 ?
                                    elfhdr64.e_shoff :
                                    (u_longlong_t)elfhdr.e_shoff));
                                dprintf("e_shnentsize\t%d\n", (is64 ?
                                    elfhdr64.e_shentsize : elfhdr.e_shentsize));
                                dprintf("e_shnum\t\t%d\n", (is64 ?
                                    elfhdr64.e_shnum : elfhdr.e_shnum));
                                dprintf("e_shstrndx\t%d\n", (is64 ?
                                    elfhdr64.e_shstrndx : elfhdr.e_shstrndx));
                        }


#ifdef  _ELF64_SUPPORT
                        dprintf("ELF file CLASS 0x%x 32 is %x 64 is %x\n",
                            elfhdr.e_ident[EI_CLASS], ELFCLASS32, ELFCLASS64);

                        if (elfhdr.e_ident[EI_CLASS] == ELFCLASS64) {
                                elf64_go2 = read_elf64(fd, print,
                                    (Elf64_Ehdr *)&elfhdr);

                                return ((elf64_go2 == FAIL_READELF64) ? FAIL :
                                    (func_t)elf64_go2);

                        } else
#endif  /* _ELF64_SUPPORT */
                                return (read_elf32(fd, print, &elfhdr));
                } else {
                        printf("File not executable.\n");
                        return (FAIL);
                }
        }
        return (FAIL);
}

/*
 * Macros to add attribute/values to the ELF bootstrap vector
 * and the aux vector. Use the type-cast to convert integers
 * to pointers first to suppress the gcc warning.
 */
#define AUX(p, a, v)    { (p)->a_type = (a); \
                        ((p)++)->a_un.a_val = (int32_t)(uintptr_t)(v); }

#define EBV(p, a, v)    { (p)->eb_tag = (a); \
                        ((p)++)->eb_un.eb_val = (Elf32_Word)(uintptr_t)(v); }

static func_t
read_elf32(int fd, int print, Elf32_Ehdr *elfhdrp)
{
        Elf32_Phdr *phdr;       /* program header */
        Elf32_Nhdr *nhdr;       /* note header */
        int nphdrs, phdrsize;
        caddr_t allphdrs;
        caddr_t namep, descp;
        Elf32_Addr loadaddr, base;
        size_t offset = 0;
        size_t size;
        uintptr_t off;
        int     i;
        int     bss_seen = 0;
        int interp = 0;                         /* interpreter required */
        static char dlname[MAXPATHLEN];         /* name of interpeter */
        uint_t dynamic;                         /* dynamic tags array */
        Elf32_Phdr *thdr;                       /* "text" program header */
        Elf32_Phdr *dhdr;                       /* "data" program header */
        func_t entrypt;                         /* entry point of standalone */

        /* Initialize pointers so we won't free bogus ones on elferror */
        allphdrs = NULL;
        nhdr = NULL;

#ifdef _ELF64_SUPPORT
        if (verbosemode)
                printf("Elf32 client\n");
#endif  /* _ELF64_SUPPORT */

        if (elfhdrp->e_phnum == 0 || elfhdrp->e_phoff == 0)
                goto elferror;

        /* use uintptr_t to suppress the gcc warning */
        entrypt = (func_t)(uintptr_t)elfhdrp->e_entry;
        if (verbosemode)
                dprintf("Entry point: %p\n", (void *)entrypt);

        /*
         * Allocate and read in all the program headers.
         */
        nphdrs = elfhdrp->e_phnum;
        phdrsize = nphdrs * elfhdrp->e_phentsize;
        allphdrs = (caddr_t)kmem_alloc(phdrsize, 0);
        if (allphdrs == NULL)
                goto elferror;
        if (verbosemode)
                dprintf("lseek: args = %x %x %x\n", fd, elfhdrp->e_phoff, 0);
        if (lseek(fd, elfhdrp->e_phoff, 0) == -1)
                goto elferror;
        if (xread(fd, allphdrs, phdrsize) != phdrsize)
                goto elferror;

        /*
         * First look for PT_NOTE headers that tell us what pagesize to
         * use in allocating program memory.
         */
        npagesize = 0;
        for (i = 0; i < nphdrs; i++) {
                void *note_buf;

                phdr = (Elf32_Phdr *)(allphdrs + elfhdrp->e_phentsize * i);
                if (phdr->p_type != PT_NOTE)
                        continue;
                if (verbosemode) {
                        dprintf("allocating 0x%x bytes for note hdr\n",
                            phdr->p_filesz);
                }
                if ((note_buf = kmem_alloc(phdr->p_filesz, 0)) == NULL)
                        goto elferror;
                if (verbosemode)
                        dprintf("seeking to 0x%x\n", phdr->p_offset);
                if (lseek(fd, phdr->p_offset, 0) == -1)
                        goto elferror;
                if (verbosemode) {
                        dprintf("reading 0x%x bytes into %p\n",
                            phdr->p_filesz, (void *)nhdr);
                }
                nhdr = (Elf32_Nhdr *)note_buf;
                if (xread(fd, (caddr_t)nhdr, phdr->p_filesz) != phdr->p_filesz)
                        goto elferror;
                if (verbosemode) {
                        dprintf("p_note namesz %x descsz %x type %x\n",
                            nhdr->n_namesz, nhdr->n_descsz, nhdr->n_type);
                }

                /*
                 * Iterate through all ELF PT_NOTE elements looking for
                 * ELF_NOTE_SOLARIS which, if present, will specify the
                 * executable's preferred pagesize.
                 */
                do {
                        namep = (caddr_t)(nhdr + 1);

                        if (nhdr->n_namesz == strlen(ELF_NOTE_SOLARIS) + 1 &&
                            strcmp(namep, ELF_NOTE_SOLARIS) == 0 &&
                            nhdr->n_type == ELF_NOTE_PAGESIZE_HINT) {
                                descp = namep + roundup(nhdr->n_namesz, 4);
                                npagesize = *(int *)descp;
                                if (verbosemode)
                                        dprintf("pagesize is %x\n", npagesize);
                        }

                        offset += sizeof (Elf32_Nhdr) + roundup(nhdr->n_namesz,
                            4) + roundup(nhdr->n_descsz, 4);

                        nhdr = (Elf32_Nhdr *)((char *)note_buf + offset);
                } while (offset < phdr->p_filesz);

                kmem_free(note_buf, phdr->p_filesz);
                nhdr = NULL;
        }

        /*
         * Next look for PT_LOAD headers to read in.
         */
        if (print)
                printf("Size: ");
        for (i = 0; i < nphdrs; i++) {
                phdr = (Elf32_Phdr *)(allphdrs + elfhdrp->e_phentsize * i);
                if (verbosemode) {
                        dprintf("Doing header 0x%x\n", i);
                        dprintf("phdr\n");
                        dprintf("\tp_offset = %x, p_vaddr = %x\n",
                            phdr->p_offset, phdr->p_vaddr);
                        dprintf("\tp_memsz = %x, p_filesz = %x\n",
                            phdr->p_memsz, phdr->p_filesz);
                }
                if (phdr->p_type == PT_LOAD) {
                        if (verbosemode)
                                dprintf("seeking to 0x%x\n", phdr->p_offset);
                        if (lseek(fd, phdr->p_offset, 0) == -1)
                                goto elferror;

                        if (phdr->p_flags == (PF_R | PF_W) &&
                            phdr->p_vaddr == 0) {
                                /*
                                 * It's a PT_LOAD segment that is RW but
                                 * not executable and has a vaddr
                                 * of zero.  This is relocation info that
                                 * doesn't need to stick around after
                                 * krtld is done with it.  We allocate boot
                                 * memory for this segment, since we don't want
                                 * it mapped in permanently as part of
                                 * the kernel image.
                                 */
                                if ((loadaddr = (uintptr_t)
                                    kmem_alloc(phdr->p_memsz, 0)) == 0)
                                        goto elferror;
                                /*
                                 * Save this to pass on
                                 * to the interpreter.
                                 */
                                phdr->p_vaddr = (Elf32_Addr)loadaddr;
                        } else {
                                if (print)
                                        printf("0x%x+", phdr->p_filesz);
                                /*
                                 * If we found a new pagesize above, use it
                                 * to adjust the memory allocation.
                                 */
                                loadaddr = phdr->p_vaddr;
                                if (use_align && npagesize != 0) {
                                        off = loadaddr & (npagesize - 1);
                                        size = roundup(phdr->p_memsz + off,
                                            npagesize);
                                        base = loadaddr - off;
                                } else {
                                        npagesize = 0;
                                        size = phdr->p_memsz;
                                        base = loadaddr;
                                }
                                /*
                                 *  Check if it's text or data.
                                 */
                                if (phdr->p_flags & PF_W)
                                        dhdr = phdr;
                                else
                                        thdr = phdr;

                                /*
                                 * If memory size is zero just ignore this
                                 * header.
                                 */
                                if (size == 0)
                                        continue;

                                if (verbosemode)
                                        dprintf("allocating memory: %x %lx "
                                            "%x\n", base, size, npagesize);
                                /*
                                 * We're all set up to read.
                                 * Now let's allocate some memory.
                                 */

                                /* use uintptr_t to suppress the gcc warning */
                                if (get_progmemory((caddr_t)(uintptr_t)base,
                                    size, npagesize))
                                        goto elferror;
                        }

                        if (verbosemode) {
                                dprintf("reading 0x%x bytes into 0x%x\n",
                                    phdr->p_filesz, loadaddr);
                        }
                        /* use uintptr_t to suppress the gcc warning */
                        if (xread(fd, (caddr_t)(uintptr_t)loadaddr,
                            phdr->p_filesz) != phdr->p_filesz)
                                goto elferror;

                        /* zero out BSS */
                        if (phdr->p_memsz > phdr->p_filesz) {
                                loadaddr += phdr->p_filesz;
                                if (verbosemode) {
                                        dprintf("bss from 0x%x size 0x%x\n",
                                            loadaddr,
                                            phdr->p_memsz - phdr->p_filesz);
                                }
                                /* use uintptr_t to suppress the gcc warning */
                                bzero((void *)(uintptr_t)loadaddr,
                                    phdr->p_memsz - phdr->p_filesz);
                                bss_seen++;
                                if (print)
                                        printf("0x%x Bytes\n",
                                            phdr->p_memsz - phdr->p_filesz);
                        }

                        /* force instructions to be visible to icache */
                        if (phdr->p_flags & PF_X) {
                                sync_instruction_memory(
                                    (caddr_t)(uintptr_t)phdr->p_vaddr,
                                    phdr->p_memsz);
                        }
                } else if (phdr->p_type == PT_INTERP) {
                        /*
                         * Dynamically-linked executable.
                         */
                        interp = 1;
                        if (lseek(fd, phdr->p_offset, 0) == -1) {
                                goto elferror;
                        }
                        /*
                         * Get the name of the interpreter.
                         */
                        if (xread(fd, dlname, phdr->p_filesz) !=
                            phdr->p_filesz ||
                            dlname[phdr->p_filesz - 1] != '\0')
                                goto elferror;
                } else if (phdr->p_type == PT_DYNAMIC) {
                        dynamic = phdr->p_vaddr;
                }
        }

        if (!bss_seen && print)
                printf("0 Bytes\n");

        /*
         * Load the interpreter
         * if there is one.
         */
        if (interp) {
                Elf32_Boot bootv[EB_MAX];               /* Bootstrap vector */
                auxv32_t auxv[__BOOT_NAUXV_IMPL];       /* Aux vector */
                Elf32_Boot *bv = bootv;
                auxv32_t *av = auxv;
                size_t vsize;

                /*
                 * Load it.
                 */
                if ((entrypt = iload32(dlname, thdr, dhdr, &av)) == FAIL)
                        goto elferror;
                /*
                 * Build bootstrap and aux vectors.
                 */
                setup_aux();
                EBV(bv, EB_AUXV, 0); /* fill in later */
                EBV(bv, EB_PAGESIZE, pagesize);
                EBV(bv, EB_DYNAMIC, dynamic);
                EBV(bv, EB_NULL, 0);

                AUX(av, AT_BASE, entrypt);
                AUX(av, AT_ENTRY, elfhdrp->e_entry);
                AUX(av, AT_PAGESZ, pagesize);
                AUX(av, AT_PHDR, allphdrs);
                AUX(av, AT_PHNUM, elfhdrp->e_phnum);
                AUX(av, AT_PHENT, elfhdrp->e_phentsize);
                if (use_align)
                        AUX(av, AT_SUN_LPAGESZ, npagesize);
                AUX(av, AT_SUN_IFLUSH, icache_flush);
                if (cpulist != NULL)
                        AUX(av, AT_SUN_CPU, cpulist);
                if (mmulist != NULL)
                        AUX(av, AT_SUN_MMU, mmulist);
                AUX(av, AT_NULL, 0);
                /*
                 * Realloc vectors and copy them.
                 */
                vsize = (caddr_t)bv - (caddr_t)bootv;
                if ((elfbootvec = (Elf32_Boot *)kmem_alloc(vsize, 0)) == NULL)
                        goto elferror;
                bcopy((char *)bootv, (char *)elfbootvec, vsize);

                size = (caddr_t)av - (caddr_t)auxv;
                if (size > sizeof (auxv)) {
                        printf("readelf: overrun of available aux vectors\n");
                        kmem_free(elfbootvec, vsize);
                        goto elferror;
                }
                /* use uintptr_t to suppress the gcc warning */
                if ((elfbootvec->eb_un.eb_ptr =
                    (Elf32_Addr)(uintptr_t)kmem_alloc(size, 0)) == 0) {
                        kmem_free(elfbootvec, vsize);
                        goto elferror;
                }
                /* use uintptr_t to suppress the gcc warning */
                bcopy(auxv,
                    (void *)(uintptr_t)(elfbootvec->eb_un.eb_ptr), size);

#if defined(_ELF64_SUPPORT)
                /*
                 * Make an LP64 copy of the vector for use by 64-bit standalones
                 * even if they have ELF32.
                 */
                if ((elfbootvecELF32_64 = (Elf32_Boot *)kmem_alloc(vsize, 0))
                    == NULL)
                        goto elferror;
                bcopy(bootv, elfbootvecELF32_64, vsize);

                size = (av - auxv) * sizeof (auxv64_t);
                /* use uintptr_t to suppress the gcc warning */
                if ((elfbootvecELF32_64->eb_un.eb_ptr =
                    (Elf32_Addr)(uintptr_t)kmem_alloc(size, 0)) == 0) {
                        kmem_free(elfbootvecELF32_64, vsize);
                        goto elferror;
                } else {
                        auxv64_t *a64 =
                            (auxv64_t *)(uintptr_t)
                            elfbootvecELF32_64->eb_un.eb_ptr;
                        auxv32_t *a = auxv;

                        for (a = auxv; a < av; a++) {
                                a64->a_type = a->a_type;
                                a64->a_un.a_val = a->a_un.a_val;
                                a64++;
                        }
                }
#endif  /* _ELF64_SUPPORT */
        } else {
                kmem_free(allphdrs, phdrsize);
        }
        return (entrypt);

elferror:
        if (allphdrs != NULL)
                kmem_free(allphdrs, phdrsize);
        if (nhdr != NULL)
                kmem_free(nhdr, phdr->p_filesz);
        printf("Elf32 read error.\n");
        return (FAIL);
}

#ifdef  _ELF64_SUPPORT
/*
 * Macros to add attribute/values to the ELF bootstrap vector
 * and the aux vector.
 */
#define AUX64(p, a, v)  { (p)->a_type = (a); \
                        ((p)++)->a_un.a_val = (uint64_t)(v); }

#define EBV64(p, a, v)  { (p)->eb_tag = (a); \
                        ((p)++)->eb_un.eb_val = (Elf64_Xword)(v); }

static uint64_t
read_elf64(int fd, int print, Elf64_Ehdr *elfhdrp)
{
        Elf64_Phdr *phdr;       /* program header */
        Elf64_Nhdr *nhdr;       /* note header */
        int nphdrs, phdrsize;
        caddr_t allphdrs;
        caddr_t namep, descp;
        Elf64_Addr loadaddr, base;
        size_t offset = 0;
        size_t size;
        int i;
        uintptr_t       off;
        int bss_seen = 0;
        int interp = 0;                         /* interpreter required */
        static char dlname[MAXPATHLEN];         /* name of interpeter */
        uintptr_t dynamic;                      /* dynamic tags array */
        Elf64_Phdr *thdr;                       /* "text" program header */
        Elf64_Phdr *dhdr;                       /* "data" program header */
        Elf64_Addr entrypt;                     /* entry point of standalone */

        /* Initialize pointers so we won't free bogus ones on elf64error */
        allphdrs = NULL;
        nhdr = NULL;
#if defined(__sparcv9)
        client_isLP64 = 1;
#endif  /* __sparcv9 */

        if (verbosemode)
                printf("Elf64 client\n");

        if (elfhdrp->e_phnum == 0 || elfhdrp->e_phoff == 0)
                goto elf64error;

        entrypt = elfhdrp->e_entry;
        if (verbosemode)
                dprintf("Entry point: 0x%llx\n", (u_longlong_t)entrypt);

        /*
         * Allocate and read in all the program headers.
         */
        nphdrs = elfhdrp->e_phnum;
        phdrsize = nphdrs * elfhdrp->e_phentsize;
        allphdrs = (caddr_t)kmem_alloc(phdrsize, 0);
        if (allphdrs == NULL)
                goto elf64error;
        if (verbosemode)
                dprintf("lseek: args = %x %llx %x\n", fd,
                    (u_longlong_t)elfhdrp->e_phoff, 0);
        if (lseek(fd, elfhdrp->e_phoff, 0) == -1)
                goto elf64error;
        if (xread(fd, allphdrs, phdrsize) != phdrsize)
                goto elf64error;

        /*
         * First look for PT_NOTE headers that tell us what pagesize to
         * use in allocating program memory.
         */
        npagesize = 0;
        for (i = 0; i < nphdrs; i++) {
                void *note_buf;

                phdr = (Elf64_Phdr *)(allphdrs + elfhdrp->e_phentsize * i);
                if (phdr->p_type != PT_NOTE)
                        continue;
                if (verbosemode) {
                        dprintf("allocating 0x%llx bytes for note hdr\n",
                            (u_longlong_t)phdr->p_filesz);
                }
                if ((note_buf = kmem_alloc(phdr->p_filesz, 0)) == NULL)
                        goto elf64error;
                if (verbosemode)
                        dprintf("seeking to 0x%llx\n",
                            (u_longlong_t)phdr->p_offset);
                if (lseek(fd, phdr->p_offset, 0) == -1)
                        goto elf64error;
                if (verbosemode) {
                        dprintf("reading 0x%llx bytes into 0x%p\n",
                            (u_longlong_t)phdr->p_filesz, (void *)nhdr);
                }
                nhdr = (Elf64_Nhdr *)note_buf;
                if (xread(fd, (caddr_t)nhdr, phdr->p_filesz) != phdr->p_filesz)
                        goto elf64error;
                if (verbosemode) {
                        dprintf("p_note namesz %x descsz %x type %x\n",
                            nhdr->n_namesz, nhdr->n_descsz, nhdr->n_type);
                }

                /*
                 * Iterate through all ELF PT_NOTE elements looking for
                 * ELF_NOTE_SOLARIS which, if present, will specify the
                 * executable's preferred pagesize.
                 */
                do {
                        namep = (caddr_t)(nhdr + 1);

                        if (nhdr->n_namesz == strlen(ELF_NOTE_SOLARIS) + 1 &&
                            strcmp(namep, ELF_NOTE_SOLARIS) == 0 &&
                            nhdr->n_type == ELF_NOTE_PAGESIZE_HINT) {
                                descp = namep + roundup(nhdr->n_namesz, 4);
                                npagesize = *(int *)descp;
                                if (verbosemode)
                                        dprintf("pagesize is %x\n", npagesize);
                        }

                        offset += sizeof (Elf64_Nhdr) + roundup(nhdr->n_namesz,
                            4) + roundup(nhdr->n_descsz, 4);

                        nhdr = (Elf64_Nhdr *)((char *)note_buf + offset);
                } while (offset < phdr->p_filesz);

                kmem_free(note_buf, phdr->p_filesz);
                nhdr = NULL;
        }

        /*
         * Next look for PT_LOAD headers to read in.
         */
        if (print)
                printf("Size: ");
        for (i = 0; i < nphdrs; i++) {
                phdr = (Elf64_Phdr *)(allphdrs + elfhdrp->e_phentsize * i);
                if (verbosemode) {
                        dprintf("Doing header 0x%x\n", i);
                        dprintf("phdr\n");
                        dprintf("\tp_offset = %llx, p_vaddr = %llx\n",
                            (u_longlong_t)phdr->p_offset,
                            (u_longlong_t)phdr->p_vaddr);
                        dprintf("\tp_memsz = %llx, p_filesz = %llx\n",
                            (u_longlong_t)phdr->p_memsz,
                            (u_longlong_t)phdr->p_filesz);
                        dprintf("\tp_type = %x, p_flags = %x\n",
                            phdr->p_type, phdr->p_flags);
                }
                if (phdr->p_type == PT_LOAD) {
                        if (verbosemode)
                                dprintf("seeking to 0x%llx\n",
                                    (u_longlong_t)phdr->p_offset);
                        if (lseek(fd, phdr->p_offset, 0) == -1)
                                goto elf64error;

                        if (phdr->p_flags == (PF_R | PF_W) &&
                            phdr->p_vaddr == 0) {
                                /*
                                 * It's a PT_LOAD segment that is RW but
                                 * not executable and has a vaddr
                                 * of zero.  This is relocation info that
                                 * doesn't need to stick around after
                                 * krtld is done with it.  We allocate boot
                                 * memory for this segment, since we don't want
                                 * it mapped in permanently as part of
                                 * the kernel image.
                                 */
                                if ((loadaddr = (Elf64_Addr)(uintptr_t)
                                    kmem_alloc(phdr->p_memsz, 0)) == 0)
                                        goto elf64error;

                                /*
                                 * Save this to pass on
                                 * to the interpreter.
                                 */
                                phdr->p_vaddr = loadaddr;
                        } else {
                                if (print)
                                        printf("0x%llx+",
                                            (u_longlong_t)phdr->p_filesz);
                                /*
                                 * If we found a new pagesize above, use it
                                 * to adjust the memory allocation.
                                 */
                                loadaddr = phdr->p_vaddr;
                                if (use_align && npagesize != 0) {
                                        off = loadaddr & (npagesize - 1);
                                        size = roundup(phdr->p_memsz + off,
                                            npagesize);
                                        base = loadaddr - off;
                                } else {
                                        npagesize = 0;
                                        size = phdr->p_memsz;
                                        base = loadaddr;
                                }
                                /*
                                 *  Check if it's text or data.
                                 */
                                if (phdr->p_flags & PF_W)
                                        dhdr = phdr;
                                else
                                        thdr = phdr;

                                if (verbosemode)
                                        dprintf(
                                            "allocating memory: %llx %lx %x\n",
                                            (u_longlong_t)base,
                                            size, npagesize);

                                /*
                                 * If memory size is zero just ignore this
                                 * header.
                                 */
                                if (size == 0)
                                        continue;

                                /*
                                 * We're all set up to read.
                                 * Now let's allocate some memory.
                                 */
                                if (get_progmemory((caddr_t)(uintptr_t)base,
                                    size, npagesize))
                                        goto elf64error;
                        }

                        if (verbosemode) {
                                dprintf("reading 0x%llx bytes into 0x%llx\n",
                                    (u_longlong_t)phdr->p_filesz,
                                    (u_longlong_t)loadaddr);
                        }
                        if (xread(fd, (caddr_t)(uintptr_t)
                            loadaddr, phdr->p_filesz) != phdr->p_filesz)
                                goto elf64error;

                        /* zero out BSS */
                        if (phdr->p_memsz > phdr->p_filesz) {
                                loadaddr += phdr->p_filesz;
                                if (verbosemode) {
                                        dprintf("bss from 0x%llx size 0x%llx\n",
                                            (u_longlong_t)loadaddr,
                                            (u_longlong_t)(phdr->p_memsz -
                                            phdr->p_filesz));
                                }

                                bzero((caddr_t)(uintptr_t)loadaddr,
                                    phdr->p_memsz - phdr->p_filesz);
                                bss_seen++;
                                if (print)
                                        printf("0x%llx Bytes\n",
                                            (u_longlong_t)(phdr->p_memsz -
                                            phdr->p_filesz));
                        }

                        /* force instructions to be visible to icache */
                        if (phdr->p_flags & PF_X)
                                sync_instruction_memory((caddr_t)(uintptr_t)
                                    phdr->p_vaddr, phdr->p_memsz);

                } else if (phdr->p_type == PT_INTERP) {
                        /*
                         * Dynamically-linked executable.
                         */
                        interp = 1;
                        if (lseek(fd, phdr->p_offset, 0) == -1) {
                                goto elf64error;
                        }
                        /*
                         * Get the name of the interpreter.
                         */
                        if (xread(fd, dlname, phdr->p_filesz) !=
                            phdr->p_filesz ||
                            dlname[phdr->p_filesz - 1] != '\0')
                                goto elf64error;
                } else if (phdr->p_type == PT_DYNAMIC) {
                        dynamic = phdr->p_vaddr;
                }
        }

        if (!bss_seen && print)
                printf("0 Bytes\n");

        /*
         * Load the interpreter
         * if there is one.
         */
        if (interp) {
                Elf64_Boot bootv[EB_MAX];               /* Bootstrap vector */
                auxv64_t auxv[__BOOT_NAUXV_IMPL];       /* Aux vector */
                Elf64_Boot *bv = bootv;
                auxv64_t *av = auxv;
                size_t vsize;

                /*
                 * Load it.
                 */
                if ((entrypt = iload64(dlname, thdr, dhdr, &av)) ==
                    FAIL_ILOAD64)
                        goto elf64error;
                /*
                 * Build bootstrap and aux vectors.
                 */
                setup_aux();
                EBV64(bv, EB_AUXV, 0); /* fill in later */
                EBV64(bv, EB_PAGESIZE, pagesize);
                EBV64(bv, EB_DYNAMIC, dynamic);
                EBV64(bv, EB_NULL, 0);

                AUX64(av, AT_BASE, entrypt);
                AUX64(av, AT_ENTRY, elfhdrp->e_entry);
                AUX64(av, AT_PAGESZ, pagesize);
                AUX64(av, AT_PHDR, (uintptr_t)allphdrs);
                AUX64(av, AT_PHNUM, elfhdrp->e_phnum);
                AUX64(av, AT_PHENT, elfhdrp->e_phentsize);
                if (npagesize)
                        AUX64(av, AT_SUN_LPAGESZ, npagesize);

                AUX64(av, AT_SUN_IFLUSH, icache_flush);
                if (cpulist != NULL)
                        AUX64(av, AT_SUN_CPU, (uintptr_t)cpulist);
                AUX64(av, AT_NULL, 0);
                /*
                 * Realloc vectors and copy them.
                 */
                vsize = (caddr_t)bv - (caddr_t)bootv;
                if ((elfbootvecELF64 =
                    (Elf64_Boot *)kmem_alloc(vsize, 0)) == NULL)
                        goto elf64error;
                bcopy((char *)bootv, (char *)elfbootvecELF64, vsize);

                size = (caddr_t)av - (caddr_t)auxv;
                if (size > sizeof (auxv)) {
                        printf("readelf: overrun of available aux vectors\n");
                        kmem_free(elfbootvecELF64, vsize);
                        goto elf64error;
                }

                if ((elfbootvecELF64->eb_un.eb_ptr =
                    (Elf64_Addr)kmem_alloc(size, 0)) == 0) {
                        kmem_free(elfbootvecELF64, vsize);
                        goto elf64error;
                }

                bcopy((char *)auxv, (char *)(elfbootvecELF64->eb_un.eb_ptr),
                    size);
        } else {
                kmem_free(allphdrs, phdrsize);
        }
        return ((uint64_t)entrypt);

elf64error:
        if (allphdrs != NULL)
                kmem_free(allphdrs, phdrsize);
        if (nhdr != NULL)
                kmem_free(nhdr, phdr->p_filesz);
        printf("Elf64 read error.\n");
        return (FAIL_READELF64);
}
#endif  /* _ELF64_SUPPORT */

/*
 * Load the interpreter.  It expects a
 * relocatable .o capable of bootstrapping
 * itself.
 */
static func_t
iload32(char *rtld, Elf32_Phdr *thdr, Elf32_Phdr *dhdr, auxv32_t **avp)
{
        Elf32_Ehdr *ehdr = NULL;
        uintptr_t dl_entry = 0;
        uint_t i;
        int fd;
        int size;
        caddr_t shdrs = NULL;
        caddr_t etext, edata;

        /* use uintptr_t to suppress the gcc warning */
        etext = (caddr_t)(uintptr_t)thdr->p_vaddr + thdr->p_memsz;
        edata = (caddr_t)(uintptr_t)dhdr->p_vaddr + dhdr->p_memsz;

        /*
         * Get the module path.
         */
        module_path = getmodpath(filename);

        if ((fd = openpath(module_path, rtld, O_RDONLY)) < 0) {
                printf("boot: cannot find %s\n", rtld);
                goto errorx;
        }
        dprintf("Opened %s OK\n", rtld);
        AUX(*avp, AT_SUN_LDNAME, rtld);
        /*
         * Allocate and read the ELF header.
         */
        if ((ehdr = (Elf32_Ehdr *)kmem_alloc(sizeof (Elf32_Ehdr), 0)) == NULL) {
                printf("boot: alloc error reading ELF header (%s).\n", rtld);
                goto error;
        }

        if (xread(fd, (char *)ehdr, sizeof (*ehdr)) != sizeof (*ehdr)) {
                printf("boot: error reading ELF header (%s).\n", rtld);
                goto error;
        }

        size = ehdr->e_shentsize * ehdr->e_shnum;
        if ((shdrs = (caddr_t)kmem_alloc(size, 0)) == NULL) {
                printf("boot: alloc error reading ELF header (%s).\n", rtld);
                goto error;
        }
        /*
         * Read the section headers.
         */
        if (lseek(fd, ehdr->e_shoff, 0) == -1 ||
            xread(fd, shdrs, size) != size) {
                printf("boot: error reading section headers\n");
                goto error;
        }
        AUX(*avp, AT_SUN_LDELF, ehdr);
        AUX(*avp, AT_SUN_LDSHDR, shdrs);
        /*
         * Load sections into the appropriate dynamic segment.
         */
        for (i = 1; i < ehdr->e_shnum; i++) {
                Elf32_Shdr *sp;
                caddr_t *spp;
                caddr_t load;

                sp = (Elf32_Shdr *)(shdrs + (i*ehdr->e_shentsize));
                /*
                 * If it's not allocated and not required
                 * to do relocation, skip it.
                 */
                if (!(sp->sh_flags & SHF_ALLOC) &&
                    sp->sh_type != SHT_RELA &&
                    sp->sh_type != SHT_SYMTAB &&
                    sp->sh_type != SHT_STRTAB)
                        continue;
                /*
                 * If the section is read-only,
                 * it goes in as text.
                 */
                spp = (sp->sh_flags & SHF_WRITE)? &edata: &etext;
                /*
                 * Make some room for it.
                 */
                load = segbrk(spp, sp->sh_size, sp->sh_addralign);
                if (load == NULL) {
                        printf("boot: allocating memory for sections failed\n");
                        goto error;
                }
                /*
                 * Compute the entry point of the linker.
                 */
                if (dl_entry == 0 &&
                    !(sp->sh_flags & SHF_WRITE) &&
                    (sp->sh_flags & SHF_EXECINSTR)) {
                        dl_entry = (uintptr_t)load + ehdr->e_entry;
                }
                /*
                 * If it's bss, just zero it out.
                 */
                if (sp->sh_type == SHT_NOBITS) {
                        bzero(load, sp->sh_size);
                } else {
                        /*
                         * Read the section contents.
                         */
                        if (lseek(fd, sp->sh_offset, 0) == -1 ||
                            xread(fd, load, sp->sh_size) != sp->sh_size) {
                                printf("boot: error reading sections\n");
                                goto error;
                        }
                }
                /*
                 * Assign the section's virtual addr. Use uintptr_t to
                 * suppress the gcc warning.
                 */
                sp->sh_addr = (Elf32_Off)(uintptr_t)load;
                /*
                 * Force instructions to be visible to icache. Use
                 * uintptr_t to suppress the gcc warning as well.
                 */
                if (sp->sh_flags & SHF_EXECINSTR)
                        sync_instruction_memory((caddr_t)(uintptr_t)sp->sh_addr,
                            sp->sh_size);
        }
        /*
         * Update sizes of segments.
         */
        thdr->p_memsz = (Elf32_Word)((uintptr_t)etext - thdr->p_vaddr);
        dhdr->p_memsz = (Elf32_Word)((uintptr_t)edata - dhdr->p_vaddr);

        /* load and relocate symbol tables in SAS */
        (void) close(fd);
        return ((func_t)dl_entry);

error:
        (void) close(fd);
errorx:
        if (ehdr)
                kmem_free(ehdr, sizeof (Elf32_Ehdr));
        if (shdrs)
                kmem_free(shdrs, size);
        printf("boot: error loading interpreter (%s)\n", rtld);
        return (FAIL);
}

#ifdef  _ELF64_SUPPORT
/*
 * Load the interpreter.  It expects a
 * relocatable .o capable of bootstrapping
 * itself.
 */
static Elf64_Addr
iload64(char *rtld, Elf64_Phdr *thdr, Elf64_Phdr *dhdr, auxv64_t **avp)
{
        Elf64_Ehdr *ehdr = NULL;
        Elf64_Addr dl_entry = (Elf64_Addr)0;
        Elf64_Addr etext, edata;
        uint_t i;
        int fd;
        int size;
        caddr_t shdrs = NULL;

        etext = thdr->p_vaddr + thdr->p_memsz;
        edata = dhdr->p_vaddr + dhdr->p_memsz;

        /*
         * Get the module path.
         */
        module_path = getmodpath(filename);

        if ((fd = openpath(module_path, rtld, O_RDONLY)) < 0) {
                printf("boot: cannot find %s\n", rtld);
                goto errorx;
        }
        dprintf("Opened %s OK\n", rtld);
        AUX64(*avp, AT_SUN_LDNAME, (uintptr_t)rtld);
        /*
         * Allocate and read the ELF header.
         */
        if ((ehdr = (Elf64_Ehdr *)kmem_alloc(sizeof (Elf64_Ehdr), 0)) == NULL) {
                printf("boot: alloc error reading ELF header (%s).\n", rtld);
                goto error;
        }

        if (xread(fd, (char *)ehdr, sizeof (*ehdr)) != sizeof (*ehdr)) {
                printf("boot: error reading ELF header (%s).\n", rtld);
                goto error;
        }

        size = ehdr->e_shentsize * ehdr->e_shnum;
        if ((shdrs = (caddr_t)kmem_alloc(size, 0)) == NULL) {
                printf("boot: alloc error reading ELF header (%s).\n", rtld);
                goto error;
        }
        /*
         * Read the section headers.
         */
        if (lseek(fd, ehdr->e_shoff, 0) == -1 ||
            xread(fd, shdrs, size) != size) {
                printf("boot: error reading section headers\n");
                goto error;
        }

        AUX64(*avp, AT_SUN_LDELF, ehdr);
        AUX64(*avp, AT_SUN_LDSHDR, shdrs);

        /*
         * Load sections into the appropriate dynamic segment.
         */
        for (i = 1; i < ehdr->e_shnum; i++) {
                Elf64_Shdr *sp;
                Elf64_Addr *spp, load;

                sp = (Elf64_Shdr *)(shdrs + (i*ehdr->e_shentsize));
                /*
                 * If it's not allocated and not required
                 * to do relocation, skip it.
                 */
                if (!(sp->sh_flags & SHF_ALLOC) &&
                    sp->sh_type != SHT_SYMTAB &&
                    sp->sh_type != SHT_STRTAB &&
                    sp->sh_type != SHT_RELA)
                        continue;
                /*
                 * If the section is read-only,
                 * it goes in as text.
                 */
                spp = (sp->sh_flags & SHF_WRITE)? &edata: &etext;

                /*
                 * Make some room for it.
                 */
                load = (Elf64_Addr)segbrk((caddr_t *)spp, sp->sh_size,
                    sp->sh_addralign);

                if (load == 0) {
                        printf("boot: allocating memory for section %d "
                            "failed\n", i);
                        goto error;
                }

                /*
                 * Compute the entry point of the linker.
                 */
                if (dl_entry == 0 &&
                    !(sp->sh_flags & SHF_WRITE) &&
                    (sp->sh_flags & SHF_EXECINSTR)) {
                        dl_entry = load + ehdr->e_entry;
                        if (verbosemode)
                                dprintf("boot: loading linker @ 0x%llx\n",
                                    (u_longlong_t)dl_entry);
                }

                /*
                 * If it's bss, just zero it out.
                 */
                if (sp->sh_type == SHT_NOBITS) {
                        bzero((caddr_t)(uintptr_t)load, sp->sh_size);
                } else {
                        /*
                         * Read the section contents.
                         */
                        if (lseek(fd, sp->sh_offset, 0) == -1 ||
                            xread(fd, (caddr_t)(uintptr_t)load, sp->sh_size) !=
                            sp->sh_size) {
                                printf("boot: error reading section %d\n", i);
                                goto error;
                        }
                }
                /*
                 * Assign the section's virtual addr.
                 */

                sp->sh_addr = load;

                if (verbosemode)
                        dprintf("boot: section %d, type %d, loaded @ 0x%llx, "
                            "size 0x%llx\n", i, sp->sh_type, (u_longlong_t)load,
                            (u_longlong_t)sp->sh_size);

                /* force instructions to be visible to icache */
                if (sp->sh_flags & SHF_EXECINSTR)
                        sync_instruction_memory((caddr_t)(uintptr_t)sp->sh_addr,
                            sp->sh_size);
        }
        /*
         * Update sizes of segments.
         */
        thdr->p_memsz = etext - thdr->p_vaddr;
        dhdr->p_memsz = edata - dhdr->p_vaddr;

        /* load and relocate symbol tables in SAS */
        (void) close(fd);
        return (dl_entry);

error:
        (void) close(fd);
errorx:
        if (ehdr)
                kmem_free((caddr_t)ehdr, sizeof (Elf64_Ehdr));
        if (shdrs)
                kmem_free(shdrs, size);
        printf("boot: error loading interpreter (%s)\n", rtld);
        return (FAIL_ILOAD64);
}
#endif  /* _ELF64_SUPPORT */

/*
 * Extend the segment's "break" value by bytes.
 */
static caddr_t
segbrk(caddr_t *spp, size_t bytes, size_t align)
{
        caddr_t va, pva;
        size_t size = 0;
        unsigned int alloc_pagesize = pagesize;
        unsigned int alloc_align = 0;

        if (npagesize) {
                alloc_align = npagesize;
                alloc_pagesize = npagesize;
        }

        va = (caddr_t)ALIGN(*spp, align);
        pva = (caddr_t)roundup((uintptr_t)*spp, alloc_pagesize);
        /*
         * Need more pages?
         */
        if (va + bytes > pva) {
                size = roundup((bytes - (pva - va)), alloc_pagesize);

                if (get_progmemory(pva, size, alloc_align)) {
                        printf("boot: segbrk allocation failed, "
                            "0x%lx bytes @ %p\n", bytes, (void *)pva);
                        return (NULL);
                }
        }
        *spp = va + bytes;

        return (va);
}

/*
 * Open the file using a search path and
 * return the file descriptor (or -1 on failure).
 */
static int
openpath(char *path, char *fname, int flags)
{
        register char *p, *q;
        char buf[MAXPATHLEN];
        int fd;

        /*
         * If the file name is absolute,
         * don't use the module search path.
         */
        if (fname[0] == '/')
                return (open(fname, flags));

        q = NULL;
        for (p = path;  /* forever */;  p = q) {

                while (*p == ' ' || *p == '\t' || *p == ':')
                        p++;
                if (*p == '\0')
                        break;
                q = p;
                while (*q && *q != ' ' && *q != '\t' && *q != ':')
                        q++;
                (void) strncpy(buf, p, q - p);
                if (q[-1] != '/') {
                        buf[q - p] = '/';
                        (void) strcpy(&buf[q - p + 1], fname);
                } else {
                        /*
                         * This checks for paths that end in '/'
                         */
                        (void) strcpy(&buf[q - p], fname);
                }

                if ((fd = open(buf, flags)) > 0)
                        return (fd);
        }
        return (-1);
}

/*
 * Get the module search path.
 */
static char *
getmodpath(char *fname)
{
        register char *p = strrchr(fname, '/');
        static char mod_path[MOD_MAXPATH];
        size_t len;
        extern char *impl_arch_name;
#if defined(__sparcv9)
        char    *isastr = "/sparcv9";
        size_t  isalen = strlen(isastr);
#endif  /* __sparcv9 */

        if (p == NULL) {
                /* strchr could not find a "/" */
                printf("%s is not a legal kernel pathname", fname);
                return (NULL);
        }
        while (p > fname && *(p - 1) == '/')
                p--;            /* remove trailing "/"s */
        if (p == fname)
                p++;            /* "/" is the modpath in this case */

        len = p - fname;
        (void) strncpy(mod_path, fname, len);
        mod_path[len] = 0;

#if defined(__sparcv9)
        len = strlen(mod_path);
        if ((len > isalen) && (strcmp(&mod_path[len - isalen], isastr) == 0)) {
                mod_path[len - isalen] = '\0';
                if ((client_isLP64 == 0) && verbosemode)
                        printf("Assuming LP64 %s client.\n", isastr);
                client_isLP64 = 1;
        }
#endif  /* __sparcv9 */
        mod_path_uname_m(mod_path, impl_arch_name);
        (void) strcat(mod_path, " ");
        (void) strcat(mod_path, MOD_DEFPATH);

        if (boothowto & RB_ASKNAME) {
                char buf[MOD_MAXPATH];

                printf("Enter default directory for modules [%s]: ", mod_path);
                (void) cons_gets(buf, sizeof (buf));
                if (buf[0] != '\0')
                        (void) strcpy(mod_path, buf);
        }
        if (verbosemode)
                printf("modpath: %s\n", mod_path);
        return (mod_path);
}