/*      $NetBSD: makemandb.c,v 1.68 2025/12/10 21:33:01 andvar Exp $    */
/*
 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
 * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <sys/cdefs.h>
__RCSID("$NetBSD: makemandb.c,v 1.68 2025/12/10 21:33:01 andvar Exp $");

#include <sys/stat.h>
#include <sys/types.h>

#include <assert.h>
#include <dirent.h>
#include <err.h>
#include <archive.h>
#include <libgen.h>
#include <md5.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <util.h>

#include "apropos-utils.h"
#include "dist/man.h"
#include "dist/mandoc.h"
#include "dist/mdoc.h"
#include "dist/roff.h"
#include "dist/mandoc_parse.h"

#define BUFLEN 1024
#define MDOC 0  //If the page is of mdoc(7) type
#define MAN 1   //If the page  is of man(7) type

/*
 * A data structure for holding section specific data.
 */
typedef struct secbuff {
        char *data;
        size_t buflen;  //Total length of buffer allocated initially
        size_t offset;  // Current offset in the buffer.
} secbuff;

typedef struct makemandb_flags {
        int optimize;
        int limit;      // limit the indexing to only NAME section
        int recreate;   // Database was created from scratch
        int verbosity;  // 0: quiet, 1: default, 2: verbose
} makemandb_flags;

typedef struct roff_mandb_rec {
        /* Fields for mandb table */
        char *name;     // for storing the name of the man page
        char *name_desc; // for storing the one line description (.Nd)
        secbuff desc; // for storing the DESCRIPTION section
        secbuff lib; // for the LIBRARY section
        secbuff return_vals; // RETURN VALUES
        secbuff env; // ENVIRONMENT
        secbuff files; // FILES
        secbuff exit_status; // EXIT STATUS
        secbuff diagnostics; // DIAGNOSTICS
        secbuff errors; // ERRORS
        char *section;

        int xr_found; // To track whether a .Xr was seen when parsing a section

        /* Fields for mandb_meta table */
        char *md5_hash;
        dev_t device;
        ino_t inode;
        time_t mtime;

        /* Fields for mandb_links table */
        char *machine;
        char *links; //all the links to a page in a space separated form
        char *file_path;

        /* Non-db fields */
        int page_type; //Indicates the type of page: mdoc or man
} mandb_rec;

typedef void (*proff_nf)(const struct roff_node *n, mandb_rec *);

static void append(secbuff *sbuff, const char *src);
static void init_secbuffs(mandb_rec *);
static void free_secbuffs(mandb_rec *);
static int check_md5(const char *, sqlite3 *, char **, void *, size_t);
static void cleanup(mandb_rec *);
static void set_section(const struct roff_meta *, mandb_rec *);
static void set_machine(const struct roff_meta *, mandb_rec *);
static int insert_into_db(sqlite3 *, mandb_rec *);
static  void begin_parse(const char *, struct mparse *, mandb_rec *, int);
static void proff_node(const struct roff_node *, mandb_rec *,
                       struct roff_meta *, const proff_nf *);
static void pmdoc_Nm(const struct roff_node *, mandb_rec *);
static void pmdoc_Nd(const struct roff_node *, mandb_rec *);
static void pmdoc_Sh(const struct roff_node *, mandb_rec *);
static void mdoc_parse_Sh(const struct roff_node *, mandb_rec *);
static void pmdoc_Xr(const struct roff_node *, mandb_rec *);
static void pmdoc_Pp(const struct roff_node *, mandb_rec *);
static void pmdoc_macro_handler(const struct roff_node *, mandb_rec *, int);
static void pman_parse_node(const struct roff_node *, secbuff *);
static void pman_parse_name(const struct roff_node *, mandb_rec *);
static void pman_sh(const struct roff_node *, mandb_rec *);
static void pman_block(const struct roff_node *, mandb_rec *);
static void traversedir(const char *, const char *, sqlite3 *, struct mparse *);
static void mdoc_parse_section(enum roff_sec, const char *, mandb_rec *);
static void man_parse_section(enum man_sec, const struct roff_node *, mandb_rec *);
static void build_file_cache(sqlite3 *, const char *, const char *,
                             struct stat *);
static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
__dead static void usage(void);
static void optimize(sqlite3 *);
static char *parse_escape(const char *);
static void replace_hyph(char *);
static makemandb_flags mflags = { .verbosity = 1 };

static  const proff_nf mdocs[MDOC_MAX - MDOC_Dd] = {
        NULL, /* Dd */
        NULL, /* Dt */
        NULL, /* Os */
        pmdoc_Sh, /* Sh */
        NULL, /* Ss */
        pmdoc_Pp, /* Pp */
        NULL, /* D1 */
        NULL, /* Dl */
        NULL, /* Bd */
        NULL, /* Ed */
        NULL, /* Bl */
        NULL, /* El */
        NULL, /* It */
        NULL, /* Ad */
        NULL, /* An */
        NULL, /* Ap */
        NULL, /* Ar */
        NULL, /* Cd */
        NULL, /* Cm */
        NULL, /* Dv */
        NULL, /* Er */
        NULL, /* Ev */
        NULL, /* Ex */
        NULL, /* Fa */
        NULL, /* Fd */
        NULL, /* Fl */
        NULL, /* Fn */
        NULL, /* Ft */
        NULL, /* Ic */
        NULL, /* In */
        NULL, /* Li */
        pmdoc_Nd, /* Nd */
        pmdoc_Nm, /* Nm */
        NULL, /* Op */
        NULL, /* Ot */
        NULL, /* Pa */
        NULL, /* Rv */
        NULL, /* St */
        NULL, /* Va */
        NULL, /* Vt */
        pmdoc_Xr, /* Xr */
        NULL, /* %A */
        NULL, /* %B */
        NULL, /* %D */
        NULL, /* %I */
        NULL, /* %J */
        NULL, /* %N */
        NULL, /* %O */
        NULL, /* %P */
        NULL, /* %R */
        NULL, /* %T */
        NULL, /* %V */
        NULL, /* Ac */
        NULL, /* Ao */
        NULL, /* Aq */
        NULL, /* At */
        NULL, /* Bc */
        NULL, /* Bf */
        NULL, /* Bo */
        NULL, /* Bq */
        NULL, /* Bsx */
        NULL, /* Bx */
        NULL, /* Db */
        NULL, /* Dc */
        NULL, /* Do */
        NULL, /* Dq */
        NULL, /* Ec */
        NULL, /* Ef */
        NULL, /* Em */
        NULL, /* Eo */
        NULL, /* Fx */
        NULL, /* Ms */
        NULL, /* No */
        NULL, /* Ns */
        NULL, /* Nx */
        NULL, /* Ox */
        NULL, /* Pc */
        NULL, /* Pf */
        NULL, /* Po */
        NULL, /* Pq */
        NULL, /* Qc */
        NULL, /* Ql */
        NULL, /* Qo */
        NULL, /* Qq */
        NULL, /* Re */
        NULL, /* Rs */
        NULL, /* Sc */
        NULL, /* So */
        NULL, /* Sq */
        NULL, /* Sm */
        NULL, /* Sx */
        NULL, /* Sy */
        NULL, /* Tn */
        NULL, /* Ux */
        NULL, /* Xc */
        NULL, /* Xo */
        NULL, /* Fo */
        NULL, /* Fc */
        NULL, /* Oo */
        NULL, /* Oc */
        NULL, /* Bk */
        NULL, /* Ek */
        NULL, /* Bt */
        NULL, /* Hf */
        NULL, /* Fr */
        NULL, /* Ud */
        NULL, /* Lb */
        NULL, /* Lp */
        NULL, /* Lk */
        NULL, /* Mt */
        NULL, /* Brq */
        NULL, /* Bro */
        NULL, /* Brc */
        NULL, /* %C */
        NULL, /* Es */
        NULL, /* En */
        NULL, /* Dx */
        NULL, /* %Q */
        NULL, /* %U */
        NULL /* Ta */
};

static  const proff_nf mans[MAN_MAX - MAN_TH] = {
        NULL,   //TH
        pman_sh, //SH
        NULL,   //SS
        NULL,   //TP
        NULL,   //LP
        NULL,   //PP
        NULL,   //P
        NULL,   //IP
        NULL,   //HP
        NULL,   //SM
        NULL,   //SB
        NULL,   //BI
        NULL,   //IB
        NULL,   //BR
        NULL,   //RB
        NULL,   //R
        pman_block,     //B
        NULL,   //I
        NULL,   //IR
        NULL,   //RI
        NULL,   //nf
        NULL,   //fi
        NULL,   //RE
        NULL,   //RS
        NULL,   //DT
        NULL,   //UC
        NULL,   //PD
        NULL,   //AT
        NULL,   //in
        NULL,   //OP
        NULL,   //EX
        NULL,   //EE
        NULL,   //UR
        NULL,   //UE
        NULL,   //MT
        NULL   //ME
};

int
main(int argc, char *argv[])
{
        FILE *file;
        const char *sqlstr, *manconf = NULL;
        char *line, *command;
        char *errmsg;
        int ch;
        struct mparse *mp;
        sqlite3 *db;
        ssize_t len;
        size_t linesize;
        struct roff_mandb_rec rec;

        while ((ch = getopt(argc, argv, "C:floQqv")) != -1) {
                switch (ch) {
                case 'C':
                        manconf = optarg;
                        break;
                case 'f':
                        mflags.recreate = 1;
                        break;
                case 'l':
                        mflags.limit = 1;
                        break;
                case 'o':
                        mflags.optimize = 1;
                        break;
                case 'Q':
                        mflags.verbosity = 0;
                        break;
                case 'q':
                        mflags.verbosity = 1;
                        break;
                case 'v':
                        mflags.verbosity = 2;
                        break;
                default:
                        usage();
                }
        }

        memset(&rec, 0, sizeof(rec));

        init_secbuffs(&rec);
        mchars_alloc();
        mp = mparse_alloc(MPARSE_SO | MPARSE_UTF8 | MPARSE_LATIN1 |
            MPARSE_VALIDATE, MANDOC_OS_OTHER, NULL);

        if (manconf) {
                char *arg;
                size_t command_len = shquote(manconf, NULL, 0) + 1;
                arg = emalloc(command_len);
                shquote(manconf, arg, command_len);
                easprintf(&command, "man -p -C %s", arg);
                free(arg);
        } else {
                command = estrdup("man -p");
                manconf = MANCONF;
        }

        /* Call man -p to get the list of man page dirs */
        if ((file = popen(command, "r")) == NULL) {
                free(command);
                err(EXIT_FAILURE, "popen failed");
        }
        free(command);

        if (mflags.recreate) {
                char *dbp = get_dbpath(manconf);
                /* No error here, it will fail in init_db in the same call */
                if (dbp != NULL)
                        remove(dbp);
        }

        if ((db = init_db(MANDB_CREATE, manconf)) == NULL)
                exit(EXIT_FAILURE);

        sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL,  &errmsg);
        if (errmsg != NULL) {
                warnx("%s", errmsg);
                free(errmsg);
                close_db(db);
                exit(EXIT_FAILURE);
        }

        sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
            &errmsg);
        if (errmsg != NULL) {
                warnx("%s", errmsg);
                free(errmsg);
                close_db(db);
                exit(EXIT_FAILURE);
        }

        /* Begin the transaction for indexing the pages */
        sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
        if (errmsg != NULL) {
                warnx("%s", errmsg);
                free(errmsg);
                close_db(db);
                exit(EXIT_FAILURE);
        }

        sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent,"
                 " file PRIMARY KEY);"
                 "CREATE UNIQUE INDEX metadb.index_file_cache_dev"
                 " ON file_cache (device, inode)";

        sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
        if (errmsg != NULL) {
                warnx("%s", errmsg);
                free(errmsg);
                close_db(db);
                exit(EXIT_FAILURE);
        }

        if (mflags.verbosity)
                printf("Building temporary file cache\n");
        line = NULL;
        linesize = 0;
        while ((len = getline(&line, &linesize, file)) != -1) {
                /* Replace the new line character at the end of string with '\0' */
                line[len - 1] = '\0';
                char *pdir = estrdup(dirname(line));
                /* Traverse the man page directories and parse the pages */
                traversedir(pdir, line, db, mp);
                free(pdir);
        }
        free(line);

        if (pclose(file) == -1) {
                close_db(db);
                cleanup(&rec);
                free_secbuffs(&rec);
                err(EXIT_FAILURE, "pclose error");
        }

        if (mflags.verbosity)
                printf("Performing index update\n");
        update_db(db, mp, &rec);
        mparse_free(mp);
        mchars_free();
        free_secbuffs(&rec);

        /* Commit the transaction */
        sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
        if (errmsg != NULL) {
                warnx("%s", errmsg);
                free(errmsg);
                close_db(db);
                exit(EXIT_FAILURE);
        }

        if (mflags.optimize)
                optimize(db);

        close_db(db);
        return 0;
}

/*
 * traversedir --
 *  Traverses the given directory recursively and passes all the man page files
 *  in the way to build_file_cache()
 */
static void
traversedir(const char *parent, const char *file, sqlite3 *db,
            struct mparse *mp)
{
        struct stat sb;
        struct dirent *dirp;
        DIR *dp;
        char *buf;

        if (stat(file, &sb) < 0) {
                if (mflags.verbosity)
                        warn("stat failed: %s", file);
                return;
        }

        /* If it is a directory, traverse it recursively */
        if (S_ISDIR(sb.st_mode)) {
                if ((dp = opendir(file)) == NULL) {
                        if (mflags.verbosity)
                                warn("opendir error: %s", file);
                        return;
                }

                while ((dirp = readdir(dp)) != NULL) {
                        /* Avoid . and .. entries in a directory */
                        if (dirp->d_name[0] != '.') {
                                easprintf(&buf, "%s/%s", file, dirp->d_name);
                                traversedir(parent, buf, db, mp);
                                free(buf);
                        }
                }
                closedir(dp);
                return;
        }

        if (!S_ISREG(sb.st_mode))
                return;

        if (sb.st_size == 0) {
                if (mflags.verbosity)
                        warnx("Empty file: %s", file);
                return;
        }
        build_file_cache(db, parent, file, &sb);
}

/* build_file_cache --
 *   This function generates an md5 hash of the file passed as its 2nd parameter
 *   and stores it in a temporary table file_cache along with the full file path.
 *   This is done to support incremental update of the database.
 *   The temporary table file_cache is dropped thereafter in the function
 *   update_db(), once the database has been updated.
 */
static void
build_file_cache(sqlite3 *db, const char *parent, const char *file,
                 struct stat *sb)
{
        const char *sqlstr;
        sqlite3_stmt *stmt = NULL;
        int rc, idx;
        assert(file != NULL);
        dev_t device_cache = sb->st_dev;
        ino_t inode_cache = sb->st_ino;
        time_t mtime_cache = sb->st_mtime;

        sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
                 " :mtime, :parent, :file)";
        rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
        if (rc != SQLITE_OK) {
                if (mflags.verbosity)
                        warnx("%s", sqlite3_errmsg(db));
                return;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":device");
        rc = sqlite3_bind_int64(stmt, idx, device_cache);
        if (rc != SQLITE_OK) {
                if (mflags.verbosity)
                        warnx("%s", sqlite3_errmsg(db));
                sqlite3_finalize(stmt);
                return;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":inode");
        rc = sqlite3_bind_int64(stmt, idx, inode_cache);
        if (rc != SQLITE_OK) {
                if (mflags.verbosity)
                        warnx("%s", sqlite3_errmsg(db));
                sqlite3_finalize(stmt);
                return;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":mtime");
        rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
        if (rc != SQLITE_OK) {
                if (mflags.verbosity)
                        warnx("%s", sqlite3_errmsg(db));
                sqlite3_finalize(stmt);
                return;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":parent");
        rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL);
        if (rc != SQLITE_OK) {
                if (mflags.verbosity)
                        warnx("%s", sqlite3_errmsg(db));
                sqlite3_finalize(stmt);
                return;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":file");
        rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
        if (rc != SQLITE_OK) {
                if (mflags.verbosity)
                        warnx("%s", sqlite3_errmsg(db));
                sqlite3_finalize(stmt);
                return;
        }

        sqlite3_step(stmt);
        sqlite3_finalize(stmt);
}

/* read_and_decompress --
 *      Reads the given file into memory. If it is compressed, decompress
 *      it before returning to the caller.
 */
static int
read_and_decompress(const char *file, void **bufp, size_t *len)
{
        size_t off;
        ssize_t r;
        struct archive *a;
        struct archive_entry *ae;
        char *buf;

        if ((a = archive_read_new()) == NULL)
                errx(EXIT_FAILURE, "memory allocation failed");

        *bufp = NULL;
        if (archive_read_support_filter_all(a) != ARCHIVE_OK ||
            archive_read_support_format_raw(a) != ARCHIVE_OK ||
            archive_read_open_filename(a, file, 65536) != ARCHIVE_OK ||
            archive_read_next_header(a, &ae) != ARCHIVE_OK)
                goto archive_error;
        *len = 65536;
        buf = emalloc(*len);
        off = 0;
        for (;;) {
                r = archive_read_data(a, buf + off, *len - off);
                if (r == ARCHIVE_OK) {
                        archive_read_free(a);
                        *bufp = buf;
                        *len = off;
                        return 0;
                }
                if (r <= 0) {
                        free(buf);
                        break;
                }
                off += r;
                if (off == *len) {
                        *len *= 2;
                        if (*len < off) {
                                if (mflags.verbosity)
                                        warnx("File too large: %s", file);
                                free(buf);
                                archive_read_free(a);
                                return -1;
                        }
                        buf = erealloc(buf, *len);
                }
        }

archive_error:
        warnx("Error while reading `%s': %s", file, archive_error_string(a));
        archive_read_free(a);
        return -1;
}

static void
update_existing_entry(sqlite3 *db, const char *file, const char *hash,
    mandb_rec *rec, int *new_count, int *link_count, int *err_count)
{
        int update_count, rc, idx;
        const char *inner_sqlstr;
        sqlite3_stmt *inner_stmt;

        update_count = sqlite3_total_changes(db);
        inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
                       " inode = :inode, mtime = :mtime WHERE"
                       " md5_hash = :md5 AND file = :file AND"
                       " (device <> :device2 OR inode <> "
                       "  :inode2 OR mtime <> :mtime2)";
        rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
        if (rc != SQLITE_OK) {
                if (mflags.verbosity)
                        warnx("%s", sqlite3_errmsg(db));
                return;
        }
        idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
        sqlite3_bind_int64(inner_stmt, idx, rec->device);
        idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
        sqlite3_bind_int64(inner_stmt, idx, rec->inode);
        idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
        sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
        idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
        sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
        idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
        sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
        idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
        sqlite3_bind_int64(inner_stmt, idx, rec->device);
        idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
        sqlite3_bind_int64(inner_stmt, idx, rec->inode);
        idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
        sqlite3_bind_int64(inner_stmt, idx, rec->mtime);

        rc = sqlite3_step(inner_stmt);
        if (rc == SQLITE_DONE) {
                /* Check if an update has been performed. */
                if (update_count != sqlite3_total_changes(db)) {
                        if (mflags.verbosity == 2)
                                printf("Updated %s\n", file);
                        (*new_count)++;
                } else {
                        /* Otherwise it was a hardlink. */
                        (*link_count)++;
                }
        } else {
                if (mflags.verbosity == 2)
                        warnx("Could not update the metadata for %s", file);
                (*err_count)++;
        }
        sqlite3_finalize(inner_stmt);
}

/* update_db --
 *      Does an incremental update of the database by checking the file_cache.
 *      It parses and adds the pages which are present in file_cache,
 *      but not in the database.
 *      It also removes the pages which are present in the database,
 *      but not in the file_cache.
 */
static void
update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
{
        const char *sqlstr;
        sqlite3_stmt *stmt = NULL;
        char *file;
        char *parent;
        char *errmsg = NULL;
        char *md5sum;
        void *buf;
        size_t buflen;
        struct sql_row {
                struct sql_row *next;
                dev_t device;
                ino_t inode;
                time_t mtime;
                char *parent;
                char *file;
        } *rows, *row;
        int new_count = 0;      /* Counter for newly indexed/updated pages */
        int total_count = 0;    /* Counter for total number of pages */
        int err_count = 0;      /* Counter for number of failed pages */
        int link_count = 0;     /* Counter for number of hard/sym links */
        int md5_status;
        int rc;

        sqlstr = "SELECT device, inode, mtime, parent, file"
                 " FROM metadb.file_cache fc"
                 " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE"
                 "  device = fc.device AND inode = fc.inode AND "
                 "  mtime = fc.mtime AND file = fc.file)";

        rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
        if (rc != SQLITE_OK) {
                if (mflags.verbosity)
                        warnx("%s", sqlite3_errmsg(db));
                close_db(db);
                errx(EXIT_FAILURE, "Could not query file cache");
        }

        buf = NULL;
        rows = NULL;
        while (sqlite3_step(stmt) == SQLITE_ROW) {
                row = emalloc(sizeof(struct sql_row));
                row->device = sqlite3_column_int64(stmt, 0);
                row->inode = sqlite3_column_int64(stmt, 1);
                row->mtime = sqlite3_column_int64(stmt, 2);
                row->parent = estrdup((const char *) sqlite3_column_text(stmt, 3));
                row->file = estrdup((const char *) sqlite3_column_text(stmt, 4));
                row->next = rows;
                rows = row;
                total_count++;
        }
        sqlite3_finalize(stmt);

        for ( ; rows != NULL; free(parent), free(file), free(buf)) {
                int fd;

                row = rows;
                rows = rows->next;

                rec->device = row->device;
                rec->inode = row->inode;
                rec->mtime = row->mtime;
                parent = row->parent;
                file = row->file;
                free(row);

                // XXX: reading twice!
                if (read_and_decompress(file, &buf, &buflen)) {
                        err_count++;
                        continue;
                }
                if ((fd = mparse_open(mp, file)) == -1) {
                        err_count++;
                        continue;
                }

                md5_status = check_md5(file, db, &md5sum, buf, buflen);
                assert(md5sum != NULL);
                if (md5_status == -1) {
                        if (mflags.verbosity)
                                warnx("An error occurred in checking md5 value"
                              " for file %s", file);
                        err_count++;
                        close(fd);
                        continue;
                }

                if (md5_status == 0) {
                        /*
                         * The MD5 hash is already present in the database,
                         * so simply update the metadata.
                         */
                        update_existing_entry(db, file, md5sum, rec,
                            &new_count, &link_count, &err_count);
                        free(md5sum);
                        close(fd);
                        continue;
                }

                if (md5_status == 1) {
                        /*
                         * The MD5 hash was not present in the database.
                         * This means is either a new file or an updated file.
                         * We should go ahead with parsing.
                         */
                        if (chdir(parent) == -1) {
                                if (mflags.verbosity)
                                        warn("chdir failed for `%s', could "
                                            "not index `%s'", parent, file);
                                err_count++;
                                free(md5sum);
                                close(fd);
                                continue;
                        }

                        if (mflags.verbosity == 2)
                                printf("Parsing: %s\n", file);
                        rec->md5_hash = md5sum;
                        rec->file_path = estrdup(file);
                        // file_path is freed by insert_into_db itself.
                        begin_parse(file, mp, rec, fd);
                        if (insert_into_db(db, rec) < 0) {
                                if (mflags.verbosity)
                                        warnx("Error in indexing `%s'", file);
                                err_count++;
                        } else {
                                new_count++;
                        }
                }
                close(fd);
        }

        if (mflags.verbosity == 2) {
                printf("Number of new or updated pages encountered: %d\n"
                    "Number of hard links found: %d\n"
                    "Number of pages that were successfully"
                    " indexed or updated: %d\n"
                    "Number of pages that could not be indexed"
                    " due to errors: %d\n",
                    total_count - link_count, link_count, new_count, err_count);
        }

        if (mflags.recreate)
                return;

        if (mflags.verbosity == 2)
                printf("Deleting stale index entries\n");

        sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
                 " (SELECT file FROM metadb.file_cache);"
                 "DELETE FROM mandb_links WHERE md5_hash NOT IN"
                 " (SELECT md5_hash from mandb_meta);"
                 "DROP TABLE metadb.file_cache;"
                 "DELETE FROM mandb WHERE rowid NOT IN"
                 " (SELECT id FROM mandb_meta);";

        sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
        if (errmsg != NULL) {
                warnx("Removing old entries failed: %s", errmsg);
                warnx("Please rebuild database from scratch with -f.");
                free(errmsg);
                return;
        }
}

/*
 * begin_parse --
 *  parses the man page using libmandoc
 */
static void
begin_parse(const char *file, struct mparse *mp, mandb_rec *rec, int fd)
{
        struct roff_meta *roff;
        mparse_reset(mp);

        rec->xr_found = 0;

        mparse_readfd(mp, fd, file);
        roff = mparse_result(mp);
        if (roff == NULL) {
                if (mflags.verbosity == 2)
                        warnx("Not a roff(7) page");
                return;
        }

        if (roff->macroset == MACROSET_MDOC) {
                rec->page_type = MDOC;
                proff_node(roff->first->child, rec, roff, mdocs);
        } else if (roff->macroset == MACROSET_MAN) {
                rec->page_type = MAN;
                proff_node(roff->first->child, rec, roff, mans);
        } else
                warnx("Unknown macroset %d", roff->macroset);
        set_machine(roff, rec);
        set_section(roff, rec);
}

/*
 * set_section --
 *  Extracts the section number and normalizes it to only the numeric part
 *  (Which should be the first character of the string).
 */
static void
set_section(const struct roff_meta *rm, mandb_rec *rec)
{
        if (!rm)
                return;
        const char *s = rm->msec == NULL ? "?" : rm->msec;
        easprintf(&rec->section, "%s", s);
        if (rec->section[0] == '?' && mflags.verbosity == 2)
                warnx("%s: Missing section number", rec->file_path);
}

/*
 * get_machine --
 *  Extracts the machine architecture information if available.
 */
static void
set_machine(const struct roff_meta *rm, mandb_rec *rec)
{
        if (rm == NULL)
                return;
        if (rm->arch)
                rec->machine = estrdup(rm->arch);
}

/*
 * pmdoc_Nm --
 *  Extracts the Name of the manual page from the .Nm macro
 */
static void
pmdoc_Nm(const struct roff_node *n, mandb_rec *rec)
{
        if (n->sec != SEC_NAME)
                return;

        for (n = n->child; n; n = n->next) {
                if (n->type == ROFFT_TEXT) {
                        char *escaped_name = parse_escape(n->string);
                        concat(&rec->name, escaped_name);
                        free(escaped_name);
                }
        }
}

/*
 * pmdoc_Nd --
 *  Extracts the one line description of the man page from the .Nd macro
 */
static void
pmdoc_Nd(const struct roff_node *n, mandb_rec *rec)
{
        if (n->type == ROFFT_BODY)
                deroff(&rec->name_desc, n);
        if (rec->name_desc)
                replace_hyph(rec->name_desc);

}

/*
 * pmdoc_macro_handler--
 *  This function is a single point of handling all the special macros that we
 *  want to handle especially. For example the .Xr macro for properly parsing
 *  the referenced page name along with the section number, or the .Pp macro
 *  for adding a new line whenever we encounter it.
 */
static void
pmdoc_macro_handler(const struct roff_node *n, mandb_rec *rec, int doct)
{
        const struct roff_node *sn;
        assert(n);

        switch (doct) {
        /*  Parse the man page references.
         * Basically the .Xr macros are used like:
         *  .Xr ls 1
         *  and formatted like this:
         *  ls(1)
         *  Prepare a buffer to format the data like the above example and call
         *  pmdoc_parse_section to append it.
         */
        case MDOC_Xr:
                n = n->child;
                while (n->type != ROFFT_TEXT && n->next)
                        n = n->next;

                if (n && n->type != ROFFT_TEXT)
                        return;
                sn = n;
                if (n->next)
                        n = n->next;

                while (n->type != ROFFT_TEXT && n->next)
                        n = n->next;

                if (n && n->type == ROFFT_TEXT) {
                        char *buf;
                        easprintf(&buf, "%s(%s)", sn->string, n->string);
                        mdoc_parse_section(n->sec, buf, rec);
                        free(buf);
                }

                break;

        /* Parse the .Pp macro to add a new line */
        case MDOC_Pp:
                if (n->type == ROFFT_TEXT)
                        mdoc_parse_section(n->sec, "\n", rec);
                break;
        default:
                break;
        }

}

/*
 * pmdoc_Xr, pmdoc_Pp--
 *  Empty stubs.
 *  The parser calls these functions each time it encounters
 *  a .Xr or .Pp macro. We are parsing all the data from
 *  the pmdoc_Sh function, so don't do anything here.
 *  (See if else blocks in pmdoc_Sh.)
 */
static void
pmdoc_Xr(const struct roff_node *n, mandb_rec *rec)
{
}

static void
pmdoc_Pp(const struct roff_node *n, mandb_rec *rec)
{
}

/*
 * pmdoc_Sh --
 *  Called when a .Sh macro is encountered and tries to parse its body
 */
static void
pmdoc_Sh(const struct roff_node *n, mandb_rec *rec)
{
        if (n == NULL)
                return;

        switch (n->sec) {
        case SEC_NAME:
        case SEC_SYNOPSIS:
        case SEC_EXAMPLES:
        case SEC_STANDARDS:
        case SEC_HISTORY:
        case SEC_AUTHORS:
        case SEC_BUGS:
                /*
                 * We don't care about text from these sections
                 */
                return;
        default:
                break;
        }

        if (n->type == ROFFT_BLOCK)
                mdoc_parse_Sh(n->body, rec);
}

/*
 *  Called from pmdoc_Sh to parse body of a .Sh macro. It calls
 *  mdoc_parse_section to append the data to the section specific buffer.
 *  The .Xr macro needs special handling, thus the separate if branch for it.
 */
static void
mdoc_parse_Sh(const struct roff_node *n, mandb_rec *rec)
{
        if (n == NULL || (n->type != ROFFT_TEXT && n->tok == MDOC_MAX))
                return;
        int xr_found = 0;

        if (n->type == ROFFT_TEXT) {
                mdoc_parse_section(n->sec, n->string, rec);
        } else if (n->tok >= MDOC_Dd && n->tok < MDOC_MAX) {
                const int tok_idx = n->tok - MDOC_Dd;
                if (mdocs[tok_idx] == pmdoc_Xr) {
                        /*
                        * When encountering other inline macros,
                        * call pmdoc_macro_handler.
                        */
                        pmdoc_macro_handler(n, rec, MDOC_Xr);
                        xr_found = 1;
                } else if (mdocs[tok_idx] == pmdoc_Pp) {
                        pmdoc_macro_handler(n, rec, MDOC_Pp);
                }
        }

        /*
         * If an Xr macro was encountered then the child node has
         * already been explored by pmdoc_macro_handler.
         */
        if (xr_found == 0)
                mdoc_parse_Sh(n->child, rec);
        mdoc_parse_Sh(n->next, rec);
}

/*
 * mdoc_parse_section--
 *  Utility function for parsing sections of the mdoc type pages.
 *  Takes two params:
 *   1. sec is an enum which indicates the section in which we are present
 *   2. string is the string which we need to append to the secbuff for this
 *      particular section.
 *  The function appends string to the global section buffer and returns.
 */
static void
mdoc_parse_section(enum roff_sec sec, const char *string, mandb_rec *rec)
{
        /*
         * If the user specified the 'l' flag, then parse and store only the
         * NAME section. Ignore the rest.
         */
        if (mflags.limit)
                return;

        switch (sec) {
        case SEC_LIBRARY:
                append(&rec->lib, string);
                break;
        case SEC_RETURN_VALUES:
                append(&rec->return_vals, string);
                break;
        case SEC_ENVIRONMENT:
                append(&rec->env, string);
                break;
        case SEC_FILES:
                append(&rec->files, string);
                break;
        case SEC_EXIT_STATUS:
                append(&rec->exit_status, string);
                break;
        case SEC_DIAGNOSTICS:
                append(&rec->diagnostics, string);
                break;
        case SEC_ERRORS:
                append(&rec->errors, string);
                break;
        default:
                append(&rec->desc, string);
                break;
        }
}

static void
proff_node(const struct roff_node * n, mandb_rec * rec,
    struct roff_meta * roff, const proff_nf * func)
{
        if (n == NULL)
                return;

        int tok_idx;

        switch (n->type) {
        case (ROFFT_BODY):
                /* FALLTHROUGH */
        case (ROFFT_BLOCK):
                /* FALLTHROUGH */
        case (ROFFT_ELEM):
                if (roff->macroset == MACROSET_MAN)
                        tok_idx = n->tok - MAN_TH;
                else if (roff->macroset == MACROSET_MDOC)
                        tok_idx = n->tok - MDOC_Dd;
                else
                        tok_idx = -1;
                if (tok_idx >= 0 && func[tok_idx] != NULL)
                        (*func[tok_idx]) (n, rec);
                break;
        default:
                break;
        }

        proff_node(n->child, rec, roff, func);
        proff_node(n->next, rec, roff, func);
}

/*
 * pman_parse_name --
 *  Parses the NAME section and puts the complete content in the name_desc
 *  variable.
 */
static void
pman_parse_name(const struct roff_node *n, mandb_rec *rec)
{
        if (n == NULL)
                return;

        if (n->type == ROFFT_TEXT) {
                char *tmp = parse_escape(n->string);
                concat(&rec->name_desc, tmp);
                free(tmp);
        }

        if (n->child)
                pman_parse_name(n->child, rec);

        if(n->next)
                pman_parse_name(n->next, rec);
}

/*
 * A stub function to be able to parse the macros like .B embedded inside
 * a section.
 */
static void
pman_block(const struct roff_node *n, mandb_rec *rec)
{
}

/*
 * pman_sh --
 * This function does one of the two things:
 *  1. If the present section is NAME, then it will:
 *    (a) Extract the name of the page (in case of multiple comma separated
 *        names, it will pick up the first one).
 *    (b) Build a space separated list of all the symlinks/hardlinks to
 *        this page and store in the buffer 'links'. These are extracted from
 *        the comma separated list of names in the NAME section as well.
 *    (c) Move on to the one line description section, which is after the list
 *        of names in the NAME section.
 *  2. Otherwise, it will check the section name and call the man_parse_section
 *     function, passing the enum corresponding to that section.
 */
static void
pman_sh(const struct roff_node *n, mandb_rec *rec)
{
        static const struct {
                enum man_sec section;
                const char *header;
        } mapping[] = {
            { MANSEC_DESCRIPTION, "DESCRIPTION" },
            { MANSEC_SYNOPSIS, "SYNOPSIS" },
            { MANSEC_LIBRARY, "LIBRARY" },
            { MANSEC_ERRORS, "ERRORS" },
            { MANSEC_FILES, "FILES" },
            { MANSEC_RETURN_VALUES, "RETURN VALUE" },
            { MANSEC_RETURN_VALUES, "RETURN VALUES" },
            { MANSEC_EXIT_STATUS, "EXIT STATUS" },
            { MANSEC_EXAMPLES, "EXAMPLES" },
            { MANSEC_EXAMPLES, "EXAMPLE" },
            { MANSEC_STANDARDS, "STANDARDS" },
            { MANSEC_HISTORY, "HISTORY" },
            { MANSEC_BUGS, "BUGS" },
            { MANSEC_AUTHORS, "AUTHORS" },
            { MANSEC_COPYRIGHT, "COPYRIGHT" },
        };
        const struct roff_node *head;
        char *name_desc;
        size_t sz;
        size_t i;

        if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
            head->type != ROFFT_TEXT)
                return;

        /*
         * Check if this section should be extracted and
         * where it should be stored. Handled the trivial cases first.
         */
        for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
                if (strcmp(head->string, mapping[i].header) == 0) {
                        man_parse_section(mapping[i].section, n, rec);
                        return;
                }
        }

        if (strcmp(head->string, "NAME") == 0) {
                /*
                 * We are in the NAME section.
                 * pman_parse_name will put the complete content in name_desc.
                 */
                pman_parse_name(n, rec);

                name_desc = rec->name_desc;
                if (name_desc == NULL)
                        return;

                /* Remove any leading spaces. */
                while (name_desc[0] == ' ')
                        name_desc++;

                /* If the line begins with a "\&", avoid those */
                if (name_desc[0] == '\\' && name_desc[1] == '&')
                        name_desc += 2;

                /* Now name_desc should be left with a comma-space
                 * separated list of names and the one line description
                 * of the page:
                 *     "a, b, c \- sample description"
                 * Take out the first name, before the first comma
                 * (or space) and store it in rec->name.
                 * If the page has aliases then they should be
                 * in the form of a comma separated list.
                 * Keep looping while there is a comma in name_desc,
                 * extract the alias name and store in rec->links.
                 * When there are no more commas left, break out.
                 */
                int has_alias = 0;      // Any more aliases left?
                while (*name_desc) {
                        /* Remove any leading spaces or hyphens. */
                        if (name_desc[0] == ' ' || name_desc[0] == '-') {
                                name_desc++;
                                continue;
                        }
                        sz = strcspn(name_desc, ", ");

                        /* Extract the first term and store it in rec->name. */
                        if (rec->name == NULL) {
                                if (name_desc[sz] == ',')
                                        has_alias = 1;
                                rec->name = estrndup(name_desc, sz);
                                /* XXX This would only happen with a poorly
                                 * written man page, maybe warn? */
                                if (name_desc[sz] == '\0')
                                        break;
                                name_desc += sz + 1;
                                continue;
                        }

                        /*
                         * Once rec->name is set, rest of the names
                         * are to be treated as links or aliases.
                         */
                        if (rec->name && has_alias) {
                                if (name_desc[sz] != ',') {
                                        /* No more commas left --> no more
                                         * aliases to take out */
                                        has_alias = 0;
                                }
                                concat2(&rec->links, name_desc, sz);
                                /* XXX This would only happen with a poorly
                                 * written man page, maybe warn? */
                                if (name_desc[sz] == '\0')
                                        break;
                                name_desc += sz + 1;
                                continue;
                        }
                        break;
                }

                /* Parse any escape sequences that might be there */
                char *temp = parse_escape(name_desc);
                free(rec->name_desc);
                rec->name_desc = temp;
                temp = parse_escape(rec->name);
                free(rec->name);
                rec->name = temp;
                return;
        }

        /* The RETURN VALUE section might be specified in multiple ways */
        if (strcmp(head->string, "RETURN") == 0 &&
            head->next != NULL && head->next->type == ROFFT_TEXT &&
            (strcmp(head->next->string, "VALUE") == 0 ||
            strcmp(head->next->string, "VALUES") == 0)) {
                man_parse_section(MANSEC_RETURN_VALUES, n, rec);
                return;
        }

        /*
         * EXIT STATUS section can also be specified all on one line or on two
         * separate lines.
         */
        if (strcmp(head->string, "EXIT") == 0 &&
            head->next != NULL && head->next->type == ROFFT_TEXT &&
            strcmp(head->next->string, "STATUS") == 0) {
                man_parse_section(MANSEC_EXIT_STATUS, n, rec);
                return;
        }

        /* Store the rest of the content in desc. */
        man_parse_section(MANSEC_NONE, n, rec);
}

/*
 * pman_parse_node --
 *  Generic function to iterate through a node. Usually called from
 *  man_parse_section to parse a particular section of the man page.
 */
static void
pman_parse_node(const struct roff_node *n, secbuff *s)
{
        if (n == NULL)
                return;

        if (n->type == ROFFT_TEXT)
                append(s, n->string);

        pman_parse_node(n->child, s);
        pman_parse_node(n->next, s);
}

/*
 * man_parse_section --
 *  Takes two parameters:
 *   sec: Tells which section we are present in
 *   n: Is the present node of the AST.
 * Depending on the section, we call pman_parse_node to parse that section and
 * concatenate the content from that section into the buffer for that section.
 */
static void
man_parse_section(enum man_sec sec, const struct roff_node *n, mandb_rec *rec)
{
        /*
         * If the user specified the 'l' flag then just parse
         * the NAME section, ignore the rest.
         */
        if (mflags.limit)
                return;

        switch (sec) {
        case MANSEC_LIBRARY:
                pman_parse_node(n, &rec->lib);
                break;
        case MANSEC_RETURN_VALUES:
                pman_parse_node(n, &rec->return_vals);
                break;
        case MANSEC_ENVIRONMENT:
                pman_parse_node(n, &rec->env);
                break;
        case MANSEC_FILES:
                pman_parse_node(n, &rec->files);
                break;
        case MANSEC_EXIT_STATUS:
                pman_parse_node(n, &rec->exit_status);
                break;
        case MANSEC_DIAGNOSTICS:
                pman_parse_node(n, &rec->diagnostics);
                break;
        case MANSEC_ERRORS:
                pman_parse_node(n, &rec->errors);
                break;
        case MANSEC_NAME:
        case MANSEC_SYNOPSIS:
        case MANSEC_EXAMPLES:
        case MANSEC_STANDARDS:
        case MANSEC_HISTORY:
        case MANSEC_BUGS:
        case MANSEC_AUTHORS:
        case MANSEC_COPYRIGHT:
                break;
        default:
                pman_parse_node(n, &rec->desc);
                break;
        }

}

/*
 * insert_into_db --
 *  Inserts the parsed data of the man page in the SQLite database.
 *  If any of the values is NULL, then we cleanup and return -1 indicating
 *  an error.
 *  Otherwise, store the data in the database and return 0.
 */
static int
insert_into_db(sqlite3 *db, mandb_rec *rec)
{
        int rc = 0;
        int idx = -1;
        const char *sqlstr = NULL;
        sqlite3_stmt *stmt = NULL;
        char *ln = NULL;
        char *errmsg = NULL;
        long int mandb_rowid;

        /*
         * At the very minimum we want to make sure that we store
         * the following data:
         *   Name, one line description, and the MD5 hash
         */
        if (rec->name == NULL || rec->name_desc == NULL ||
            rec->md5_hash == NULL) {
                cleanup(rec);
                return -1;
        }

        /* Write null byte at the end of all the sec_buffs */
        rec->desc.data[rec->desc.offset] = 0;
        rec->lib.data[rec->lib.offset] = 0;
        rec->env.data[rec->env.offset] = 0;
        rec->return_vals.data[rec->return_vals.offset] = 0;
        rec->exit_status.data[rec->exit_status.offset] = 0;
        rec->files.data[rec->files.offset] = 0;
        rec->diagnostics.data[rec->diagnostics.offset] = 0;
        rec->errors.data[rec->errors.offset] = 0;

        /*
         * In case of a mdoc page: (sorry, no better place to put this code)
         * parse the comma separated list of names of man pages,
         * the first name will be stored in the mandb table, rest will be
         * treated as links and put in the mandb_links table.
         */
        if (rec->page_type == MDOC) {
                char *tmp;
                rec->links = estrdup(rec->name);
                free(rec->name);
                size_t sz = strcspn(rec->links, " \0");
                rec->name = emalloc(sz + 1);
                memcpy(rec->name, rec->links, sz);
                if(rec->name[sz - 1] == ',')
                        rec->name[sz - 1] = 0;
                else
                        rec->name[sz] = 0;
                while (rec->links[sz] == ' ')
                        ++sz;
                tmp = estrdup(rec->links + sz);
                free(rec->links);
                rec->links = tmp;
        }

/*------------------------ Populate the mandb table---------------------------*/
        sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
                 " :lib, :return_vals, :env, :files, :exit_status,"
                 " :diagnostics, :errors, :md5_hash, :machine)";

        rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
        if (rc != SQLITE_OK)
                goto Out;

        idx = sqlite3_bind_parameter_index(stmt, ":name");
        rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":section");
        rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
        rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":desc");
        rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
                               rec->desc.offset + 1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":lib");
        rc = sqlite3_bind_text(stmt, idx, rec->lib.data,
            rec->lib.offset + 1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
        rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
                              rec->return_vals.offset + 1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":env");
        rc = sqlite3_bind_text(stmt, idx, rec->env.data,
            rec->env.offset + 1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":files");
        rc = sqlite3_bind_text(stmt, idx, rec->files.data,
                               rec->files.offset + 1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
        rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
                               rec->exit_status.offset + 1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
        rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
                               rec->diagnostics.offset + 1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":errors");
        rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
                               rec->errors.offset + 1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
        rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":machine");
        if (rec->machine)
                rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
        else
                rc = sqlite3_bind_null(stmt, idx);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        rc = sqlite3_step(stmt);
        if (rc != SQLITE_DONE) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        sqlite3_finalize(stmt);

        /* Get the row id of the last inserted row */
        mandb_rowid = sqlite3_last_insert_rowid(db);

/*------------------------Populate the mandb_meta table-----------------------*/
        sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
                 " :file, :md5_hash, :id)";
        rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
        if (rc != SQLITE_OK)
                goto Out;

        idx = sqlite3_bind_parameter_index(stmt, ":device");
        rc = sqlite3_bind_int64(stmt, idx, rec->device);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":inode");
        rc = sqlite3_bind_int64(stmt, idx, rec->inode);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":mtime");
        rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":file");
        rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
        rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":id");
        rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
        if (rc != SQLITE_OK) {
                sqlite3_finalize(stmt);
                goto Out;
        }

        rc = sqlite3_step(stmt);
        sqlite3_finalize(stmt);
        if (rc == SQLITE_CONSTRAINT_UNIQUE) {
                /* The *most* probable reason for reaching here is that
                 * the UNIQUE constraint on the file column of the mandb_meta
                 * table was violated.
                 * This can happen when a file was updated/modified.
                 * To fix this we need to do two things:
                 * 1. Delete the row for the older version of this file
                 *    from mandb table.
                 * 2. Run an UPDATE query to update the row for this file
                 *    in the mandb_meta table.
                 */
                warnx("Trying to update index for %s", rec->file_path);
                char *sql = sqlite3_mprintf("DELETE FROM mandb "
                                            "WHERE rowid = (SELECT id"
                                            "  FROM mandb_meta"
                                            "  WHERE file = %Q)",
                                            rec->file_path);
                sqlite3_exec(db, sql, NULL, NULL, &errmsg);
                sqlite3_free(sql);
                if (errmsg != NULL) {
                        if (mflags.verbosity)
                                warnx("%s", errmsg);
                        free(errmsg);
                }
                sqlstr = "UPDATE mandb_meta SET device = :device,"
                         " inode = :inode, mtime = :mtime, id = :id,"
                         " md5_hash = :md5 WHERE file = :file";
                rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
                if (rc != SQLITE_OK) {
                        if (mflags.verbosity)
                                warnx("Update failed with error: %s",
                            sqlite3_errmsg(db));
                        close_db(db);
                        cleanup(rec);
                        errx(EXIT_FAILURE,
                            "Consider running makemandb with -f option");
                }

                idx = sqlite3_bind_parameter_index(stmt, ":device");
                sqlite3_bind_int64(stmt, idx, rec->device);
                idx = sqlite3_bind_parameter_index(stmt, ":inode");
                sqlite3_bind_int64(stmt, idx, rec->inode);
                idx = sqlite3_bind_parameter_index(stmt, ":mtime");
                sqlite3_bind_int64(stmt, idx, rec->mtime);
                idx = sqlite3_bind_parameter_index(stmt, ":id");
                sqlite3_bind_int64(stmt, idx, mandb_rowid);
                idx = sqlite3_bind_parameter_index(stmt, ":md5");
                sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
                idx = sqlite3_bind_parameter_index(stmt, ":file");
                sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
                rc = sqlite3_step(stmt);
                sqlite3_finalize(stmt);

                if (rc != SQLITE_DONE) {
                        if (mflags.verbosity)
                                warnx("%s", sqlite3_errmsg(db));
                        close_db(db);
                        cleanup(rec);
                        errx(EXIT_FAILURE,
                            "Consider running makemandb with -f option");
                }
        } else if (rc != SQLITE_DONE) {
                /* Otherwise make this error fatal */
                warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
                cleanup(rec);
                close_db(db);
                exit(EXIT_FAILURE);
        }

/*------------------------ Populate the mandb_links table---------------------*/
        char *str = NULL;
        char *links;
        if (rec->links && strlen(rec->links)) {
                links = rec->links;
                for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
                        if (ln[0] == ',')
                                ln++;
                        if(ln[strlen(ln) - 1] == ',')
                                ln[strlen(ln) - 1] = 0;

                        str = sqlite3_mprintf("INSERT INTO mandb_links"
                                              " VALUES (%Q, %Q, %Q, %Q, %Q, %Q)",
                                              ln, rec->name, rec->section,
                                              rec->machine, rec->md5_hash, rec->name_desc);
                        sqlite3_exec(db, str, NULL, NULL, &errmsg);
                        sqlite3_free(str);
                        if (errmsg != NULL) {
                                warnx("%s", errmsg);
                                cleanup(rec);
                                free(errmsg);
                                return -1;
                        }
                }
        }

        cleanup(rec);
        return 0;

  Out:
        if (mflags.verbosity)
                warnx("%s", sqlite3_errmsg(db));
        cleanup(rec);
        return -1;
}

/*
 * check_md5--
 *  Generates the md5 hash of the file and checks if it already doesn't exist
 *  in the table.
 *  This function is being used to avoid hardlinks.
 *  On successful completion it will also set the value of the fourth parameter
 *  to the md5 hash of the file (computed previously). It is the responsibility
 *  of the caller to free this buffer.
 *  Return values:
 *  -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
 *  0: If the md5 hash does not exist in the table.
 *  1: If the hash exists in the database.
 */
static int
check_md5(const char *file, sqlite3 *db, char **md5, void *buf, size_t buflen)
{
        int rc = 0;
        int idx = -1;
        char *sqlstr = NULL;
        char *mymd5;
        sqlite3_stmt *stmt = NULL;
        *md5 = NULL;

        assert(file != NULL);
        if ((mymd5 = MD5Data(buf, buflen, NULL)) == NULL) {
                if (mflags.verbosity)
                        warn("md5 failed: %s", file);
                return -1;
        }

        easprintf(&sqlstr, "SELECT * FROM mandb_meta WHERE md5_hash = :md5_hash");
        rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
        if (rc != SQLITE_OK) {
                free(sqlstr);
                free(mymd5);
                return -1;
        }

        idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
        rc = sqlite3_bind_text(stmt, idx, mymd5, -1, NULL);
        if (rc != SQLITE_OK) {
                if (mflags.verbosity)
                        warnx("%s", sqlite3_errmsg(db));
                sqlite3_finalize(stmt);
                free(sqlstr);
                free(mymd5);
                return -1;
        }

        *md5 = mymd5;
        if (sqlite3_step(stmt) == SQLITE_ROW) {
                sqlite3_finalize(stmt);
                free(sqlstr);
                return 0;
        }

        sqlite3_finalize(stmt);
        free(sqlstr);
        return 1;
}

/* Optimize the index for faster search */
static void
optimize(sqlite3 *db)
{
        const char *sqlstr;
        char *errmsg = NULL;

        if (mflags.verbosity == 2)
                printf("Optimizing the database index\n");
        sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
                 "VACUUM";
        sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
        if (errmsg != NULL) {
                if (mflags.verbosity)
                        warnx("%s", errmsg);
                free(errmsg);
                return;
        }
}

/*
 * cleanup --
 *  cleans up the global buffers
 */
static void
cleanup(mandb_rec *rec)
{
        rec->desc.offset = 0;
        rec->lib.offset = 0;
        rec->return_vals.offset = 0;
        rec->env.offset = 0;
        rec->exit_status.offset = 0;
        rec->diagnostics.offset = 0;
        rec->errors.offset = 0;
        rec->files.offset = 0;

        free(rec->machine);
        rec->machine = NULL;

        free(rec->links);
        rec->links = NULL;

        free(rec->file_path);
        rec->file_path = NULL;

        free(rec->name);
        rec->name = NULL;

        free(rec->name_desc);
        rec->name_desc = NULL;

        free(rec->md5_hash);
        rec->md5_hash = NULL;

        free(rec->section);
        rec->section = NULL;
}

/*
 * init_secbuffs--
 *  Sets the value of buflen for all the sec_buff field of rec. And then
 *  allocate memory to each sec_buff member of rec.
 */
static void
init_secbuffs(mandb_rec *rec)
{
        /*
         * Some sec_buff might need more memory, for example desc,
         * which stores the data of the DESCRIPTION section,
         * while some might need very small amount of memory.
         * Therefore explicitly setting the value of buflen field for
         * each sec_buff.
         */
        rec->desc.buflen = 10 * BUFLEN;
        rec->desc.data = emalloc(rec->desc.buflen);
        rec->desc.offset = 0;

        rec->lib.buflen = BUFLEN / 2;
        rec->lib.data = emalloc(rec->lib.buflen);
        rec->lib.offset = 0;

        rec->return_vals.buflen = BUFLEN;
        rec->return_vals.data = emalloc(rec->return_vals.buflen);
        rec->return_vals.offset = 0;

        rec->exit_status.buflen = BUFLEN;
        rec->exit_status.data = emalloc(rec->exit_status.buflen);
        rec->exit_status.offset = 0;

        rec->env.buflen = BUFLEN;
        rec->env.data = emalloc(rec->env.buflen);
        rec->env.offset = 0;

        rec->files.buflen = BUFLEN;
        rec->files.data = emalloc(rec->files.buflen);
        rec->files.offset = 0;

        rec->diagnostics.buflen = BUFLEN;
        rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
        rec->diagnostics.offset = 0;

        rec->errors.buflen = BUFLEN;
        rec->errors.data = emalloc(rec->errors.buflen);
        rec->errors.offset = 0;
}

/*
 * free_secbuffs--
 *  This function should be called at the end, when all the pages have been
 *  parsed.
 *  It frees the memory allocated to sec_buffs by init_secbuffs in the starting.
 */
static void
free_secbuffs(mandb_rec *rec)
{
        free(rec->desc.data);
        free(rec->lib.data);
        free(rec->return_vals.data);
        free(rec->exit_status.data);
        free(rec->env.data);
        free(rec->files.data);
        free(rec->diagnostics.data);
        free(rec->errors.data);
}

static void
replace_hyph(char *str)
{
        char *iter = str;
        while ((iter = strchr(iter, ASCII_HYPH)) != NULL)
                *iter = '-';

        iter = str;
        while ((iter = strchr(iter, ASCII_NBRSP)) != NULL)
                *iter = '-';
}

static char *
parse_escape(const char *str)
{
        const char *backslash, *last_backslash;
        char *result, *iter;
        size_t len;

        assert(str);

        last_backslash = str;
        backslash = strchr(str, '\\');
        if (backslash == NULL) {
                result = estrdup(str);
                replace_hyph(result);
                return result;
        }

        result = emalloc(strlen(str) + 1);
        iter = result;

        do {
                len = backslash - last_backslash;
                memcpy(iter, last_backslash, len);
                iter += len;
                if (backslash[1] == '-' || backslash[1] == ' ') {
                        *iter++ = backslash[1];
                        last_backslash = backslash + 2;
                        backslash = strchr(last_backslash, '\\');
                } else {
                        ++backslash;
                        mandoc_escape(&backslash, NULL, NULL);
                        last_backslash = backslash;
                        if (backslash == NULL)
                                break;
                        backslash = strchr(last_backslash, '\\');
                }
        } while (backslash != NULL);
        if (last_backslash != NULL)
                strcpy(iter, last_backslash);

        replace_hyph(result);
        return result;
}

/*
 * append--
 *  Concatenates a space and src at the end of sbuff->data (much like concat in
 *  apropos-utils.c).
 *  Rather than reallocating space for writing data, it uses the value of the
 *  offset field of sec_buff to write new data at the free space left in the
 *  buffer.
 *  In case the size of the data to be appended exceeds the number of bytes left
 *  in the buffer, it reallocates buflen number of bytes and then continues.
 *  Value of offset field should be adjusted as new data is written.
 *
 *  NOTE: This function does not write the null byte at the end of the buffers,
 *  write a null byte at the position pointed to by offset before inserting data
 *  in the db.
 */
static void
append(secbuff *sbuff, const char *src)
{
        short flag = 0;
        size_t srclen, newlen;
        char *temp;

        assert(src != NULL);
        temp = parse_escape(src);
        srclen = strlen(temp);

        if (sbuff->data == NULL) {
                sbuff->data = emalloc(sbuff->buflen);
                sbuff->offset = 0;
        }

        newlen = sbuff->offset + srclen + 2;
        if (newlen >= sbuff->buflen) {
                while (sbuff->buflen < newlen)
                        sbuff->buflen += sbuff->buflen;
                sbuff->data = erealloc(sbuff->data, sbuff->buflen);
                flag = 1;
        }

        /* Append a space at the end of the buffer. */
        if (sbuff->offset || flag)
                sbuff->data[sbuff->offset++] = ' ';
        /* Now, copy src at the end of the buffer. */
        memcpy(sbuff->data + sbuff->offset, temp, srclen);
        sbuff->offset += srclen;
        free(temp);
}

static void
usage(void)
{
        fprintf(stderr, "Usage: %s [-floQqv] [-C path]\n", getprogname());
        exit(1);
}