/*      $NetBSD: custom_apropos_tokenizer.c,v 1.6 2023/08/07 20:35:21 tnn Exp $ */
/*
** 2006 September 30
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** Implementation of the full-text-search tokenizer that implements
** a Porter stemmer.
*/

/*
** The code in this file is only compiled if:
**
**     * The FTS3 module is being built as an extension
**       (in which case SQLITE_CORE is not defined), or
**
**     * The FTS3 module is being built into the core of
**       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
*/

#include <assert.h>
#include <ctype.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "custom_apropos_tokenizer.h"
#include "fts3_tokenizer.h"
#include "nostem.c"

/*
 * Class derived from sqlite3_tokenizer
 */
typedef struct custom_apropos_tokenizer {
        sqlite3_tokenizer base; /* Base class */
} custom_apropos_tokenizer;

/*
 * Class derived from sqlite3_tokenizer_cursor
 */
typedef struct custom_apropos_tokenizer_cursor {
        sqlite3_tokenizer_cursor base;
        const char *zInput;     /* input we are tokenizing */
        size_t nInput;          /* size of the input */
        size_t iOffset;         /* current position in zInput */
        size_t iToken;          /* index of next token to be returned */
        char *zToken;           /* storage for current token */
        size_t nAllocated;              /* space allocated to zToken buffer */
} custom_apropos_tokenizer_cursor;

/*
 * Create a new tokenizer instance.
 */
static int
aproposPorterCreate(int argc, const char *const * argv,
    sqlite3_tokenizer ** ppTokenizer)
{
        custom_apropos_tokenizer *t;
        t = calloc(1, sizeof(*t));
        if (t == NULL)
                return SQLITE_NOMEM;
        *ppTokenizer = &t->base;
        return SQLITE_OK;
}

/*
 * Destroy a tokenizer
 */
static int
aproposPorterDestroy(sqlite3_tokenizer * pTokenizer)
{
        free(pTokenizer);
        return SQLITE_OK;
}

/*
 * Prepare to begin tokenizing a particular string.  The input
 * string to be tokenized is zInput[0..nInput-1].  A cursor
 * used to incrementally tokenize this string is returned in
 * *ppCursor.
 */
static int
aproposPorterOpen(
    sqlite3_tokenizer * pTokenizer,     /* The tokenizer */
    const char *zInput, int nInput,     /* String to be tokenized */
    sqlite3_tokenizer_cursor ** ppCursor        /* OUT: Tokenization cursor */
)
{
        custom_apropos_tokenizer_cursor *c;

        c = calloc(1, sizeof(*c));
        if (c == NULL)
                return SQLITE_NOMEM;

        c->zInput = zInput;
        if (zInput != 0) {
                if (nInput < 0)
                        c->nInput = strlen(zInput);
                else
                        c->nInput = nInput;
        }

        *ppCursor = &c->base;
        return SQLITE_OK;
}

/*
 * Close a tokenization cursor previously opened by a call to
 * aproposPorterOpen() above.
 */
static int
aproposPorterClose(sqlite3_tokenizer_cursor *pCursor)
{
        custom_apropos_tokenizer_cursor *c = (custom_apropos_tokenizer_cursor *) pCursor;
        free(c->zToken);
        free(c);
        return SQLITE_OK;
}

/*
 * Vowel or consonant
 */
static const char cType[] = {
        0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
        1, 1, 1, 2, 1
};

/*
 * isConsonant() and isVowel() determine if their first character in
 * the string they point to is a consonant or a vowel, according
 * to Porter ruls.
 *
 * A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
 * 'Y' is a consonant unless it follows another consonant,
 * in which case it is a vowel.
 *
 * In these routine, the letters are in reverse order.  So the 'y' rule
 * is that 'y' is a consonant unless it is followed by another
 * consonent.
 */
static int isVowel(const char*);

static int
isConsonant(const char *z)
{
        int j;
        char x = *z;
        if (x == 0)
                return 0;
        assert(x >= 'a' && x <= 'z');
        j = cType[x - 'a'];
        if (j < 2)
                return j;
        return z[1] == 0 || isVowel(z + 1);
}

static int
isVowel(const char *z)
{
        int j;
        char x = *z;
        if (x == 0)
                return 0;
        assert(x >= 'a' && x <= 'z');
        j = cType[x - 'a'];
        if (j < 2)
                return 1 - j;
        return isConsonant(z + 1);
}

/*
 * Let any sequence of one or more vowels be represented by V and let
 * C be sequence of one or more consonants.  Then every word can be
 * represented as:
 *
 *           [C] (VC){m} [V]
 *
 * In prose:  A word is an optional consonant followed by zero or
 * vowel-consonant pairs followed by an optional vowel.  "m" is the
 * number of vowel consonant pairs.  This routine computes the value
 * of m for the first i bytes of a word.
 *
 * Return true if the m-value for z is 1 or more.  In other words,
 * return true if z contains at least one vowel that is followed
 * by a consonant.
 *
 * In this routine z[] is in reverse order.  So we are really looking
 * for an instance of a consonant followed by a vowel.
 */
static int
m_gt_0(const char *z)
{
        while (isVowel(z)) {
                z++;
        }
        if (*z == 0)
                return 0;
        while (isConsonant(z)) {
                z++;
        }
        return *z != 0;
}

/* Like mgt0 above except we are looking for a value of m which is
 * exactly 1
 */
static int
m_eq_1(const char *z)
{
        while (isVowel(z)) {
                z++;
        }
        if (*z == 0)
                return 0;
        while (isConsonant(z)) {
                z++;
        }
        if (*z == 0)
                return 0;
        while (isVowel(z)) {
                z++;
        }
        if (*z == 0)
                return 1;
        while (isConsonant(z)) {
                z++;
        }
        return *z == 0;
}

/* Like mgt0 above except we are looking for a value of m>1 instead
 * or m>0
 */
static int
m_gt_1(const char *z)
{
        while (isVowel(z)) {
                z++;
        }
        if (*z == 0)
                return 0;
        while (isConsonant(z)) {
                z++;
        }
        if (*z == 0)
                return 0;
        while (isVowel(z)) {
                z++;
        }
        if (*z == 0)
                return 0;
        while (isConsonant(z)) {
                z++;
        }
        return *z != 0;
}

/*
 * Return TRUE if there is a vowel anywhere within z[0..n-1]
 */
static int
hasVowel(const char *z)
{
        while (isConsonant(z)) {
                z++;
        }
        return *z != 0;
}

/*
 * Return TRUE if the word ends in a double consonant.
 *
 * The text is reversed here. So we are really looking at
 * the first two characters of z[].
 */
static int
doubleConsonant(const char *z)
{
        return isConsonant(z) && z[0] == z[1];
}

/*
 * Return TRUE if the word ends with three letters which
 * are consonant-vowel-consonent and where the final consonant
 * is not 'w', 'x', or 'y'.
 *
 * The word is reversed here.  So we are really checking the
 * first three letters and the first one cannot be in [wxy].
 */
static int
star_oh(const char *z)
{
        return isConsonant(z) &&
            z[0] != 'w' && z[0] != 'x' && z[0] != 'y' &&
            isVowel(z + 1) &&
            isConsonant(z + 2);
}

/*
 * If the word ends with zFrom and xCond() is true for the stem
 * of the word that precedes the zFrom ending, then change the
 * ending to zTo.
 *
 * The input word *pz and zFrom are both in reverse order.  zTo
 * is in normal order.
 *
 * Return TRUE if zFrom matches.  Return FALSE if zFrom does not
 * match.  Not that TRUE is returned even if xCond() fails and
 * no substitution occurs.
 */
static int
stem(
    char **pz,                  /* The word being stemmed (Reversed) */
    const char *zFrom,          /* If the ending matches this... (Reversed) */
    const char *zTo,            /* ... change the ending to this (not reversed) */
    int (*xCond) (const char *) /* Condition that must be true */
)
{
        char *z = *pz;
        while (*zFrom && *zFrom == *z) {
                z++;
                zFrom++;
        }
        if (*zFrom != 0)
                return 0;
        if (xCond && !xCond(z))
                return 1;
        while (*zTo) {
                *(--z) = *(zTo++);
        }
        *pz = z;
        return 1;
}

/*
 * This is the fallback stemmer used when the porter stemmer is
 * inappropriate.  The input word is copied into the output with
 * US-ASCII case folding.  If the input word is too long (more
 * than 20 bytes if it contains no digits or more than 6 bytes if
 * it contains digits) then word is truncated to 20 or 6 bytes
 * by taking 10 or 3 bytes from the beginning and end.
 */
static void
copy_stemmer(const char *zIn, size_t nIn, char *zOut, size_t *pnOut)
{
        size_t i, mx, j;
        int hasDigit = 0;
        for (i = 0; i < nIn; i++) {
                char c = zIn[i];
                if (c >= 'A' && c <= 'Z') {
                        zOut[i] = c - 'A' + 'a';
                } else {
                        if (c >= '0' && c <= '9')
                                hasDigit = 1;
                        zOut[i] = c;
                }
        }
        mx = hasDigit ? 3 : 10;
        if (nIn > mx * 2) {
                for (j = mx, i = nIn - mx; i < nIn; i++, j++) {
                        zOut[j] = zOut[i];
                }
                i = j;
        }
        zOut[i] = 0;
        *pnOut = i;
}


/*
 * Stem the input word zIn[0..nIn-1].  Store the output in zOut.
 * zOut is at least big enough to hold nIn bytes.  Write the actual
 * size of the output word (exclusive of the '\0' terminator) into *pnOut.
 *
 * Any upper-case characters in the US-ASCII character set ([A-Z])
 * are converted to lower case.  Upper-case UTF characters are
 * unchanged.
 *
 * Words that are longer than about 20 bytes are stemmed by retaining
 * a few bytes from the beginning and the end of the word.  If the
 * word contains digits, 3 bytes are taken from the beginning and
 * 3 bytes from the end.  For long words without digits, 10 bytes
 * are taken from each end.  US-ASCII case folding still applies.
 *
 * If the input word contains not digits but does characters not
 * in [a-zA-Z] then no stemming is attempted and this routine just
 * copies the input into the input into the output with US-ASCII
 * case folding.
 *
 * Stemming never increases the length of the word.  So there is
 * no chance of overflowing the zOut buffer.
 */
static void
porter_stemmer(const char *zIn, size_t nIn, char *zOut, size_t *pnOut)
{
        size_t i, j;
        char zReverse[28];
        char *z, *z2;
        if (nIn < 3 || nIn >= sizeof(zReverse) - 7) {
                /* The word is too big or too small for the porter stemmer.
                 * Fallback to the copy stemmer
                 */
                copy_stemmer(zIn, nIn, zOut, pnOut);
                return;
        }

        for (i = 0, j = sizeof(zReverse) - 6; i < nIn; i++, j--) {
                char c = zIn[i];
                if (c >= 'A' && c <= 'Z') {
                        zReverse[j] = c + 'a' - 'A';
                } else if (c >= 'a' && c <= 'z') {
                        zReverse[j] = c;
                } else {
                        /* The use of a character not in [a-zA-Z] means that
                         * we fallback * to the copy stemmer
                         */
                        copy_stemmer(zIn, nIn, zOut, pnOut);
                        return;
                }
        }
        memset(&zReverse[sizeof(zReverse) - 5], 0, 5);
        z = &zReverse[j + 1];


        /* Step 1a */
        if (z[0] == 's') {
                if (
                    !stem(&z, "sess", "ss", 0) &&
                    !stem(&z, "sei", "i", 0) &&
                    !stem(&z, "ss", "ss", 0)
                    ) {
                        z++;
                }
        }
        /* Step 1b */
        z2 = z;
        if (stem(&z, "dee", "ee", m_gt_0)) {
                /* Do nothing.  The work was all in the test */
        } else if (
                    (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
                    && z != z2
            ) {
                if (stem(&z, "ta", "ate", 0) ||
                    stem(&z, "lb", "ble", 0) ||
                    stem(&z, "zi", "ize", 0)) {
                        /* Do nothing.  The work was all in the test */
                } else if (doubleConsonant(z) && (*z != 'l' && *z != 's' && *z != 'z')) {
                        z++;
                } else if (m_eq_1(z) && star_oh(z)) {
                        *(--z) = 'e';
                }
        }
        /* Step 1c */
        if (z[0] == 'y' && hasVowel(z + 1)) {
                z[0] = 'i';
        }
        /* Step 2 */
        switch (z[1]) {
        case 'a':
                if (!stem(&z, "lanoita", "ate", m_gt_0)) {
                        stem(&z, "lanoit", "tion", m_gt_0);
                }
                break;
        case 'c':
                if (!stem(&z, "icne", "ence", m_gt_0)) {
                        stem(&z, "icna", "ance", m_gt_0);
                }
                break;
        case 'e':
                stem(&z, "rezi", "ize", m_gt_0);
                break;
        case 'g':
                stem(&z, "igol", "log", m_gt_0);
                break;
        case 'l':
                if (!stem(&z, "ilb", "ble", m_gt_0)
                    && !stem(&z, "illa", "al", m_gt_0)
                    && !stem(&z, "iltne", "ent", m_gt_0)
                    && !stem(&z, "ile", "e", m_gt_0)
                    ) {
                        stem(&z, "ilsuo", "ous", m_gt_0);
                }
                break;
        case 'o':
                if (!stem(&z, "noitazi", "ize", m_gt_0)
                    && !stem(&z, "noita", "ate", m_gt_0)
                    ) {
                        stem(&z, "rota", "ate", m_gt_0);
                }
                break;
        case 's':
                if (!stem(&z, "msila", "al", m_gt_0)
                    && !stem(&z, "ssenevi", "ive", m_gt_0)
                    && !stem(&z, "ssenluf", "ful", m_gt_0)
                    ) {
                        stem(&z, "ssensuo", "ous", m_gt_0);
                }
                break;
        case 't':
                if (!stem(&z, "itila", "al", m_gt_0)
                    && !stem(&z, "itivi", "ive", m_gt_0)
                    ) {
                        stem(&z, "itilib", "ble", m_gt_0);
                }
                break;
        }

        /* Step 3 */
        switch (z[0]) {
        case 'e':
                if (!stem(&z, "etaci", "ic", m_gt_0)
                    && !stem(&z, "evita", "", m_gt_0)
                    ) {
                        stem(&z, "ezila", "al", m_gt_0);
                }
                break;
        case 'i':
                stem(&z, "itici", "ic", m_gt_0);
                break;
        case 'l':
                if (!stem(&z, "laci", "ic", m_gt_0)) {
                        stem(&z, "luf", "", m_gt_0);
                }
                break;
        case 's':
                stem(&z, "ssen", "", m_gt_0);
                break;
        }

        /* Step 4 */
        switch (z[1]) {
        case 'a':
                if (z[0] == 'l' && m_gt_1(z + 2)) {
                        z += 2;
                }
                break;
        case 'c':
                if (z[0] == 'e' && z[2] == 'n' && (z[3] == 'a' || z[3] == 'e') && m_gt_1(z + 4)) {
                        z += 4;
                }
                break;
        case 'e':
                if (z[0] == 'r' && m_gt_1(z + 2)) {
                        z += 2;
                }
                break;
        case 'i':
                if (z[0] == 'c' && m_gt_1(z + 2)) {
                        z += 2;
                }
                break;
        case 'l':
                if (z[0] == 'e' && z[2] == 'b' && (z[3] == 'a' || z[3] == 'i') && m_gt_1(z + 4)) {
                        z += 4;
                }
                break;
        case 'n':
                if (z[0] == 't') {
                        if (z[2] == 'a') {
                                if (m_gt_1(z + 3)) {
                                        z += 3;
                                }
                        } else if (z[2] == 'e') {
                                if (!stem(&z, "tneme", "", m_gt_1)
                                    && !stem(&z, "tnem", "", m_gt_1)
                                    ) {
                                        stem(&z, "tne", "", m_gt_1);
                                }
                        }
                }
                break;
        case 'o':
                if (z[0] == 'u') {
                        if (m_gt_1(z + 2)) {
                                z += 2;
                        }
                } else if (z[3] == 's' || z[3] == 't') {
                        stem(&z, "noi", "", m_gt_1);
                }
                break;
        case 's':
                if (z[0] == 'm' && z[2] == 'i' && m_gt_1(z + 3)) {
                        z += 3;
                }
                break;
        case 't':
                if (!stem(&z, "eta", "", m_gt_1)) {
                        stem(&z, "iti", "", m_gt_1);
                }
                break;
        case 'u':
                if (z[0] == 's' && z[2] == 'o' && m_gt_1(z + 3)) {
                        z += 3;
                }
                break;
        case 'v':
        case 'z':
                if (z[0] == 'e' && z[2] == 'i' && m_gt_1(z + 3)) {
                        z += 3;
                }
                break;
        }

        /* Step 5a */
        if (z[0] == 'e') {
                if (m_gt_1(z + 1)) {
                        z++;
                } else if (m_eq_1(z + 1) && !star_oh(z + 1)) {
                        z++;
                }
        }
        /* Step 5b */
        if (m_gt_1(z) && z[0] == 'l' && z[1] == 'l') {
                z++;
        }
        /* z[] is now the stemmed word in reverse order.  Flip it back
         * around into forward order and return.
         */
        *pnOut = i = strlen(z);
        zOut[i] = 0;
        while (*z) {
                zOut[--i] = *(z++);
        }
}

/*
 * Based on whether the input word is in the nostem list or not
 * call porter stemmer to stem it, or call copy_stemmer to keep it
 * as it is (copy_stemmer converts simply converts it to lower case)
 * Returns  SQLITE_OK if stemming is successful, an error code for
 * any errors
 */
static int
do_stem(const char *zIn, size_t nIn, char *zOut, size_t *pnOut)
{
        /* Before looking up the word in the hash table, convert it to lower-case */
        char *dupword = malloc(nIn);
        if (dupword == NULL)
                return SQLITE_NOMEM;

        for (size_t i = 0; i < nIn; i++)
                dupword[i] = tolower((unsigned char) zIn[i]);

        size_t idx = nostem_hash(dupword, nIn);
        if (strncmp(nostem[idx], dupword, nIn) == 0 && nostem[idx][nIn] == 0)
                copy_stemmer(zIn, nIn, zOut, pnOut);
        else
                porter_stemmer(zIn, nIn, zOut, pnOut);

        free(dupword);
        return SQLITE_OK;
}


/*
 * Characters that can be part of a token.  We assume any character
 * whose value is greater than 0x80 (any UTF character) can be
 * part of a token.  In other words, delimiters all must have
 * values of 0x7f or lower.
 */
static const char porterIdChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
        0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 5x */
        0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
};

#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30]))

/*
 * Extract the next token from a tokenization cursor.  The cursor must
 * have been opened by a prior call to aproposPorterOpen().
 */
static int
aproposPorterNext(
    sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by aproposPorterOpen */
    const char **pzToken,       /* OUT: *pzToken is the token text */
    int *pnBytes,               /* OUT: Number of bytes in token */
    int *piStartOffset,         /* OUT: Starting offset of token */
    int *piEndOffset,           /* OUT: Ending offset of token */
    int *piPosition             /* OUT: Position integer of token */
)
{
        custom_apropos_tokenizer_cursor *c = (custom_apropos_tokenizer_cursor *) pCursor;
        const char *z = c->zInput;

        while (c->iOffset < c->nInput) {
                size_t iStartOffset, ch;

                /* Scan past delimiter characters */
                while (c->iOffset < c->nInput && isDelim(z[c->iOffset])) {
                        c->iOffset++;
                }

                /* Count non-delimiter characters. */
                iStartOffset = c->iOffset;
                while (c->iOffset < c->nInput && !isDelim(z[c->iOffset])) {
                        c->iOffset++;
                }

                if (c->iOffset > iStartOffset) {
                        size_t n = c->iOffset - iStartOffset;
                        if (n > c->nAllocated) {
                                char *pNew;
                                c->nAllocated = n + 20;
                                pNew = realloc(c->zToken, c->nAllocated);
                                if (!pNew)
                                        return SQLITE_NOMEM;
                                c->zToken = pNew;
                        }

                        size_t temp;
                        int stemStatus = do_stem(&z[iStartOffset], n, c->zToken, &temp);
                        if (stemStatus != SQLITE_OK)
                                return stemStatus;
                        *pnBytes = temp;

                        *pzToken = c->zToken;
                        *piStartOffset = iStartOffset;
                        *piEndOffset = c->iOffset;
                        *piPosition = c->iToken++;
                        return SQLITE_OK;
                }
        }
        return SQLITE_DONE;
}

/*
 * The set of routines that implement the porter-stemmer tokenizer
 */
static const sqlite3_tokenizer_module aproposPorterTokenizerModule = {
        0,
        aproposPorterCreate,
        aproposPorterDestroy,
        aproposPorterOpen,
        aproposPorterClose,
        aproposPorterNext,
        0
};

/*
 * Allocate a new porter tokenizer.  Return a pointer to the new
 * tokenizer in *ppModule
 */
void
get_custom_apropos_tokenizer(sqlite3_tokenizer_module const ** ppModule)
{
        *ppModule = &aproposPorterTokenizerModule;
}