/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2009, Intel Corporation
 * All rights reserved.
 */

/*
 *      str[n]cmp - compare chars between two string
 */

#include "SYS.h"
#include "proc64_id.h"

#define LABEL(s) .strcmp##s

#ifdef USE_AS_STRNCMP
        /*
         * Since the counter, %r11, is unsigned, we branch to strcmp_exitz
         * if the new counter > the old one or is 0.
         */
#define UPDATE_STRNCMP_COUNTER                          \
        /* calculate left number to compare */          \
        lea     -16(%rcx, %r11), %r9;                   \
        cmp     %r9, %r11;                              \
        jb      LABEL(strcmp_exitz);                    \
        test    %r9, %r9;                               \
        je      LABEL(strcmp_exitz);                    \
        mov     %r9, %r11
#else
#define UPDATE_STRNCMP_COUNTER
#endif

        /*
         * This implementation uses SSE to compare up to 16 bytes at a time.
         */
#ifdef USE_AS_STRNCMP
        ENTRY(strncmp)
        test    %rdx, %rdx
        je      LABEL(strcmp_exitz)
        mov     %rdx, %r11
#else
        ENTRY(strcmp)                   /* (const char *, const char *) */
#endif
        mov     %esi, %ecx
        mov     %edi, %eax
        and     $0x3f, %rcx             /* rsi alignment in cache line */
        and     $0x3f, %rax             /* rdi alignment in cache line */
        cmp     $0x30, %ecx
        ja      LABEL(crosscache)       /* rsi: 16-byte load will cross cache line */
        cmp     $0x30, %eax
        ja      LABEL(crosscache)       /* rdi: 16-byte load will cross cache line */
        movlpd  (%rdi), %xmm1
        movlpd  (%rsi), %xmm2
        movhpd  8(%rdi), %xmm1
        movhpd  8(%rsi), %xmm2
        pxor    %xmm0, %xmm0            /* clear %xmm0 for null char checks */
        pcmpeqb %xmm1, %xmm0            /* Any null chars? */
        pcmpeqb %xmm2, %xmm1            /* compare first 16 bytes for equality */
        psubb   %xmm0, %xmm1            /* packed sub of comparison results*/
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx           /* if first 16 bytes are same, edx == 0xffff */
        jnz     LABEL(less16bytes)      /* If not, found mismatch or null char */
#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)     /* finish comparision */
#endif
        add     $16, %rsi               /* prepare to search next 16 bytes */
        add     $16, %rdi               /* prepare to search next 16 bytes */

        /*
         * Determine rdi and rsi string offsets from 16-byte alignment.
         * Use relative offset difference between the two to determine which case
         * below to use.
         */
        .p2align 4
LABEL(crosscache):
        and     $0xfffffffffffffff0, %rsi       /* force %rsi to be 16 byte aligned */
        and     $0xfffffffffffffff0, %rdi       /* force %rdi to be 16 byte aligned */
        mov     $0xffff, %edx                   /* for equivalent offset */
        xor     %r8d, %r8d
        and     $0xf, %ecx                      /* offset of rsi */
        and     $0xf, %eax                      /* offset of rdi */
        cmp     %eax, %ecx
        je      LABEL(ashr_0)                   /* both strings have the same alignment */
        ja      LABEL(bigger)
        mov     %edx, %r8d                      /* r8d is offset flag for exit tail */
        xchg    %ecx, %eax
        xchg    %rsi, %rdi
LABEL(bigger):
        mov     %rcx, %r9
        sub     %rax, %r9
        lea     LABEL(unaligned_table)(%rip), %r10
        movslq  (%r10, %r9, 4), %r9
        lea     (%r10, %r9), %r10
        jmp     *%r10                           /* jump to corresponding case */

/*
 * ashr_0 handles the following cases:
 *      str1 offset = str2 offset
 */
        .p2align 4
LABEL(ashr_0):
        movdqa  (%rsi), %xmm1
        pxor    %xmm0, %xmm0                    /* clear %xmm0 for null char check */
        pcmpeqb %xmm1, %xmm0                    /* Any null chars? */
        pcmpeqb (%rdi), %xmm1                   /* compare 16 bytes for equality */
        psubb   %xmm0, %xmm1                    /* packed sub of comparison results*/
        pmovmskb %xmm1, %r9d
        shr     %cl, %edx                       /* adjust 0xffff for offset */
        shr     %cl, %r9d                       /* adjust for 16-byte offset */
        sub     %r9d, %edx
        /*
         * edx must be the same with r9d if in left byte (16-rcx) is equal to
         * the start from (16-rax) and no null char was seen.
         */
        jne     LABEL(less32bytes)              /* mismatch or null char */
        UPDATE_STRNCMP_COUNTER
        mov     $16, %rcx
        mov     $16, %r9
        pxor    %xmm0, %xmm0                    /* clear xmm0, may have changed above */

        /*
         * Now both strings are aligned at 16-byte boundary. Loop over strings
         * checking 32-bytes per iteration.
         */
        .p2align 4
LABEL(loop_ashr_0):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)             /* mismatch or null char seen */

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        add     $16, %rcx
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        add     $16, %rcx
        jmp     LABEL(loop_ashr_0)

/*
 * ashr_1 handles the following cases:
 *      abs(str1 offset - str2 offset) = 15
 */
        .p2align 4
LABEL(ashr_1):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0            /* Any null chars? */
        pslldq  $15, %xmm2              /* shift first string to align with second */
        pcmpeqb %xmm1, %xmm2            /* compare 16 bytes for equality */
        psubb   %xmm0, %xmm2            /* packed sub of comparison results*/
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx               /* adjust 0xffff for offset */
        shr     %cl, %r9d               /* adjust for 16-byte offset */
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)      /* mismatch or null char seen */
        movdqa  (%rdi), %xmm3
        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx               /* index for loads */
        mov     $1, %r9d                /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     1(%rdi), %r10
        and     $0xfff, %r10            /* offset into 4K page */
        sub     $0x1000, %r10           /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_1):
        add     $16, %r10
        jg      LABEL(nibble_ashr_1)    /* cross page boundary */

LABEL(gobble_ashr_1):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4             /* store for next cycle */

        psrldq  $1, %xmm3
        pslldq  $15, %xmm2
        por     %xmm3, %xmm2            /* merge into one 16byte value */

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_1)    /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4            /* store for next cycle */

        psrldq  $1, %xmm3
        pslldq  $15, %xmm2
        por     %xmm3, %xmm2            /* merge into one 16byte value */

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_1)

        /*
         * Nibble avoids loads across page boundary. This is to avoid a potential
         * access into unmapped memory.
         */
        .p2align 4
LABEL(nibble_ashr_1):
        psrldq  $1, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x7fff, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $15, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_1)

/*
 * ashr_2 handles the following cases:
 *      abs(str1 offset - str2 offset) = 14
 */
        .p2align 4
LABEL(ashr_2):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $14, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3
        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $2, %r9d        /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     2(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_2):
        add     $16, %r10
        jg      LABEL(nibble_ashr_2)

LABEL(gobble_ashr_2):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $2, %xmm3
        pslldq  $14, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_2)    /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $2, %xmm3
        pslldq  $14, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_2)

        .p2align 4
LABEL(nibble_ashr_2):
        psrldq  $2, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x3fff, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $14, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_2)

/*
 * ashr_3 handles the following cases:
 *      abs(str1 offset - str2 offset) = 13
 */
        .p2align 4
LABEL(ashr_3):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $13, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $3, %r9d        /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     3(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_3):
        add     $16, %r10
        jg      LABEL(nibble_ashr_3)

LABEL(gobble_ashr_3):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $3, %xmm3
        pslldq  $13, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_3)    /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $3, %xmm3
        pslldq  $13, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_3)

        .p2align 4
LABEL(nibble_ashr_3):
        psrldq  $3, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x1fff, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $13, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_3)

/*
 * ashr_4 handles the following cases:
 *      abs(str1 offset - str2 offset) = 12
 */
        .p2align 4
LABEL(ashr_4):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $12, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $4, %r9d        /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     4(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_4):
        add     $16, %r10
        jg      LABEL(nibble_ashr_4)

LABEL(gobble_ashr_4):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $4, %xmm3
        pslldq  $12, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_4)    /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $4, %xmm3
        pslldq  $12, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_4)

        .p2align 4
LABEL(nibble_ashr_4):
        psrldq  $4, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x0fff, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $12, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_4)

/*
 * ashr_5 handles the following cases:
 *      abs(str1 offset - str2 offset) = 11
 */
        .p2align 4
LABEL(ashr_5):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $11, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $5, %r9d        /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     5(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_5):
        add     $16, %r10
        jg      LABEL(nibble_ashr_5)

LABEL(gobble_ashr_5):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $5, %xmm3
        pslldq  $11, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_5)    /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $5, %xmm3
        pslldq  $11, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_5)

        .p2align 4
LABEL(nibble_ashr_5):
        psrldq  $5, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x07ff, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $11, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_5)

/*
 * ashr_6 handles the following cases:
 *      abs(str1 offset - str2 offset) = 10
 */
        .p2align 4
LABEL(ashr_6):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $10, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $6, %r9d        /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     6(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_6):
        add     $16, %r10
        jg      LABEL(nibble_ashr_6)

LABEL(gobble_ashr_6):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $6, %xmm3
        pslldq  $10, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_6)    /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $6, %xmm3
        pslldq  $10, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_6)

        .p2align 4
LABEL(nibble_ashr_6):
        psrldq  $6, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x03ff, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $10, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_6)

/*
 * ashr_7 handles the following cases:
 *      abs(str1 offset - str2 offset) = 9
 */
        .p2align 4
LABEL(ashr_7):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $9, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $7, %r9d        /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     7(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_7):
        add     $16, %r10
        jg      LABEL(nibble_ashr_7)

LABEL(gobble_ashr_7):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $7, %xmm3
        pslldq  $9, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_7)    /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $7, %xmm3
        pslldq  $9, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_7)

        .p2align 4
LABEL(nibble_ashr_7):
        psrldq  $7, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x01ff, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $9, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_7)

/*
 * ashr_8 handles the following cases:
 *      abs(str1 offset - str2 offset) = 8
 */
        .p2align 4
LABEL(ashr_8):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $8, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $8, %r9d        /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     8(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_8):
        add     $16, %r10
        jg      LABEL(nibble_ashr_8)

LABEL(gobble_ashr_8):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $8, %xmm3
        pslldq  $8, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_8)    /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $8, %xmm3
        pslldq  $8, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_8)

        .p2align 4
LABEL(nibble_ashr_8):
        psrldq  $8, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x00ff, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $8, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_8)

/*
 * ashr_9 handles the following cases:
 *      abs(str1 offset - str2 offset) = 7
 */
        .p2align 4
LABEL(ashr_9):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $7, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $9, %r9d        /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     9(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_9):
        add     $16, %r10
        jg      LABEL(nibble_ashr_9)

LABEL(gobble_ashr_9):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $9, %xmm3
        pslldq  $7, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_9)    /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $9, %xmm3
        pslldq  $7, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3            /* store for next cycle */
        jmp     LABEL(loop_ashr_9)

        .p2align 4
LABEL(nibble_ashr_9):
        psrldq  $9, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x007f, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $7, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_9)

/*
 * ashr_10 handles the following cases:
 *      abs(str1 offset - str2 offset) = 6
 */
        .p2align 4
LABEL(ashr_10):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $6, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $10, %r9d       /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     10(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_10):
        add     $16, %r10
        jg      LABEL(nibble_ashr_10)

LABEL(gobble_ashr_10):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $10, %xmm3
        pslldq  $6, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_10)   /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $10, %xmm3
        pslldq  $6, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_10)

        .p2align 4
LABEL(nibble_ashr_10):
        psrldq  $10, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x003f, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $6, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_10)

/*
 * ashr_11 handles the following cases:
 *      abs(str1 offset - str2 offset) = 5
 */
        .p2align 4
LABEL(ashr_11):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $5, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $11, %r9d       /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     11(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_11):
        add     $16, %r10
        jg      LABEL(nibble_ashr_11)

LABEL(gobble_ashr_11):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $11, %xmm3
        pslldq  $5, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_11)   /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $11, %xmm3
        pslldq  $5, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_11)

        .p2align 4
LABEL(nibble_ashr_11):
        psrldq  $11, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x001f, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $5, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_11)

/*
 * ashr_12 handles the following cases:
 *      abs(str1 offset - str2 offset) = 4
 */
        .p2align 4
LABEL(ashr_12):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $4, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $12, %r9d       /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     12(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_12):
        add     $16, %r10
        jg      LABEL(nibble_ashr_12)

LABEL(gobble_ashr_12):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $12, %xmm3
        pslldq  $4, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_12)   /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $12, %xmm3
        pslldq  $4, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_12)

        .p2align 4
LABEL(nibble_ashr_12):
        psrldq  $12, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x000f, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $4, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_12)

/*
 * ashr_13 handles the following cases:
 *      abs(str1 offset - str2 offset) = 3
 */
        .p2align 4
LABEL(ashr_13):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $3, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $13, %r9d       /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     13(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_13):
        add     $16, %r10
        jg      LABEL(nibble_ashr_13)

LABEL(gobble_ashr_13):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $13, %xmm3
        pslldq  $3, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_13)   /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $13, %xmm3
        pslldq  $3, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_13)

        .p2align 4
LABEL(nibble_ashr_13):
        psrldq  $13, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x0007, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $3, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_13)

/*
 * ashr_14 handles the following cases:
 *      abs(str1 offset - str2 offset) = 2
 */
        .p2align 4
LABEL(ashr_14):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $2, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)
        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $14, %r9d       /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     14(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_14):
        add     $16, %r10
        jg      LABEL(nibble_ashr_14)

LABEL(gobble_ashr_14):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $14, %xmm3
        pslldq  $2, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_14)   /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $14, %xmm3
        pslldq  $2, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_14)

        .p2align 4
LABEL(nibble_ashr_14):
        psrldq  $14, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x0003, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $2, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_14)

/*
 * ashr_15 handles the following cases:
 *      abs(str1 offset - str2 offset) = 1
 */
        .p2align 4
LABEL(ashr_15):
        pxor    %xmm0, %xmm0
        movdqa  (%rdi), %xmm2
        movdqa  (%rsi), %xmm1
        pcmpeqb %xmm1, %xmm0
        pslldq  $1, %xmm2
        pcmpeqb %xmm1, %xmm2
        psubb   %xmm0, %xmm2
        pmovmskb %xmm2, %r9d
        shr     %cl, %edx
        shr     %cl, %r9d
        sub     %r9d, %edx
        jnz     LABEL(less32bytes)

        movdqa  (%rdi), %xmm3

        UPDATE_STRNCMP_COUNTER

        pxor    %xmm0, %xmm0
        mov     $16, %rcx       /* index for loads */
        mov     $15, %r9d       /* rdi bytes already examined. Used in exit code */
        /*
         * Setup %r10 value allows us to detect crossing a page boundary.
         * When %r10 goes positive we are crossing a page boundary and
         * need to do a nibble.
         */
        lea     15(%rdi), %r10
        and     $0xfff, %r10    /* offset into 4K page */
        sub     $0x1000, %r10   /* subtract 4K pagesize */
        movdqa  %xmm3, %xmm4

        .p2align 4
LABEL(loop_ashr_15):
        add     $16, %r10
        jg      LABEL(nibble_ashr_15)

LABEL(gobble_ashr_15):
        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $15, %xmm3
        pslldq  $1, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3

        add     $16, %r10
        jg      LABEL(nibble_ashr_15)   /* cross page boundary */

        movdqa  (%rsi, %rcx), %xmm1
        movdqa  (%rdi, %rcx), %xmm2
        movdqa  %xmm2, %xmm4

        psrldq  $15, %xmm3
        pslldq  $1, %xmm2
        por     %xmm3, %xmm2

        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm2, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0xffff, %edx
        jnz     LABEL(exit)

#ifdef USE_AS_STRNCMP
        sub     $16, %r11
        jbe     LABEL(strcmp_exitz)
#endif

        add     $16, %rcx
        movdqa  %xmm4, %xmm3
        jmp     LABEL(loop_ashr_15)

        .p2align 4
LABEL(nibble_ashr_15):
        psrldq  $15, %xmm4
        movdqa  (%rsi, %rcx), %xmm1
        pcmpeqb %xmm1, %xmm0
        pcmpeqb %xmm4, %xmm1
        psubb   %xmm0, %xmm1
        pmovmskb %xmm1, %edx
        sub     $0x0001, %edx
        jnz     LABEL(exit)
#ifdef USE_AS_STRNCMP
        cmp     $1, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        pxor    %xmm0, %xmm0
        sub     $0x1000, %r10           /* subtract 4K from %r10 */
        jmp     LABEL(gobble_ashr_15)

        .p2align 4
LABEL(exit):
        lea     -16(%r9, %rcx), %rax    /* locate the exact offset for rdi */
LABEL(less32bytes):
        lea     (%rdi, %rax), %rdi      /* locate the exact address for first operand(rdi) */
        lea     (%rsi, %rcx), %rsi      /* locate the exact address for second operand(rsi) */
        test    %r8d, %r8d
        jz      LABEL(ret)
        xchg    %rsi, %rdi              /* recover original order according to flag(%r8d) */

        .p2align 4
LABEL(ret):
LABEL(less16bytes):
        /*
         * Check to see if BSF is fast on this processor. If not, use a different
         * exit tail.
         */
        testl   $USE_BSF,.memops_method(%rip)
        jz      LABEL(AMD_exit)
        bsf     %rdx, %rdx              /* find and store bit index in %rdx */

#ifdef USE_AS_STRNCMP
        sub     %rdx, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        xor     %ecx, %ecx              /* clear %ecx */
        xor     %eax, %eax              /* clear %eax */

        movb    (%rsi, %rdx), %cl
        movb    (%rdi, %rdx), %al

        sub     %ecx, %eax
        ret

#ifdef USE_AS_STRNCMP
LABEL(strcmp_exitz):
        xor     %eax, %eax
        ret
#endif

        /*
         * This exit tail does not use the bsf instruction.
         */
        .p2align 4
LABEL(AMD_exit):
        test    %dl, %dl
        jz      LABEL(next_8_bytes)

        test    $0x01, %dl
        jnz     LABEL(Byte0)

        test    $0x02, %dl
        jnz     LABEL(Byte1)

        test    $0x04, %dl
        jnz     LABEL(Byte2)

        test    $0x08, %dl
        jnz     LABEL(Byte3)

        test    $0x10, %dl
        jnz     LABEL(Byte4)

        test    $0x20, %dl
        jnz     LABEL(Byte5)

        test    $0x40, %dl
        jnz     LABEL(Byte6)

#ifdef USE_AS_STRNCMP
        sub     $7, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        movzx   7(%rsi), %ecx
        movzx   7(%rdi), %eax

        sub     %ecx, %eax
        ret

        .p2align 4
LABEL(Byte0):
        /*
         * never need to handle byte 0 for strncmpy
#ifdef USE_AS_STRNCMP
        sub     $0, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        */
        movzx   (%rsi), %ecx
        movzx   (%rdi), %eax

        sub     %ecx, %eax
        ret

        .p2align 4
LABEL(Byte1):

#ifdef USE_AS_STRNCMP
        sub     $1, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        movzx   1(%rsi), %ecx
        movzx   1(%rdi), %eax

        sub     %ecx, %eax
        ret

        .p2align 4
LABEL(Byte2):

#ifdef USE_AS_STRNCMP
        sub     $2, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        movzx   2(%rsi), %ecx
        movzx   2(%rdi), %eax

        sub     %ecx, %eax
        ret

        .p2align 4
LABEL(Byte3):

#ifdef USE_AS_STRNCMP
        sub     $3, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        movzx   3(%rsi), %ecx
        movzx   3(%rdi), %eax

        sub     %ecx, %eax
        ret

        .p2align 4
LABEL(Byte4):

#ifdef USE_AS_STRNCMP
        sub     $4, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        movzx   4(%rsi), %ecx
        movzx   4(%rdi), %eax

        sub     %ecx, %eax
        ret

        .p2align 4
LABEL(Byte5):

#ifdef USE_AS_STRNCMP
        sub     $5, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        movzx   5(%rsi), %ecx
        movzx   5(%rdi), %eax

        sub     %ecx, %eax
        ret

        .p2align 4
LABEL(Byte6):

#ifdef USE_AS_STRNCMP
        sub     $6, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        movzx   6(%rsi), %ecx
        movzx   6(%rdi), %eax

        sub     %ecx, %eax
        ret

        .p2align 4
LABEL(next_8_bytes):
        add     $8, %rdi
        add     $8, %rsi
#ifdef USE_AS_STRNCMP
        sub     $8, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        test    $0x01, %dh
        jnz     LABEL(Byte0)

        test    $0x02, %dh
        jnz     LABEL(Byte1)

        test    $0x04, %dh
        jnz     LABEL(Byte2)

        test    $0x08, %dh
        jnz     LABEL(Byte3)

        test    $0x10, %dh
        jnz     LABEL(Byte4)

        test    $0x20, %dh
        jnz     LABEL(Byte5)

        test    $0x40, %dh
        jnz     LABEL(Byte6)

#ifdef USE_AS_STRNCMP
        sub     $7, %r11
        jbe     LABEL(strcmp_exitz)
#endif
        movzx   7(%rsi), %ecx
        movzx   7(%rdi), %eax

        sub     %ecx, %eax
        ret

        .pushsection .rodata
        .p2align 4
LABEL(unaligned_table):
        .int    LABEL(ashr_0) - LABEL(unaligned_table)
        .int    LABEL(ashr_15) - LABEL(unaligned_table)
        .int    LABEL(ashr_14) - LABEL(unaligned_table)
        .int    LABEL(ashr_13) - LABEL(unaligned_table)
        .int    LABEL(ashr_12) - LABEL(unaligned_table)
        .int    LABEL(ashr_11) - LABEL(unaligned_table)
        .int    LABEL(ashr_10) - LABEL(unaligned_table)
        .int    LABEL(ashr_9) - LABEL(unaligned_table)
        .int    LABEL(ashr_8) - LABEL(unaligned_table)
        .int    LABEL(ashr_7) - LABEL(unaligned_table)
        .int    LABEL(ashr_6) - LABEL(unaligned_table)
        .int    LABEL(ashr_5) - LABEL(unaligned_table)
        .int    LABEL(ashr_4) - LABEL(unaligned_table)
        .int    LABEL(ashr_3) - LABEL(unaligned_table)
        .int    LABEL(ashr_2) - LABEL(unaligned_table)
        .int    LABEL(ashr_1) - LABEL(unaligned_table)
        .popsection
#ifdef USE_AS_STRNCMP
        SET_SIZE(strncmp)
#else
        SET_SIZE(strcmp)                /* (const char *, const char *) */
#endif