/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2009, Intel Corporation
 * All rights reserved.
 */

/*
 *      strlen - calculate the length of string
 */

#include "SYS.h"
#include "proc64_id.h"

#define LABEL(s) .strlen##s

        /*
         * This implementation uses SSE instructions to compare up to 16 bytes
         * at a time looking for the end of string (null char).
         */
        ENTRY(strlen)                   /* (const char *s) */
        mov     %rdi, %rsi              /* keep original %rdi value */
        mov     %rsi, %rcx
        pxor    %xmm0, %xmm0            /* 16 null chars */
        and     $15, %rcx
        jz      LABEL(align16_loop)     /* string is 16 byte aligned */

        /*
         * Unaligned case. Round down to 16-byte boundary before comparing
         * 16 bytes for a null char. The code then compensates for any extra chars
         * preceding the start of the string.
         */
LABEL(unalign16):
        and     $0xfffffffffffffff0, %rsi

        pcmpeqb (%rsi), %xmm0
        lea     16(%rdi), %rsi
        pmovmskb %xmm0, %edx

        shr     %cl, %edx               /* Compensate for bytes preceding the string */
        test    %edx, %edx
        jnz     LABEL(exit)
        sub     %rcx, %rsi              /* no null, adjust to next 16-byte boundary */
        pxor    %xmm0, %xmm0            /* clear xmm0, may have been changed... */

        .p2align 4
LABEL(align16_loop):                    /* 16 byte aligned */
        pcmpeqb (%rsi), %xmm0           /* look for null bytes */
        pmovmskb %xmm0, %edx            /* move each byte mask of %xmm0 to edx */

        add     $16, %rsi               /* prepare to search next 16 bytes */
        test    %edx, %edx              /* if no null byte, %edx must be 0 */
        jnz     LABEL(exit)             /* found a null */

        pcmpeqb (%rsi), %xmm0
        pmovmskb %xmm0, %edx
        add     $16, %rsi
        test    %edx, %edx
        jnz     LABEL(exit)

        pcmpeqb (%rsi), %xmm0
        pmovmskb %xmm0, %edx
        add     $16, %rsi
        test    %edx, %edx
        jnz     LABEL(exit)

        pcmpeqb (%rsi), %xmm0
        pmovmskb %xmm0, %edx
        add     $16, %rsi
        test    %edx, %edx
        jz      LABEL(align16_loop)

        .p2align 4
LABEL(exit):
        neg     %rdi
        /*
         * Check to see if BSF is fast on this processor. If not, use a different
         * exit tail to find first bit set indicating null byte match.
         */
        testl   $USE_BSF, .memops_method(%rip)
        jz      LABEL(AMD_exit)

        lea     -16(%rdi, %rsi), %rax   /* calculate exact offset */
        bsf     %edx, %ecx              /* Least significant 1 bit is index of null */
        lea     (%rax, %rcx),%rax
        ret

        /*
         * This exit tail does not use the bsf instruction.
         */
        .p2align 4
LABEL(AMD_exit):
        lea     -16(%rdi, %rsi), %rax
        test    %dl, %dl
        jz      LABEL(exit_high)
        test    $0x01, %dl
        jnz     LABEL(exit_tail0)

        test    $0x02, %dl
        jnz     LABEL(exit_tail1)

        .p2align 4
        test    $0x04, %dl
        jnz     LABEL(exit_tail2)

        test    $0x08, %dl
        jnz     LABEL(exit_tail3)

        test    $0x10, %dl
        jnz     LABEL(exit_tail4)

        test    $0x20, %dl
        jnz     LABEL(exit_tail5)

        test    $0x40, %dl
        jnz     LABEL(exit_tail6)
        add     $7, %rax
        ret

        .p2align 4
LABEL(exit_high):
        add     $8, %rax
        test    $0x01, %dh
        jnz     LABEL(exit_tail0)

        test    $0x02, %dh
        jnz     LABEL(exit_tail1)

        test    $0x04, %dh
        jnz     LABEL(exit_tail2)

        test    $0x08, %dh
        jnz     LABEL(exit_tail3)

        test    $0x10, %dh
        jnz     LABEL(exit_tail4)

        test    $0x20, %dh
        jnz     LABEL(exit_tail5)

        test    $0x40, %dh
        jnz     LABEL(exit_tail6)
        add     $7, %rax
        ret

        .p2align 4
LABEL(exit_tail0):
        xor     %ecx, %ecx
        ret

        .p2align 4
LABEL(exit_tail1):
        add     $1, %rax
        ret

        .p2align 4
LABEL(exit_tail2):
        add     $2, %rax
        ret

        .p2align 4
LABEL(exit_tail3):
        add     $3, %rax
        ret

        .p2align 4
LABEL(exit_tail4):
        add     $4, %rax
        ret

        .p2align 4
LABEL(exit_tail5):
        add     $5, %rax
        ret

        .p2align 4
LABEL(exit_tail6):
        add     $6, %rax
        ret
        SET_SIZE(strlen)