/*
 * Copyright 2002, 2003 Andi Kleen, SuSE Labs.
 *
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file COPYING in the main directory of this archive
 * for more details. No warranty for anything given at all.
 */
#include <linux/linkage.h>
#include <asm/errno.h>
#include <asm/asm.h>

/*
 * Checksum copy with exception handling.
 * On exceptions src_err_ptr or dst_err_ptr is set to -EFAULT and the
 * destination is zeroed.
 *
 * Input
 * rdi  source
 * rsi  destination
 * edx  len (32bit)
 *
 * Output
 * eax  64bit sum. undefined in case of exception.
 *
 * Wrappers need to take care of valid exception sum and zeroing.
 * They also should align source or destination to 8 bytes.
 */

        .macro source
10:
        _ASM_EXTABLE_UA(10b, .Lfault)
        .endm

        .macro dest
20:
        _ASM_EXTABLE_UA(20b, .Lfault)
        .endm

SYM_FUNC_START(csum_partial_copy_generic)
        subq  $5*8, %rsp
        movq  %rbx, 0*8(%rsp)
        movq  %r12, 1*8(%rsp)
        movq  %r14, 2*8(%rsp)
        movq  %r13, 3*8(%rsp)
        movq  %r15, 4*8(%rsp)

        movl  $-1, %eax
        xorl  %r9d, %r9d
        movl  %edx, %ecx
        cmpl  $8, %ecx
        jb    .Lshort

        testb  $7, %sil
        jne   .Lunaligned
.Laligned:
        movl  %ecx, %r12d

        shrq  $6, %r12
        jz      .Lhandle_tail       /* < 64 */

        clc

        /* main loop. clear in 64 byte blocks */
        /* r9: zero, r8: temp2, rbx: temp1, rax: sum, rcx: saved length */
        /* r11: temp3, rdx: temp4, r12 loopcnt */
        /* r10: temp5, r15: temp6, r14 temp7, r13 temp8 */
        .p2align 4
.Lloop:
        source
        movq  (%rdi), %rbx
        source
        movq  8(%rdi), %r8
        source
        movq  16(%rdi), %r11
        source
        movq  24(%rdi), %rdx

        source
        movq  32(%rdi), %r10
        source
        movq  40(%rdi), %r15
        source
        movq  48(%rdi), %r14
        source
        movq  56(%rdi), %r13

30:
        /*
         * No _ASM_EXTABLE_UA; this is used for intentional prefetch on a
         * potentially unmapped kernel address.
         */
        _ASM_EXTABLE(30b, 2f)
        prefetcht0 5*64(%rdi)
2:
        adcq  %rbx, %rax
        adcq  %r8, %rax
        adcq  %r11, %rax
        adcq  %rdx, %rax
        adcq  %r10, %rax
        adcq  %r15, %rax
        adcq  %r14, %rax
        adcq  %r13, %rax

        decl %r12d

        dest
        movq %rbx, (%rsi)
        dest
        movq %r8, 8(%rsi)
        dest
        movq %r11, 16(%rsi)
        dest
        movq %rdx, 24(%rsi)

        dest
        movq %r10, 32(%rsi)
        dest
        movq %r15, 40(%rsi)
        dest
        movq %r14, 48(%rsi)
        dest
        movq %r13, 56(%rsi)

        leaq 64(%rdi), %rdi
        leaq 64(%rsi), %rsi

        jnz     .Lloop

        adcq  %r9, %rax

        /* do last up to 56 bytes */
.Lhandle_tail:
        /* ecx: count, rcx.63: the end result needs to be rol8 */
        movq %rcx, %r10
        andl $63, %ecx
        shrl $3, %ecx
        jz      .Lfold
        clc
        .p2align 4
.Lloop_8:
        source
        movq (%rdi), %rbx
        adcq %rbx, %rax
        decl %ecx
        dest
        movq %rbx, (%rsi)
        leaq 8(%rsi), %rsi /* preserve carry */
        leaq 8(%rdi), %rdi
        jnz     .Lloop_8
        adcq %r9, %rax  /* add in carry */

.Lfold:
        /* reduce checksum to 32bits */
        movl %eax, %ebx
        shrq $32, %rax
        addl %ebx, %eax
        adcl %r9d, %eax

        /* do last up to 6 bytes */
.Lhandle_7:
        movl %r10d, %ecx
        andl $7, %ecx
.L1:                            /* .Lshort rejoins the common path here */
        shrl $1, %ecx
        jz   .Lhandle_1
        movl $2, %edx
        xorl %ebx, %ebx
        clc
        .p2align 4
.Lloop_1:
        source
        movw (%rdi), %bx
        adcl %ebx, %eax
        decl %ecx
        dest
        movw %bx, (%rsi)
        leaq 2(%rdi), %rdi
        leaq 2(%rsi), %rsi
        jnz .Lloop_1
        adcl %r9d, %eax /* add in carry */

        /* handle last odd byte */
.Lhandle_1:
        testb $1, %r10b
        jz    .Lende
        xorl  %ebx, %ebx
        source
        movb (%rdi), %bl
        dest
        movb %bl, (%rsi)
        addl %ebx, %eax
        adcl %r9d, %eax         /* carry */

.Lende:
        testq %r10, %r10
        js  .Lwas_odd
.Lout:
        movq 0*8(%rsp), %rbx
        movq 1*8(%rsp), %r12
        movq 2*8(%rsp), %r14
        movq 3*8(%rsp), %r13
        movq 4*8(%rsp), %r15
        addq $5*8, %rsp
        RET
.Lshort:
        movl %ecx, %r10d
        jmp  .L1
.Lunaligned:
        xorl %ebx, %ebx
        testb $1, %sil
        jne  .Lodd
1:      testb $2, %sil
        je   2f
        source
        movw (%rdi), %bx
        dest
        movw %bx, (%rsi)
        leaq 2(%rdi), %rdi
        subq $2, %rcx
        leaq 2(%rsi), %rsi
        addq %rbx, %rax
2:      testb $4, %sil
        je .Laligned
        source
        movl (%rdi), %ebx
        dest
        movl %ebx, (%rsi)
        leaq 4(%rdi), %rdi
        subq $4, %rcx
        leaq 4(%rsi), %rsi
        addq %rbx, %rax
        jmp .Laligned

.Lodd:
        source
        movb (%rdi), %bl
        dest
        movb %bl, (%rsi)
        leaq 1(%rdi), %rdi
        leaq 1(%rsi), %rsi
        /* decrement, set MSB */
        leaq -1(%rcx, %rcx), %rcx
        rorq $1, %rcx
        shll $8, %ebx
        addq %rbx, %rax
        jmp 1b

.Lwas_odd:
        roll $8, %eax
        jmp .Lout

        /* Exception: just return 0 */
.Lfault:
        xorl %eax, %eax
        jmp  .Lout
SYM_FUNC_END(csum_partial_copy_generic)