/*- * Copyright (c) 2012 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Matt Thomas of 3am Software Foundry. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include <machine/asm.h> RCSID("$NetBSD: cpu_in_cksum_buffer.S,v 1.8 2012/12/23 13:24:22 matt Exp $") /* * Special note: * The use of cmp is avoided so that APSR.C (carry) is never overwritten. */ #ifdef _ARM_ARCH_DWORD_OK #define LOAD_DWORD_INTO_R4(r) ldrd r4, [r], #8 #define LOAD_DWORD_INTO_R6(r) ldrd r6, [r], #8 #else #define LOAD_DWORD_INTO_R4(r) ldmia r!, {r4-r5} #define LOAD_DWORD_INTO_R6(r) ldmia r!, {r6-r7} #endif #define RLOFFSET r8 /* register for leading offset */ #define RTMASK r9 /* register for trailing mask */ #if defined(__ARMEL__) || !defined(_ARM_ARCH_DWORD_OK) #define RLO r4 #define RHI r5 #else #define RLO r5 #define RHI r4 #endif /* * uint16_t cpu_in_cksum_buffer(const void *, size_t, uint32_t initial_csum); */ ENTRY(cpu_in_cksum_buffer) #if defined(_ARM_ARCH_DWORD_OK) && !defined(__OPTIMIZE_SIZE__) pld [r0] /* prefetch the first data */ #endif adds ip, r2, #0 /* initialize accumulator/clear carry */ teq r1, #0 /* did we get passed a zero length? */ beq .Lfold /* fold the checksum */ add r2, r0, r1 /* point r2 just past end */ push {r4-r5,RLOFFSET,RTMASK} /* save registers */ mvn RTMASK, #0 /* initialize trailing mask */ ands r3, r2, #3 /* limit to a word */ beq 1f /* no trailing bytes? */ /* * This buffer doesn't end on a word boundary so create a mask * to discard the unneeded bytes in the last word and then round * up the length and ending address to a word boundary. */ rsb r3, r3, #4 /* find out how many bytes to clear */ add r2, r2, r3 /* align to word boundary */ add r1, r1, r3 /* align to word boundary */ mov r3, r3, lsl #3 /* bytes -> bits */ #ifdef __ARMEL__ mov RTMASK, RTMASK, lsr r3 /* replace with zero bits */ #else mov RTMASK, RTMASK, lsl r3 /* replace with zero bits */ #endif 1: ands RLOFFSET, r0, #7 /* test for dword alignment */ bne .Ldword_misaligned /* no, fixup non dword aligned */ /* * If the (now rounded up) length is 4, then only bit 2 will be set. * So if we clear that bit and the result is 0, then the length must * have been 4. */ bics RLO, r1, #4 /* more than 1 word (and zero RLO)? */ beq .Lfinal_word_load /* no, just load final word */ LOAD_DWORD_INTO_R4(r0) /* load first dword */ #if defined(_ARM_ARCH_DWORD_OK) && !defined(__OPTIMIZE_SIZE__) pld [r0, #32] /* prefetch data */ #endif .p2align 3 .Ldword_aligned_noload: sub r1, r2, r0 /* how much is remaining? */ bics r3, r1, #15 /* at least 16 bytes to do? */ beq .Lfinal_words /* no, but we have at least 1 word */ push {r6-r7} #if !defined(__OPTIMIZE_SIZE__) tst r1, #16 bne .Lloop16 tst r1, #32 bne .Lloop32 tst r1, #64 bne .Lloop64 .Lloop128: /* 8 qwords left */ LOAD_DWORD_INTO_R6(r0) /* 16 dwords left */ adcs ip, ip, r4 adcs ip, ip, r5 LOAD_DWORD_INTO_R4(r0) /* 15 dwords left */ adcs ip, ip, r6 adcs ip, ip, r7 LOAD_DWORD_INTO_R6(r0) /* 14 dwords left */ adcs ip, ip, r4 adcs ip, ip, r5 LOAD_DWORD_INTO_R4(r0) /* 13 dwords left */ adcs ip, ip, r6 adcs ip, ip, r7 LOAD_DWORD_INTO_R6(r0) /* 12 dwords left */ adcs ip, ip, r4 adcs ip, ip, r5 LOAD_DWORD_INTO_R4(r0) /* 11 dwords left */ adcs ip, ip, r6 adcs ip, ip, r7 LOAD_DWORD_INTO_R6(r0) /* 10 dwords left */ adcs ip, ip, r4 adcs ip, ip, r5 LOAD_DWORD_INTO_R4(r0) /* 9 dwords left */ adcs ip, ip, r6 adcs ip, ip, r7 .Lloop64: /* 4 qwords left */ LOAD_DWORD_INTO_R6(r0) /* 8 dwords left */ adcs ip, ip, r4 adcs ip, ip, r5 LOAD_DWORD_INTO_R4(r0) /* 7 dwords left */ adcs ip, ip, r6 adcs ip, ip, r7 LOAD_DWORD_INTO_R6(r0) /* 6 dwords left */ adcs ip, ip, r4 adcs ip, ip, r5 LOAD_DWORD_INTO_R4(r0) /* 5 dwords left */ adcs ip, ip, r6 adcs ip, ip, r7 .Lloop32: /* 2 qwords left */ LOAD_DWORD_INTO_R6(r0) /* 4 dwords left */ adcs ip, ip, r4 adcs ip, ip, r5 LOAD_DWORD_INTO_R4(r0) /* 3 dwords left */ adcs ip, ip, r6 adcs ip, ip, r7 #endif .Lloop16: /* 1 qword left */ LOAD_DWORD_INTO_R6(r0) /* 2 dwords left */ adcs ip, ip, r4 adcs ip, ip, r5 LOAD_DWORD_INTO_R4(r0) /* 1 dwords left */ adcs ip, ip, r6 adcs ip, ip, r7 sub r1, r2, r0 /* how much is remaining? */ #if defined(__OPTIMIZE_SIZE__) bics r3, r1, #15 /* do we have at least 1 qword left? */ bne .Lloop16 #else bics r3, r1, #127 /* >= 8 qwords left? */ bne .Lloop128 tst r1, #64 /* >= 4 qwords left? */ bne .Lloop64 tst r1, #32 /* >= 2 qwords left? */ bne .Lloop32 bics r3, r1, #15 /* >= 1 qwords left? */ bne .Lloop16 /* see which of */ #endif pop {r6-r7} teq r1, #0 /* how much left?? */ beq .Ladd_final_dword /* = 0? do the final add */ .Lfinal_words: /* * We have 1 to 3 words left to load. */ tst r1, #8 /* at least one dword (5+ bytes)? */ beq .Lfinal_word_load /* no, deal with the final word. */ /* * We have at least 8 bytes left so accumulate the pending dword * and then load the next dword. */ adcs ip, ip, r4 adcs ip, ip, r5 LOAD_DWORD_INTO_R4(r0) /* * At this point r1 is either 8 or 12 so we can just clear bit 3 * to see if we have one more word to read. */ bics r1, r1, #8 /* subtract dword from length */ beq .Ladd_final_dword /* = 0? do the final add */ .Lfinal_word_load: /* * Finally we are at the word to load. */ adcs ip, ip, RHI /* accumulate RHI */ ldr RHI, [r0] /* load last word into RHI */ .Ladd_final_dword: adcs ip, ip, RLO /* add RLO to accumulator */ .Ladd_final_word: and RHI, RHI, RTMASK /* apply trailing mask to RHI */ adcs ip, ip, RHI /* add RHI to accumulator */ /* * Fall into fold. */ tst RLOFFSET, #1 /* was starting address odd? */ movne ip, ip, ror #8 /* yes, compensate */ pop {r4-r5,RLOFFSET,RTMASK} /* we don't need these anymore */ .Lfold: /* * We now have the 33-bit result in <carry>, ip. Pull in the * standard folding code. */ #include "cpu_in_cksum_fold.S" .Ldword_misaligned: #ifdef _ARM_ARCH_DWORD_OK pld [r0, #32] /* preload next cacheline */ #endif mvn r3, #0 /* initialize leading mask */ tst RLOFFSET, #3 /* are exactly word aligned? */ beq .Lword_aligned /* yes, then just load 1 word */ /* * We aren't even word aligned so we have to make the start address * word aligned and generate a mask to clear the leading bytes. */ bic r0, r0, #3 /* make start address word aligned */ and r4, RLOFFSET, #3 /* limit to a single word length */ mov r4, r4, lsl #3 /* bytes -> bits */ #ifdef __ARMEL__ mov r3, r3, lsl r4 /* replace with zero bits */ #else mov r3, r3, lsr r4 /* replace with zero bits */ #endif /* * Now check to see if we need to load one word or a full dword. */ tst r0, #4 /* are we dword aligned? */ bne .Lword_aligned /* no, just load a single word */ bics r4, r1, #4 /* just dealing with 1 word? */ beq .Lword_aligned /* yes, just load a single word */ /* * We are dword aligned and have a full dword to load. */ LOAD_DWORD_INTO_R4(r0) and RLO, RLO, r3 /* clear leading bytes */ teq r0, r2 /* addr == end? */ bne .Ldword_aligned_noload /* no? accumulate it and loop */ beq .Ladd_final_dword /* yes? just do the final add */ .Lword_aligned: ldr RHI, [r0], #4 /* load one word */ and RHI, RHI, r3 /* clear leading bytes */ teq r0, r2 /* addr == end? */ movne RLO, #0 /* no? clear RLO */ bne .Ldword_aligned_noload /* no? accumulate it and loop */ b .Ladd_final_word /* yes? just do the final add */ END(cpu_in_cksum_buffer)
