/* Do not modify. This file is auto-generated from aes-gcm-armv8_64.pl. */
#include "arm_arch.h"

#if __ARM_MAX_ARCH__>=8
.arch   armv8-a+crypto
.text
.globl  aes_gcm_enc_128_kernel
.type   aes_gcm_enc_128_kernel,%function
.align  4
aes_gcm_enc_128_kernel:
        AARCH64_VALID_CALL_TARGET
        cbz     x1, .L128_enc_ret
        stp     x19, x20, [sp, #-112]!
        mov     x16, x4
        mov     x8, x5
        stp     x21, x22, [sp, #16]
        stp     x23, x24, [sp, #32]
        stp     d8, d9, [sp, #48]
        stp     d10, d11, [sp, #64]
        stp     d12, d13, [sp, #80]
        stp     d14, d15, [sp, #96]

        ldp     x10, x11, [x16]              //ctr96_b64, ctr96_t32
#ifdef __AARCH64EB__
        rev     x10, x10
        rev     x11, x11
#endif
        ldp     x13, x14, [x8, #160]                     //load rk10
#ifdef __AARCH64EB__
        ror     x13, x13, #32
        ror     x14, x14, #32
#endif
        ld1     {v11.16b}, [x3]
        ext     v11.16b, v11.16b, v11.16b, #8
        rev64   v11.16b, v11.16b
        lsr     x5, x1, #3              //byte_len
        mov     x15, x5

        ld1     {v18.4s}, [x8], #16                                                               //load rk0
        add     x4, x0, x1, lsr #3   //end_input_ptr
        sub     x5, x5, #1      //byte_len - 1

        lsr     x12, x11, #32
        ldr     q15, [x3, #112]                        //load h4l | h4h
#ifndef __AARCH64EB__
        ext     v15.16b, v15.16b, v15.16b, #8
#endif
        fmov    d1, x10                               //CTR block 1
        rev     w12, w12                                //rev_ctr32

        add     w12, w12, #1                            //increment rev_ctr32
        orr     w11, w11, w11
        ld1     {v19.4s}, [x8], #16                                                               //load rk1

        rev     w9, w12                                 //CTR block 1
        add     w12, w12, #1                            //CTR block 1
        fmov    d3, x10                               //CTR block 3

        orr     x9, x11, x9, lsl #32            //CTR block 1
        ld1     { v0.16b}, [x16]                             //special case vector load initial counter so we can start first AES block as quickly as possible

        fmov    v1.d[1], x9                               //CTR block 1
        rev     w9, w12                                 //CTR block 2

        fmov    d2, x10                               //CTR block 2
        orr     x9, x11, x9, lsl #32            //CTR block 2
        add     w12, w12, #1                            //CTR block 2

        fmov    v2.d[1], x9                               //CTR block 2
        rev     w9, w12                                 //CTR block 3

        orr     x9, x11, x9, lsl #32            //CTR block 3
        ld1     {v20.4s}, [x8], #16                                                               //load rk2

        add     w12, w12, #1                            //CTR block 3
        fmov    v3.d[1], x9                               //CTR block 3

        ldr     q14, [x3, #80]                         //load h3l | h3h
#ifndef __AARCH64EB__
        ext     v14.16b, v14.16b, v14.16b, #8
#endif
        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 0
        ld1     {v21.4s}, [x8], #16                                                               //load rk3

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 0
        ldr     q12, [x3, #32]                         //load h1l | h1h
#ifndef __AARCH64EB__
        ext     v12.16b, v12.16b, v12.16b, #8
#endif

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 0
        ld1     {v22.4s}, [x8], #16                                                               //load rk4

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 0
        ld1     {v23.4s}, [x8], #16                                                               //load rk5

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 1
        trn2    v17.2d,  v14.2d,    v15.2d                      //h4l | h3l

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 1
        ld1     {v24.4s}, [x8], #16                                                               //load rk6

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 1
        ld1     {v25.4s}, [x8], #16                                                               //load rk7

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 1
        trn1    v9.2d, v14.2d,    v15.2d                      //h4h | h3h

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 2
        ld1     {v26.4s}, [x8], #16                                                               //load rk8

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 2
        ldr     q13, [x3, #64]                         //load h2l | h2h
#ifndef __AARCH64EB__
        ext     v13.16b, v13.16b, v13.16b, #8
#endif

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 2

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 2
        eor     v17.16b, v17.16b, v9.16b                  //h4k | h3k

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 3

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 3

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 3
        ld1     {v27.4s}, [x8], #16                                                               //load rk9

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 3

        and     x5, x5, #0xffffffffffffffc0    //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
        trn2    v16.2d,  v12.2d,    v13.2d                      //h2l | h1l

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 4
        add     x5, x5, x0

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 4
        cmp     x0, x5                   //check if we have <= 4 blocks

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 4

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 5

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 5

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 5

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 6

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 4

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 6
        trn1    v8.2d,    v12.2d,    v13.2d                      //h2h | h1h

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 6

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 5

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 7

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 7

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 6

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 7

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 8

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 7

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 8

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 8

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 8

        aese    v2.16b, v27.16b                                      //AES block 2 - round 9

        aese    v0.16b, v27.16b                                      //AES block 0 - round 9

        eor     v16.16b, v16.16b, v8.16b                     //h2k | h1k

        aese    v1.16b, v27.16b                                      //AES block 1 - round 9

        aese    v3.16b, v27.16b                                      //AES block 3 - round 9
        b.ge    .L128_enc_tail                                    //handle tail

        ldp     x6, x7, [x0, #0]            //AES block 0 - load plaintext
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        ldp     x21, x22, [x0, #32]           //AES block 2 - load plaintext
#ifdef __AARCH64EB__
        rev     x21, x21
        rev     x22, x22
#endif
        ldp     x19, x20, [x0, #16]           //AES block 1 - load plaintext
#ifdef __AARCH64EB__
        rev     x19, x19
        rev     x20, x20
#endif
        ldp     x23, x24, [x0, #48]           //AES block 3 - load plaintext
#ifdef __AARCH64EB__
        rev     x23, x23
        rev     x24, x24
#endif
        eor     x6, x6, x13                     //AES block 0 - round 10 low
        eor     x7, x7, x14                     //AES block 0 - round 10 high

        eor     x21, x21, x13                     //AES block 2 - round 10 low
        fmov    d4, x6                               //AES block 0 - mov low

        eor     x19, x19, x13                     //AES block 1 - round 10 low
        eor     x22, x22, x14                     //AES block 2 - round 10 high
        fmov    v4.d[1], x7                           //AES block 0 - mov high

        fmov    d5, x19                               //AES block 1 - mov low
        eor     x20, x20, x14                     //AES block 1 - round 10 high

        eor     x23, x23, x13                     //AES block 3 - round 10 low
        fmov    v5.d[1], x20                           //AES block 1 - mov high

        fmov    d6, x21                               //AES block 2 - mov low
        eor     x24, x24, x14                     //AES block 3 - round 10 high
        rev     w9, w12                                 //CTR block 4

        fmov    v6.d[1], x22                           //AES block 2 - mov high
        orr     x9, x11, x9, lsl #32            //CTR block 4

        eor     v4.16b, v4.16b, v0.16b                          //AES block 0 - result
        fmov    d0, x10                               //CTR block 4
        add     w12, w12, #1                            //CTR block 4

        fmov    v0.d[1], x9                               //CTR block 4
        rev     w9, w12                                 //CTR block 5

        eor     v5.16b, v5.16b, v1.16b                          //AES block 1 - result
        fmov    d1, x10                               //CTR block 5
        orr     x9, x11, x9, lsl #32            //CTR block 5

        add     w12, w12, #1                            //CTR block 5
        add     x0, x0, #64                       //AES input_ptr update
        fmov    v1.d[1], x9                               //CTR block 5

        fmov    d7, x23                               //AES block 3 - mov low
        rev     w9, w12                                 //CTR block 6
        st1     { v4.16b}, [x2], #16                     //AES block 0 - store result

        fmov    v7.d[1], x24                           //AES block 3 - mov high
        orr     x9, x11, x9, lsl #32            //CTR block 6

        add     w12, w12, #1                            //CTR block 6
        eor     v6.16b, v6.16b, v2.16b                          //AES block 2 - result
        st1     { v5.16b}, [x2], #16                     //AES block 1 - store result

        fmov    d2, x10                               //CTR block 6
        cmp     x0, x5                   //check if we have <= 8 blocks

        fmov    v2.d[1], x9                               //CTR block 6
        rev     w9, w12                                 //CTR block 7
        st1     { v6.16b}, [x2], #16                     //AES block 2 - store result

        orr     x9, x11, x9, lsl #32            //CTR block 7

        eor     v7.16b, v7.16b, v3.16b                          //AES block 3 - result
        st1     { v7.16b}, [x2], #16                     //AES block 3 - store result
        b.ge    .L128_enc_prepretail                              //do prepretail

.L128_enc_main_loop:    //main  loop start
        ldp     x23, x24, [x0, #48]           //AES block 4k+3 - load plaintext
#ifdef __AARCH64EB__
        rev     x23, x23
        rev     x24, x24
#endif
        rev64   v4.16b, v4.16b                                    //GHASH block 4k (only t0 is free)
        rev64   v6.16b, v6.16b                                    //GHASH block 4k+2 (t0, t1, and t2 free)

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 0
        fmov    d3, x10                               //CTR block 4k+3

        ext     v11.16b, v11.16b, v11.16b, #8                     //PRE 0
        rev64   v5.16b, v5.16b                                    //GHASH block 4k+1 (t0 and t1 free)

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 0
        add     w12, w12, #1                            //CTR block 4k+3
        fmov    v3.d[1], x9                               //CTR block 4k+3

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 0
        mov     d31, v6.d[1]                                  //GHASH block 4k+2 - mid

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 1
        mov     d30, v5.d[1]                                  //GHASH block 4k+1 - mid

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 1
        eor     v4.16b, v4.16b, v11.16b                           //PRE 1

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 0
        eor     x24, x24, x14                     //AES block 4k+3 - round 10 high

        pmull2  v28.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high
        eor     v31.8b, v31.8b, v6.8b                          //GHASH block 4k+2 - mid
        ldp     x6, x7, [x0, #0]            //AES block 4k+4 - load plaintext
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 1
        rev     w9, w12                                 //CTR block 4k+8

        eor     v30.8b, v30.8b, v5.8b                          //GHASH block 4k+1 - mid
        mov     d8, v4.d[1]                                  //GHASH block 4k - mid
        orr     x9, x11, x9, lsl #32            //CTR block 4k+8

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
        add     w12, w12, #1                            //CTR block 4k+8
        mov     d10, v17.d[1]                               //GHASH block 4k - mid

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 2

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
        eor     v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 2

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 3
        eor     v9.16b, v9.16b, v28.16b                         //GHASH block 4k+1 - high

        pmull   v28.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

        pmull   v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid
        rev64   v7.16b, v7.16b                                    //GHASH block 4k+3 (t0, t1, t2 and t3 free)

        pmull   v30.1q, v30.1d, v17.1d                          //GHASH block 4k+1 - mid

        pmull   v29.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low
        ins     v31.d[1], v31.d[0]                                //GHASH block 4k+2 - mid

        pmull2  v8.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high
        eor     x7, x7, x14                     //AES block 4k+4 - round 10 high

        eor     v10.16b, v10.16b, v30.16b                         //GHASH block 4k+1 - mid
        mov     d30, v7.d[1]                                  //GHASH block 4k+3 - mid

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 1
        eor     v11.16b, v11.16b, v29.16b                         //GHASH block 4k+1 - low

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 2
        eor     x6, x6, x13                     //AES block 4k+4 - round 10 low

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 3
        eor     v30.8b, v30.8b, v7.8b                          //GHASH block 4k+3 - mid

        pmull2  v4.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 3
        eor     v9.16b, v9.16b, v8.16b                         //GHASH block 4k+2 - high

        pmull2  v31.1q, v31.2d, v16.2d                          //GHASH block 4k+2 - mid

        pmull   v29.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low
        movi    v8.8b, #0xc2

        pmull   v30.1q, v30.1d, v16.1d                          //GHASH block 4k+3 - mid
        eor     v11.16b, v11.16b, v28.16b                         //GHASH block 4k+2 - low

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 4

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 2
        shl     d8, d8, #56               //mod_constant

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 4
        eor     v9.16b, v9.16b, v4.16b                         //GHASH block 4k+3 - high

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 5
        ldp     x19, x20, [x0, #16]           //AES block 4k+5 - load plaintext
#ifdef __AARCH64EB__
        rev     x19, x19
        rev     x20, x20
#endif
        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 3
        eor     v10.16b, v10.16b, v31.16b                         //GHASH block 4k+2 - mid

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 5
        ldp     x21, x22, [x0, #32]           //AES block 4k+6 - load plaintext
#ifdef __AARCH64EB__
        rev     x21, x21
        rev     x22, x22
#endif
        pmull   v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
        eor     v11.16b, v11.16b, v29.16b                         //GHASH block 4k+3 - low

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 4
        eor     x19, x19, x13                     //AES block 4k+5 - round 10 low

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 4
        eor     v10.16b, v10.16b, v30.16b                         //GHASH block 4k+3 - mid

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 6
        eor     x23, x23, x13                     //AES block 4k+3 - round 10 low

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 5
        eor     v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

        fmov    d4, x6                               //AES block 4k+4 - mov low
        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 6
        fmov    v4.d[1], x7                           //AES block 4k+4 - mov high

        add     x0, x0, #64                       //AES input_ptr update
        fmov    d7, x23                               //AES block 4k+3 - mov low
        ext     v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 5
        fmov    d5, x19                               //AES block 4k+5 - mov low

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 7
        eor     v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 6
        eor     x20, x20, x14                     //AES block 4k+5 - round 10 high

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 7
        fmov    v5.d[1], x20                           //AES block 4k+5 - mov high

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 8
        fmov    v7.d[1], x24                           //AES block 4k+3 - mov high

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 6
        cmp     x0, x5                   //.LOOP CONTROL

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 8
        eor     v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

        aese    v0.16b, v27.16b                                      //AES block 4k+4 - round 9
        eor     x21, x21, x13                     //AES block 4k+6 - round 10 low
        eor     x22, x22, x14                     //AES block 4k+6 - round 10 high

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 7
        fmov    d6, x21                               //AES block 4k+6 - mov low

        aese    v1.16b, v27.16b                                      //AES block 4k+5 - round 9
        fmov    v6.d[1], x22                           //AES block 4k+6 - mov high

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 7
        eor     v4.16b, v4.16b, v0.16b                          //AES block 4k+4 - result

        fmov    d0, x10                               //CTR block 4k+8
        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 8

        fmov    v0.d[1], x9                               //CTR block 4k+8
        rev     w9, w12                                 //CTR block 4k+9
        eor     v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 8
        eor     v5.16b, v5.16b, v1.16b                          //AES block 4k+5 - result

        add     w12, w12, #1                            //CTR block 4k+9
        orr     x9, x11, x9, lsl #32            //CTR block 4k+9
        fmov    d1, x10                               //CTR block 4k+9

        pmull   v9.1q, v10.1d, v8.1d            //MODULO - mid 64b align with low
        fmov    v1.d[1], x9                               //CTR block 4k+9
        rev     w9, w12                                 //CTR block 4k+10

        aese    v2.16b, v27.16b                                      //AES block 4k+6 - round 9
        st1     { v4.16b}, [x2], #16                     //AES block 4k+4 - store result
        eor     v6.16b, v6.16b, v2.16b                          //AES block 4k+6 - result
        orr     x9, x11, x9, lsl #32            //CTR block 4k+10

        aese    v3.16b, v27.16b                                      //AES block 4k+7 - round 9
        add     w12, w12, #1                            //CTR block 4k+10
        ext     v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment
        fmov    d2, x10                               //CTR block 4k+10

        eor     v11.16b, v11.16b, v9.16b                         //MODULO - fold into low
        st1     { v5.16b}, [x2], #16                     //AES block 4k+5 - store result

        fmov    v2.d[1], x9                               //CTR block 4k+10
        st1     { v6.16b}, [x2], #16                     //AES block 4k+6 - store result
        rev     w9, w12                                 //CTR block 4k+11

        orr     x9, x11, x9, lsl #32            //CTR block 4k+11
        eor     v7.16b, v7.16b, v3.16b                          //AES block 4k+3 - result

        eor     v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
        st1     { v7.16b}, [x2], #16                     //AES block 4k+3 - store result
        b.lt    .L128_enc_main_loop

.L128_enc_prepretail:   //PREPRETAIL
        rev64   v4.16b, v4.16b                                    //GHASH block 4k (only t0 is free)
        fmov    d3, x10                               //CTR block 4k+3
        rev64   v5.16b, v5.16b                                    //GHASH block 4k+1 (t0 and t1 free)

        ext     v11.16b, v11.16b, v11.16b, #8                     //PRE 0
        add     w12, w12, #1                            //CTR block 4k+3
        fmov    v3.d[1], x9                               //CTR block 4k+3

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 0
        rev64   v6.16b, v6.16b                                    //GHASH block 4k+2 (t0, t1, and t2 free)

        pmull   v29.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low

        rev64   v7.16b, v7.16b                                    //GHASH block 4k+3 (t0, t1, t2 and t3 free)
        eor     v4.16b, v4.16b, v11.16b                           //PRE 1

        pmull2  v28.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 0
        mov     d30, v5.d[1]                                  //GHASH block 4k+1 - mid

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
        mov     d8, v4.d[1]                                  //GHASH block 4k - mid

        mov     d31, v6.d[1]                                  //GHASH block 4k+2 - mid
        mov     d10, v17.d[1]                               //GHASH block 4k - mid

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 1
        eor     v30.8b, v30.8b, v5.8b                          //GHASH block 4k+1 - mid

        eor     v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
        eor     v31.8b, v31.8b, v6.8b                          //GHASH block 4k+2 - mid

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 1

        pmull   v30.1q, v30.1d, v17.1d                          //GHASH block 4k+1 - mid
        eor     v11.16b, v11.16b, v29.16b                         //GHASH block 4k+1 - low

        pmull   v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 0
        ins     v31.d[1], v31.d[0]                                //GHASH block 4k+2 - mid

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 0

        eor     v10.16b, v10.16b, v30.16b                         //GHASH block 4k+1 - mid
        mov     d30, v7.d[1]                                  //GHASH block 4k+3 - mid

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 1
        eor     v9.16b, v9.16b, v28.16b                         //GHASH block 4k+1 - high

        pmull2  v31.1q, v31.2d, v16.2d                          //GHASH block 4k+2 - mid

        pmull2  v8.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high
        eor     v30.8b, v30.8b, v7.8b                          //GHASH block 4k+3 - mid

        pmull2  v4.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

        pmull   v28.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 1
        eor     v9.16b, v9.16b, v8.16b                         //GHASH block 4k+2 - high

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 2

        pmull   v29.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low
        movi    v8.8b, #0xc2

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 2
        eor     v11.16b, v11.16b, v28.16b                         //GHASH block 4k+2 - low

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 2

        pmull   v30.1q, v30.1d, v16.1d                          //GHASH block 4k+3 - mid
        eor     v10.16b, v10.16b, v31.16b                         //GHASH block 4k+2 - mid

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 3

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 2
        eor     v9.16b, v9.16b, v4.16b                         //GHASH block 4k+3 - high

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 3

        eor     v10.16b, v10.16b, v30.16b                         //GHASH block 4k+3 - mid
        shl     d8, d8, #56               //mod_constant

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 3
        eor     v11.16b, v11.16b, v29.16b                         //GHASH block 4k+3 - low

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 4

        pmull   v28.1q, v9.1d, v8.1d
        eor     v10.16b, v10.16b, v9.16b                         //karatsuba tidy up

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 4

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 5
        ext     v9.16b, v9.16b, v9.16b, #8

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 3

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 4
        eor     v10.16b, v10.16b, v11.16b

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 6

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 4

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 5

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 5
        eor     v10.16b, v10.16b, v28.16b

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 5

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 6

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 6

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 6
        eor     v10.16b, v10.16b, v9.16b

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 7

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 7

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 7

        pmull   v28.1q, v10.1d, v8.1d

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 7
        ext     v10.16b, v10.16b, v10.16b, #8

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 8

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 8
        eor     v11.16b, v11.16b, v28.16b

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 8

        aese    v3.16b, v27.16b                                      //AES block 4k+7 - round 9

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 8

        aese    v0.16b, v27.16b                                      //AES block 4k+4 - round 9

        aese    v1.16b, v27.16b                                      //AES block 4k+5 - round 9
        eor     v11.16b, v11.16b, v10.16b

        aese    v2.16b, v27.16b                                      //AES block 4k+6 - round 9
.L128_enc_tail: //TAIL

        sub     x5, x4, x0   //main_end_input_ptr is number of bytes left to process
        ldp     x6, x7, [x0], #16           //AES block 4k+4 - load plaintext
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        cmp     x5, #48

        ext     v8.16b, v11.16b, v11.16b, #8                     //prepare final partial tag
        eor     x6, x6, x13                     //AES block 4k+4 - round 10 low
        eor     x7, x7, x14                     //AES block 4k+4 - round 10 high

        fmov    d4, x6                               //AES block 4k+4 - mov low

        fmov    v4.d[1], x7                           //AES block 4k+4 - mov high

        eor     v5.16b, v4.16b, v0.16b                          //AES block 4k+4 - result

        b.gt    .L128_enc_blocks_more_than_3

        sub     w12, w12, #1
        movi    v11.8b, #0
        mov     v3.16b, v2.16b

        cmp     x5, #32
        mov     v2.16b, v1.16b
        movi    v9.8b, #0

        movi    v10.8b, #0
        b.gt    .L128_enc_blocks_more_than_2

        mov     v3.16b, v1.16b
        cmp     x5, #16

        sub     w12, w12, #1
        b.gt    .L128_enc_blocks_more_than_1

        sub     w12, w12, #1
        b       .L128_enc_blocks_less_than_1
.L128_enc_blocks_more_than_3:   //blocks        left >  3
        st1     { v5.16b}, [x2], #16                     //AES final-3 block  - store result

        ldp     x6, x7, [x0], #16           //AES final-2 block - load input low & high
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        rev64   v4.16b, v5.16b                                    //GHASH final-3 block

        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag
        eor     x7, x7, x14                     //AES final-2 block - round 10 high
        eor     x6, x6, x13                     //AES final-2 block - round 10 low

        fmov    d5, x6                                 //AES final-2 block - mov low

        movi    v8.8b, #0                                        //suppress further partial tag feed in
        fmov    v5.d[1], x7                             //AES final-2 block - mov high

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH final-3 block - low
        mov     d22, v4.d[1]                                 //GHASH final-3 block - mid

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH final-3 block - high

        mov     d10, v17.d[1]                               //GHASH final-3 block - mid

        eor     v5.16b, v5.16b, v1.16b                            //AES final-2 block - result
        eor     v22.8b, v22.8b, v4.8b                      //GHASH final-3 block - mid

        pmull   v10.1q, v22.1d, v10.1d                    //GHASH final-3 block - mid
.L128_enc_blocks_more_than_2:   //blocks        left >  2

        st1     { v5.16b}, [x2], #16                     //AES final-2 block - store result

        rev64   v4.16b, v5.16b                                    //GHASH final-2 block
        ldp     x6, x7, [x0], #16           //AES final-1 block - load input low & high
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag

        eor     x6, x6, x13                     //AES final-1 block - round 10 low

        fmov    d5, x6                                 //AES final-1 block - mov low
        eor     x7, x7, x14                     //AES final-1 block - round 10 high

        pmull2  v20.1q, v4.2d, v14.2d                          //GHASH final-2 block - high
        fmov    v5.d[1], x7                             //AES final-1 block - mov high

        mov     d22, v4.d[1]                                 //GHASH final-2 block - mid

        pmull   v21.1q, v4.1d, v14.1d                          //GHASH final-2 block - low

        eor     v9.16b, v9.16b, v20.16b                            //GHASH final-2 block - high

        eor     v22.8b, v22.8b, v4.8b                      //GHASH final-2 block - mid

        eor     v5.16b, v5.16b, v2.16b                            //AES final-1 block - result

        eor     v11.16b, v11.16b, v21.16b                            //GHASH final-2 block - low

        pmull   v22.1q, v22.1d, v17.1d                      //GHASH final-2 block - mid

        movi    v8.8b, #0                                        //suppress further partial tag feed in

        eor     v10.16b, v10.16b, v22.16b                       //GHASH final-2 block - mid
.L128_enc_blocks_more_than_1:   //blocks        left >  1

        st1     { v5.16b}, [x2], #16                     //AES final-1 block - store result

        rev64   v4.16b, v5.16b                                    //GHASH final-1 block
        ldp     x6, x7, [x0], #16           //AES final block - load input low & high
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag

        eor     x7, x7, x14                     //AES final block - round 10 high
        eor     x6, x6, x13                     //AES final block - round 10 low

        fmov    d5, x6                                 //AES final block - mov low

        pmull2  v20.1q, v4.2d, v13.2d                          //GHASH final-1 block - high
        fmov    v5.d[1], x7                             //AES final block - mov high

        mov     d22, v4.d[1]                                 //GHASH final-1 block - mid

        pmull   v21.1q, v4.1d, v13.1d                          //GHASH final-1 block - low

        eor     v22.8b, v22.8b, v4.8b                      //GHASH final-1 block - mid

        eor     v5.16b, v5.16b, v3.16b                            //AES final block - result

        ins     v22.d[1], v22.d[0]                            //GHASH final-1 block - mid

        pmull2  v22.1q, v22.2d, v16.2d                      //GHASH final-1 block - mid

        eor     v11.16b, v11.16b, v21.16b                            //GHASH final-1 block - low

        eor     v9.16b, v9.16b, v20.16b                            //GHASH final-1 block - high

        eor     v10.16b, v10.16b, v22.16b                       //GHASH final-1 block - mid
        movi    v8.8b, #0                                        //suppress further partial tag feed in
.L128_enc_blocks_less_than_1:   //blocks        left <= 1

        and     x1, x1, #127                    //bit_length %= 128
        mvn     x13, xzr                                      //rk10_l = 0xffffffffffffffff

        mvn     x14, xzr                                      //rk10_h = 0xffffffffffffffff
        sub     x1, x1, #128                    //bit_length -= 128

        neg     x1, x1                          //bit_length = 128 - #bits in input (in range [1,128])

        and     x1, x1, #127                    //bit_length %= 128

        lsr     x14, x14, x1                     //rk10_h is mask for top 64b of last block
        cmp     x1, #64

        csel    x6, x13, x14, lt
        csel    x7, x14, xzr, lt

        fmov    d0, x6                                 //ctr0b is mask for last block

        fmov    v0.d[1], x7

        and     v5.16b, v5.16b, v0.16b                            //possibly partial last block has zeroes in highest bits

        rev64   v4.16b, v5.16b                                    //GHASH final block

        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag

        mov     d8, v4.d[1]                                  //GHASH final block - mid

        pmull   v21.1q, v4.1d, v12.1d                          //GHASH final block - low
        ld1     { v18.16b}, [x2]                            //load existing bytes where the possibly partial last block is to be stored

        eor     v8.8b, v8.8b, v4.8b                          //GHASH final block - mid
#ifndef __AARCH64EB__
        rev     w9, w12
#else
        mov     w9, w12
#endif
        pmull2  v20.1q, v4.2d, v12.2d                          //GHASH final block - high

        pmull   v8.1q, v8.1d, v16.1d                          //GHASH final block - mid

        eor     v11.16b, v11.16b, v21.16b                            //GHASH final block - low

        eor     v9.16b, v9.16b, v20.16b                            //GHASH final block - high

        eor     v10.16b, v10.16b, v8.16b                         //GHASH final block - mid
        movi    v8.8b, #0xc2

        eor     v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

        shl     d8, d8, #56               //mod_constant

        eor     v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

        pmull   v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid

        ext     v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

        eor     v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

        eor     v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

        pmull   v9.1q, v10.1d, v8.1d            //MODULO - mid 64b align with low

        ext     v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

        bif     v5.16b, v18.16b, v0.16b                              //insert existing bytes in top end of result before storing

        eor     v11.16b, v11.16b, v9.16b                         //MODULO - fold into low
        st1     { v5.16b}, [x2]                          //store all 16B

        str     w9, [x16, #12]                          //store the updated counter

        eor     v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
        ext     v11.16b, v11.16b, v11.16b, #8
        rev64   v11.16b, v11.16b
        mov     x0, x15
        st1     { v11.16b }, [x3]
        ldp     x21, x22, [sp, #16]
        ldp     x23, x24, [sp, #32]
        ldp     d8, d9, [sp, #48]
        ldp     d10, d11, [sp, #64]
        ldp     d12, d13, [sp, #80]
        ldp     d14, d15, [sp, #96]
        ldp     x19, x20, [sp], #112
        ret

.L128_enc_ret:
        mov     w0, #0x0
        ret
.size   aes_gcm_enc_128_kernel,.-aes_gcm_enc_128_kernel
.globl  aes_gcm_dec_128_kernel
.type   aes_gcm_dec_128_kernel,%function
.align  4
aes_gcm_dec_128_kernel:
        AARCH64_VALID_CALL_TARGET
        cbz     x1, .L128_dec_ret
        stp     x19, x20, [sp, #-112]!
        mov     x16, x4
        mov     x8, x5
        stp     x21, x22, [sp, #16]
        stp     x23, x24, [sp, #32]
        stp     d8, d9, [sp, #48]
        stp     d10, d11, [sp, #64]
        stp     d12, d13, [sp, #80]
        stp     d14, d15, [sp, #96]

        lsr     x5, x1, #3              //byte_len
        mov     x15, x5
        ldp     x10, x11, [x16]              //ctr96_b64, ctr96_t32
#ifdef __AARCH64EB__
        rev     x10, x10
        rev     x11, x11
#endif
        ldp     x13, x14, [x8, #160]                     //load rk10
#ifdef __AARCH64EB__
        ror     x14, x14, 32
        ror     x13, x13, 32
#endif
        sub     x5, x5, #1      //byte_len - 1
        ld1     {v18.4s}, [x8], #16                                //load rk0

        and     x5, x5, #0xffffffffffffffc0 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
        ld1     { v0.16b}, [x16]                             //special case vector load initial counter so we can start first AES block as quickly as possible

        ldr     q13, [x3, #64]                         //load h2l | h2h
#ifndef __AARCH64EB__
        ext     v13.16b, v13.16b, v13.16b, #8
#endif
        lsr     x12, x11, #32
        fmov    d2, x10                               //CTR block 2

        ld1     {v19.4s}, [x8], #16                                //load rk1
        orr     w11, w11, w11
        rev     w12, w12                                //rev_ctr32

        fmov    d1, x10                               //CTR block 1
        add     w12, w12, #1                            //increment rev_ctr32

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 0
        rev     w9, w12                                 //CTR block 1

        orr     x9, x11, x9, lsl #32            //CTR block 1
        ld1     {v20.4s}, [x8], #16                                //load rk2
        add     w12, w12, #1                            //CTR block 1

        fmov    v1.d[1], x9                               //CTR block 1
        rev     w9, w12                                 //CTR block 2
        add     w12, w12, #1                            //CTR block 2

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 1
        orr     x9, x11, x9, lsl #32            //CTR block 2

        fmov    v2.d[1], x9                               //CTR block 2
        rev     w9, w12                                 //CTR block 3

        fmov    d3, x10                               //CTR block 3
        orr     x9, x11, x9, lsl #32            //CTR block 3
        add     w12, w12, #1                            //CTR block 3

        fmov    v3.d[1], x9                               //CTR block 3
        add     x4, x0, x1, lsr #3   //end_input_ptr

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 0
        ld1     {v21.4s}, [x8], #16                                //load rk3

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 2
        ld1     {v22.4s}, [x8], #16                                //load rk4

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 0
        ld1     {v23.4s}, [x8], #16                                //load rk5

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 1
        ld1     {v24.4s}, [x8], #16                                //load rk6

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 0

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 1

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 2

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 1
        ld1     { v11.16b}, [x3]
        ext     v11.16b, v11.16b, v11.16b, #8
        rev64   v11.16b, v11.16b

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 3
        ld1     {v25.4s}, [x8], #16                                //load rk7

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 3

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 2

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 2
        ld1     {v26.4s}, [x8], #16                                //load rk8

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 4

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 3

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 3
        ldr     q14, [x3, #80]                         //load h3l | h3h
#ifndef __AARCH64EB__
        ext     v14.16b, v14.16b, v14.16b, #8
#endif
        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 4
        ld1     {v27.4s}, [x8], #16                                //load rk9

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 5

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 4

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 4

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 5

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 5
        ldr     q12, [x3, #32]                         //load h1l | h1h
#ifndef __AARCH64EB__
        ext     v12.16b, v12.16b, v12.16b, #8
#endif
        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 5

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 6

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 6

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 6

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 6
        trn1    v8.2d,    v12.2d,    v13.2d                      //h2h | h1h

        ldr     q15, [x3, #112]                        //load h4l | h4h
#ifndef __AARCH64EB__
        ext     v15.16b, v15.16b, v15.16b, #8
#endif
        trn2    v16.2d,  v12.2d,    v13.2d                      //h2l | h1l
        add     x5, x5, x0

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 7

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 7

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 7
        eor     v16.16b, v16.16b, v8.16b                     //h2k | h1k

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 7

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 8
        trn2    v17.2d,  v14.2d,    v15.2d                      //h4l | h3l

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 8

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 8

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 8
        trn1    v9.2d, v14.2d,    v15.2d                      //h4h | h3h

        aese    v2.16b, v27.16b                                      //AES block 2 - round 9

        aese    v3.16b, v27.16b                                      //AES block 3 - round 9

        aese    v0.16b, v27.16b                                      //AES block 0 - round 9
        cmp     x0, x5                   //check if we have <= 4 blocks

        aese    v1.16b, v27.16b                                      //AES block 1 - round 9
        eor     v17.16b, v17.16b, v9.16b                  //h4k | h3k
        b.ge    .L128_dec_tail                                    //handle tail

        ld1     {v4.16b, v5.16b}, [x0], #32               //AES block 0 - load ciphertext; AES block 1 - load ciphertext

        eor     v1.16b, v5.16b, v1.16b                            //AES block 1 - result
        ld1     {v6.16b}, [x0], #16                       //AES block 2 - load ciphertext

        eor     v0.16b, v4.16b, v0.16b                            //AES block 0 - result
        rev64   v4.16b, v4.16b                                    //GHASH block 0
        rev     w9, w12                                 //CTR block 4

        orr     x9, x11, x9, lsl #32            //CTR block 4
        add     w12, w12, #1                            //CTR block 4
        ld1     {v7.16b}, [x0], #16                       //AES block 3 - load ciphertext

        rev64   v5.16b, v5.16b                                    //GHASH block 1
        mov     x19, v1.d[0]                            //AES block 1 - mov low

        mov     x20, v1.d[1]                            //AES block 1 - mov high

        mov     x6, v0.d[0]                            //AES block 0 - mov low
        cmp     x0, x5                   //check if we have <= 8 blocks

        mov     x7, v0.d[1]                            //AES block 0 - mov high

        fmov    d0, x10                               //CTR block 4

        fmov    v0.d[1], x9                               //CTR block 4
        rev     w9, w12                                 //CTR block 5
        eor     x19, x19, x13                   //AES block 1 - round 10 low
#ifdef __AARCH64EB__
        rev     x19, x19
#endif
        fmov    d1, x10                               //CTR block 5
        add     w12, w12, #1                            //CTR block 5
        orr     x9, x11, x9, lsl #32            //CTR block 5

        fmov    v1.d[1], x9                               //CTR block 5
        rev     w9, w12                                 //CTR block 6
        add     w12, w12, #1                            //CTR block 6

        orr     x9, x11, x9, lsl #32            //CTR block 6

        eor     x20, x20, x14                   //AES block 1 - round 10 high
#ifdef __AARCH64EB__
        rev     x20, x20
#endif
        eor     x6, x6, x13                   //AES block 0 - round 10 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        eor     v2.16b, v6.16b, v2.16b                            //AES block 2 - result

        eor     x7, x7, x14                   //AES block 0 - round 10 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        stp     x6, x7, [x2], #16        //AES block 0 - store result

        stp     x19, x20, [x2], #16        //AES block 1 - store result
        b.ge    .L128_dec_prepretail                              //do prepretail

.L128_dec_main_loop:    //main  loop start
        eor     v3.16b, v7.16b, v3.16b                            //AES block 4k+3 - result
        ext     v11.16b, v11.16b, v11.16b, #8                     //PRE 0
        mov     x21, v2.d[0]                            //AES block 4k+2 - mov low

        pmull2  v28.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high
        mov     x22, v2.d[1]                            //AES block 4k+2 - mov high

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 0
        fmov    d2, x10                               //CTR block 4k+6

        rev64   v6.16b, v6.16b                                    //GHASH block 4k+2
        fmov    v2.d[1], x9                               //CTR block 4k+6
        rev     w9, w12                                 //CTR block 4k+7

        mov     x23, v3.d[0]                            //AES block 4k+3 - mov low
        eor     v4.16b, v4.16b, v11.16b                           //PRE 1
        mov     d30, v5.d[1]                                  //GHASH block 4k+1 - mid

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 1
        rev64   v7.16b, v7.16b                                    //GHASH block 4k+3

        pmull   v29.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low
        mov     x24, v3.d[1]                            //AES block 4k+3 - mov high
        orr     x9, x11, x9, lsl #32            //CTR block 4k+7

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
        fmov    d3, x10                               //CTR block 4k+7
        eor     v30.8b, v30.8b, v5.8b                          //GHASH block 4k+1 - mid

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 2
        fmov    v3.d[1], x9                               //CTR block 4k+7

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 0
        mov     d10, v17.d[1]                               //GHASH block 4k - mid

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
        eor     v11.16b, v11.16b, v29.16b                         //GHASH block 4k+1 - low

        pmull   v29.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 3
        mov     d8, v4.d[1]                                  //GHASH block 4k - mid

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 0
        eor     v9.16b, v9.16b, v28.16b                         //GHASH block 4k+1 - high

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 0

        pmull   v28.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low
        eor     v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 1
        eor     x23, x23, x13                   //AES block 4k+3 - round 10 low
#ifdef __AARCH64EB__
        rev     x23, x23
#endif
        pmull   v30.1q, v30.1d, v17.1d                          //GHASH block 4k+1 - mid
        eor     x22, x22, x14                   //AES block 4k+2 - round 10 high
#ifdef __AARCH64EB__
        rev     x22, x22
#endif
        mov     d31, v6.d[1]                                  //GHASH block 4k+2 - mid

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 1
        eor     v11.16b, v11.16b, v28.16b                         //GHASH block 4k+2 - low

        pmull   v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 2
        eor     v31.8b, v31.8b, v6.8b                          //GHASH block 4k+2 - mid

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 2

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 4
        eor     v10.16b, v10.16b, v30.16b                         //GHASH block 4k+1 - mid

        pmull2  v8.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 3
        ins     v31.d[1], v31.d[0]                                //GHASH block 4k+2 - mid

        pmull2  v4.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 1
        mov     d30, v7.d[1]                                  //GHASH block 4k+3 - mid

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 4
        eor     v9.16b, v9.16b, v8.16b                         //GHASH block 4k+2 - high

        pmull2  v31.1q, v31.2d, v16.2d                          //GHASH block 4k+2 - mid
        eor     x24, x24, x14                   //AES block 4k+3 - round 10 high
#ifdef __AARCH64EB__
        rev     x24, x24
#endif
        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 2
        eor     v30.8b, v30.8b, v7.8b                          //GHASH block 4k+3 - mid

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 5
        eor     x21, x21, x13                   //AES block 4k+2 - round 10 low
#ifdef __AARCH64EB__
        rev     x21, x21
#endif
        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 5
        movi    v8.8b, #0xc2

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 3
        eor     v11.16b, v11.16b, v29.16b                         //GHASH block 4k+3 - low

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 6

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 6
        eor     v10.16b, v10.16b, v31.16b                         //GHASH block 4k+2 - mid

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 4
        stp     x21, x22, [x2], #16        //AES block 4k+2 - store result

        pmull   v30.1q, v30.1d, v16.1d                          //GHASH block 4k+3 - mid
        eor     v9.16b, v9.16b, v4.16b                         //GHASH block 4k+3 - high
        ld1     {v4.16b}, [x0], #16                       //AES block 4k+3 - load ciphertext

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 7
        add     w12, w12, #1                            //CTR block 4k+7

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 7
        shl     d8, d8, #56               //mod_constant

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 5
        eor     v10.16b, v10.16b, v30.16b                         //GHASH block 4k+3 - mid

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 8
        stp     x23, x24, [x2], #16        //AES block 4k+3 - store result

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 8
        eor     v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 3
        rev     w9, w12                                 //CTR block 4k+8

        pmull   v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
        ld1     {v5.16b}, [x0], #16                       //AES block 4k+4 - load ciphertext
        ext     v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

        aese    v0.16b, v27.16b                                      //AES block 4k+4 - round 9
        orr     x9, x11, x9, lsl #32            //CTR block 4k+8

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 4
        eor     v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

        aese    v1.16b, v27.16b                                      //AES block 4k+5 - round 9

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 6
        eor     v0.16b, v4.16b, v0.16b                            //AES block 4k+4 - result

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 5
        ld1     {v6.16b}, [x0], #16                       //AES block 4k+5 - load ciphertext

        add     w12, w12, #1                            //CTR block 4k+8
        eor     v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid
        eor     v1.16b, v5.16b, v1.16b                            //AES block 4k+5 - result

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 7
        ld1     {v7.16b}, [x0], #16                       //AES block 4k+6 - load ciphertext

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 6

        rev64   v5.16b, v5.16b                                    //GHASH block 4k+5
        eor     v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid
        mov     x7, v0.d[1]                            //AES block 4k+4 - mov high

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 8
        mov     x6, v0.d[0]                            //AES block 4k+4 - mov low

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 7
        fmov    d0, x10                               //CTR block 4k+8

        pmull   v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low
        fmov    v0.d[1], x9                               //CTR block 4k+8
        rev     w9, w12                                 //CTR block 4k+9

        aese    v2.16b, v27.16b                                      //AES block 4k+6 - round 9
        orr     x9, x11, x9, lsl #32            //CTR block 4k+9
        ext     v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 8
        eor     x7, x7, x14                   //AES block 4k+4 - round 10 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        eor     v11.16b, v11.16b, v8.16b               //MODULO - fold into low
        mov     x20, v1.d[1]                            //AES block 4k+5 - mov high
        eor     x6, x6, x13                   //AES block 4k+4 - round 10 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        eor     v2.16b, v6.16b, v2.16b                            //AES block 4k+6 - result
        mov     x19, v1.d[0]                            //AES block 4k+5 - mov low
        add     w12, w12, #1                            //CTR block 4k+9

        aese    v3.16b, v27.16b                                      //AES block 4k+7 - round 9
        fmov    d1, x10                               //CTR block 4k+9
        cmp     x0, x5                   //.LOOP CONTROL

        rev64   v4.16b, v4.16b                                    //GHASH block 4k+4
        eor     v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
        fmov    v1.d[1], x9                               //CTR block 4k+9

        rev     w9, w12                                 //CTR block 4k+10
        add     w12, w12, #1                            //CTR block 4k+10

        eor     x20, x20, x14                   //AES block 4k+5 - round 10 high
#ifdef __AARCH64EB__
        rev     x20, x20
#endif
        stp     x6, x7, [x2], #16        //AES block 4k+4 - store result

        eor     x19, x19, x13                   //AES block 4k+5 - round 10 low
#ifdef __AARCH64EB__
        rev     x19, x19
#endif
        stp     x19, x20, [x2], #16        //AES block 4k+5 - store result

        orr     x9, x11, x9, lsl #32            //CTR block 4k+10
        b.lt    .L128_dec_main_loop

.L128_dec_prepretail:   //PREPRETAIL
        ext     v11.16b, v11.16b, v11.16b, #8                     //PRE 0
        mov     x21, v2.d[0]                            //AES block 4k+2 - mov low
        mov     d30, v5.d[1]                                  //GHASH block 4k+1 - mid

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 0
        eor     v3.16b, v7.16b, v3.16b                            //AES block 4k+3 - result

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 0
        mov     x22, v2.d[1]                            //AES block 4k+2 - mov high

        eor     v4.16b, v4.16b, v11.16b                           //PRE 1
        fmov    d2, x10                               //CTR block 4k+6
        rev64   v6.16b, v6.16b                                    //GHASH block 4k+2

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 1
        fmov    v2.d[1], x9                               //CTR block 4k+6

        rev     w9, w12                                 //CTR block 4k+7
        mov     x23, v3.d[0]                            //AES block 4k+3 - mov low
        eor     v30.8b, v30.8b, v5.8b                          //GHASH block 4k+1 - mid

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
        mov     d10, v17.d[1]                               //GHASH block 4k - mid
        mov     x24, v3.d[1]                            //AES block 4k+3 - mov high

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 1
        mov     d31, v6.d[1]                                  //GHASH block 4k+2 - mid

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 2
        orr     x9, x11, x9, lsl #32            //CTR block 4k+7

        pmull   v29.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low
        mov     d8, v4.d[1]                                  //GHASH block 4k - mid
        fmov    d3, x10                               //CTR block 4k+7

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 0
        fmov    v3.d[1], x9                               //CTR block 4k+7

        pmull   v30.1q, v30.1d, v17.1d                          //GHASH block 4k+1 - mid
        eor     v31.8b, v31.8b, v6.8b                          //GHASH block 4k+2 - mid

        rev64   v7.16b, v7.16b                                    //GHASH block 4k+3

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 1
        eor     v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 0
        ins     v31.d[1], v31.d[0]                                //GHASH block 4k+2 - mid

        pmull2  v28.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high

        pmull   v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid
        eor     v11.16b, v11.16b, v29.16b                         //GHASH block 4k+1 - low

        pmull   v29.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low

        pmull2  v31.1q, v31.2d, v16.2d                          //GHASH block 4k+2 - mid
        eor     v9.16b, v9.16b, v28.16b                         //GHASH block 4k+1 - high

        eor     v10.16b, v10.16b, v30.16b                         //GHASH block 4k+1 - mid

        pmull2  v4.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

        pmull2  v8.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high
        mov     d30, v7.d[1]                                  //GHASH block 4k+3 - mid

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 2
        eor     v10.16b, v10.16b, v31.16b                         //GHASH block 4k+2 - mid

        pmull   v28.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

        eor     v9.16b, v9.16b, v8.16b                         //GHASH block 4k+2 - high
        movi    v8.8b, #0xc2

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 1
        eor     v30.8b, v30.8b, v7.8b                          //GHASH block 4k+3 - mid

        eor     v11.16b, v11.16b, v28.16b                         //GHASH block 4k+2 - low

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 2
        eor     v9.16b, v9.16b, v4.16b                         //GHASH block 4k+3 - high

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 2
        eor     x23, x23, x13                   //AES block 4k+3 - round 10 low
#ifdef __AARCH64EB__
        rev     x23, x23
#endif
        pmull   v30.1q, v30.1d, v16.1d                          //GHASH block 4k+3 - mid
        eor     x21, x21, x13                   //AES block 4k+2 - round 10 low
#ifdef __AARCH64EB__
        rev     x21, x21
#endif
        eor     v11.16b, v11.16b, v29.16b                         //GHASH block 4k+3 - low

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 3

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 3
        shl     d8, d8, #56               //mod_constant

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 3

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 4
        eor     v10.16b, v10.16b, v30.16b                         //GHASH block 4k+3 - mid

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 4

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 3
        eor     v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 5

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 5

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 4

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 4
        eor     v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

        pmull   v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 6
        ext     v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 5

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 5
        eor     v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 7

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 6

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 6

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 8
        eor     v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 6

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 7

        aese    v1.16b, v27.16b                                      //AES block 4k+5 - round 9

        pmull   v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low
        eor     x24, x24, x14                   //AES block 4k+3 - round 10 high
#ifdef __AARCH64EB__
        rev     x24, x24
#endif
        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 7
        ext     v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 7

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 8
        eor     v11.16b, v11.16b, v8.16b               //MODULO - fold into low

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 8

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 8
        eor     x22, x22, x14                   //AES block 4k+2 - round 10 high
#ifdef __AARCH64EB__
        rev     x22, x22
#endif
        aese    v0.16b, v27.16b                                      //AES block 4k+4 - round 9
        stp     x21, x22, [x2], #16        //AES block 4k+2 - store result

        aese    v2.16b, v27.16b                                      //AES block 4k+6 - round 9
        add     w12, w12, #1                            //CTR block 4k+7
        stp     x23, x24, [x2], #16        //AES block 4k+3 - store result

        aese    v3.16b, v27.16b                                      //AES block 4k+7 - round 9
        eor     v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
.L128_dec_tail: //TAIL

        sub     x5, x4, x0   //main_end_input_ptr is number of bytes left to process
        ld1     { v5.16b}, [x0], #16                      //AES block 4k+4 - load ciphertext

        eor     v0.16b, v5.16b, v0.16b                            //AES block 4k+4 - result

        mov     x7, v0.d[1]                            //AES block 4k+4 - mov high

        mov     x6, v0.d[0]                            //AES block 4k+4 - mov low

        cmp     x5, #48

        eor     x7, x7, x14                   //AES block 4k+4 - round 10 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        ext     v8.16b, v11.16b, v11.16b, #8                     //prepare final partial tag
        eor     x6, x6, x13                   //AES block 4k+4 - round 10 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        b.gt    .L128_dec_blocks_more_than_3

        mov     v3.16b, v2.16b
        sub     w12, w12, #1
        movi    v11.8b, #0

        movi    v9.8b, #0
        mov     v2.16b, v1.16b

        movi    v10.8b, #0
        cmp     x5, #32
        b.gt    .L128_dec_blocks_more_than_2

        cmp     x5, #16

        mov     v3.16b, v1.16b
        sub     w12, w12, #1
        b.gt    .L128_dec_blocks_more_than_1

        sub     w12, w12, #1
        b       .L128_dec_blocks_less_than_1
.L128_dec_blocks_more_than_3:   //blocks        left >  3
        rev64   v4.16b, v5.16b                                    //GHASH final-3 block
        ld1     { v5.16b}, [x0], #16                      //AES final-2 block - load ciphertext

        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag

        mov     d10, v17.d[1]                               //GHASH final-3 block - mid
        stp     x6, x7, [x2], #16        //AES final-3 block  - store result
        eor     v0.16b, v5.16b, v1.16b                            //AES final-2 block - result

        mov     d22, v4.d[1]                                 //GHASH final-3 block - mid
        mov     x7, v0.d[1]                            //AES final-2 block - mov high

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH final-3 block - low
        mov     x6, v0.d[0]                            //AES final-2 block - mov low

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH final-3 block - high

        eor     v22.8b, v22.8b, v4.8b                      //GHASH final-3 block - mid

        movi    v8.8b, #0                                        //suppress further partial tag feed in
        eor     x7, x7, x14                   //AES final-2 block - round 10 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        pmull   v10.1q, v22.1d, v10.1d                    //GHASH final-3 block - mid
        eor     x6, x6, x13                   //AES final-2 block - round 10 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
.L128_dec_blocks_more_than_2:   //blocks        left >  2

        rev64   v4.16b, v5.16b                                    //GHASH final-2 block
        ld1     { v5.16b}, [x0], #16                      //AES final-1 block - load ciphertext

        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag

        eor     v0.16b, v5.16b, v2.16b                            //AES final-1 block - result
        stp     x6, x7, [x2], #16        //AES final-2 block  - store result

        mov     d22, v4.d[1]                                 //GHASH final-2 block - mid

        pmull   v21.1q, v4.1d, v14.1d                          //GHASH final-2 block - low

        pmull2  v20.1q, v4.2d, v14.2d                          //GHASH final-2 block - high
        mov     x6, v0.d[0]                            //AES final-1 block - mov low

        mov     x7, v0.d[1]                            //AES final-1 block - mov high
        eor     v22.8b, v22.8b, v4.8b                      //GHASH final-2 block - mid

        movi    v8.8b, #0                                        //suppress further partial tag feed in

        pmull   v22.1q, v22.1d, v17.1d                      //GHASH final-2 block - mid

        eor     x6, x6, x13                   //AES final-1 block - round 10 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        eor     v11.16b, v11.16b, v21.16b                            //GHASH final-2 block - low

        eor     v9.16b, v9.16b, v20.16b                            //GHASH final-2 block - high

        eor     v10.16b, v10.16b, v22.16b                       //GHASH final-2 block - mid
        eor     x7, x7, x14                   //AES final-1 block - round 10 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
.L128_dec_blocks_more_than_1:   //blocks        left >  1

        rev64   v4.16b, v5.16b                                    //GHASH final-1 block

        ld1     { v5.16b}, [x0], #16                      //AES final block - load ciphertext
        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag

        mov     d22, v4.d[1]                                 //GHASH final-1 block - mid

        eor     v0.16b, v5.16b, v3.16b                            //AES final block - result

        eor     v22.8b, v22.8b, v4.8b                      //GHASH final-1 block - mid

        stp     x6, x7, [x2], #16        //AES final-1 block  - store result
        mov     x6, v0.d[0]                            //AES final block - mov low

        mov     x7, v0.d[1]                            //AES final block - mov high
        ins     v22.d[1], v22.d[0]                            //GHASH final-1 block - mid

        pmull   v21.1q, v4.1d, v13.1d                          //GHASH final-1 block - low

        pmull2  v20.1q, v4.2d, v13.2d                          //GHASH final-1 block - high

        pmull2  v22.1q, v22.2d, v16.2d                      //GHASH final-1 block - mid
        movi    v8.8b, #0                                        //suppress further partial tag feed in

        eor     v11.16b, v11.16b, v21.16b                            //GHASH final-1 block - low

        eor     v9.16b, v9.16b, v20.16b                            //GHASH final-1 block - high
        eor     x7, x7, x14                   //AES final block - round 10 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        eor     x6, x6, x13                   //AES final block - round 10 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        eor     v10.16b, v10.16b, v22.16b                       //GHASH final-1 block - mid
.L128_dec_blocks_less_than_1:   //blocks        left <= 1

        mvn     x14, xzr                                      //rk10_h = 0xffffffffffffffff
        and     x1, x1, #127                    //bit_length %= 128

        mvn     x13, xzr                                      //rk10_l = 0xffffffffffffffff
        sub     x1, x1, #128                    //bit_length -= 128

        neg     x1, x1                          //bit_length = 128 - #bits in input (in range [1,128])

        and     x1, x1, #127                    //bit_length %= 128

        lsr     x14, x14, x1                     //rk10_h is mask for top 64b of last block
        cmp     x1, #64

        csel    x10, x14, xzr, lt
        csel    x9, x13, x14, lt

        fmov    d0, x9                                   //ctr0b is mask for last block

        mov     v0.d[1], x10

        and     v5.16b, v5.16b, v0.16b                            //possibly partial last block has zeroes in highest bits

        rev64   v4.16b, v5.16b                                    //GHASH final block

        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag

        ldp     x4, x5, [x2] //load existing bytes we need to not overwrite

        and     x7, x7, x10

        pmull2  v20.1q, v4.2d, v12.2d                          //GHASH final block - high
        mov     d8, v4.d[1]                                  //GHASH final block - mid

        eor     v8.8b, v8.8b, v4.8b                          //GHASH final block - mid
        eor     v9.16b, v9.16b, v20.16b                            //GHASH final block - high

        pmull   v8.1q, v8.1d, v16.1d                          //GHASH final block - mid

        pmull   v21.1q, v4.1d, v12.1d                          //GHASH final block - low
        bic     x4, x4, x9           //mask out low existing bytes
        and     x6, x6, x9

#ifndef __AARCH64EB__
        rev     w9, w12
#else
        mov     w9, w12
#endif

        eor     v10.16b, v10.16b, v8.16b                         //GHASH final block - mid
        movi    v8.8b, #0xc2

        eor     v11.16b, v11.16b, v21.16b                            //GHASH final block - low

        bic     x5, x5, x10   //mask out high existing bytes
        shl     d8, d8, #56               //mod_constant

        eor     v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

        pmull   v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid

        eor     v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

        orr     x6, x6, x4
        str     w9, [x16, #12]                          //store the updated counter

        orr     x7, x7, x5
        stp     x6, x7, [x2]
        ext     v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

        eor     v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

        eor     v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

        pmull   v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low
        ext     v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

        eor     v11.16b, v11.16b, v8.16b               //MODULO - fold into low

        eor     v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
        ext     v11.16b, v11.16b, v11.16b, #8
        rev64   v11.16b, v11.16b
        mov     x0, x15
        st1     { v11.16b }, [x3]

        ldp     x21, x22, [sp, #16]
        ldp     x23, x24, [sp, #32]
        ldp     d8, d9, [sp, #48]
        ldp     d10, d11, [sp, #64]
        ldp     d12, d13, [sp, #80]
        ldp     d14, d15, [sp, #96]
        ldp     x19, x20, [sp], #112
        ret

.L128_dec_ret:
        mov     w0, #0x0
        ret
.size   aes_gcm_dec_128_kernel,.-aes_gcm_dec_128_kernel
.globl  aes_gcm_enc_192_kernel
.type   aes_gcm_enc_192_kernel,%function
.align  4
aes_gcm_enc_192_kernel:
        AARCH64_VALID_CALL_TARGET
        cbz     x1, .L192_enc_ret
        stp     x19, x20, [sp, #-112]!
        mov     x16, x4
        mov     x8, x5
        stp     x21, x22, [sp, #16]
        stp     x23, x24, [sp, #32]
        stp     d8, d9, [sp, #48]
        stp     d10, d11, [sp, #64]
        stp     d12, d13, [sp, #80]
        stp     d14, d15, [sp, #96]

        ldp     x10, x11, [x16]             //ctr96_b64, ctr96_t32
#ifdef __AARCH64EB__
        rev     x10, x10
        rev     x11, x11
#endif
        ldp     x13, x14, [x8, #192]                     //load rk12
#ifdef __AARCH64EB__
        ror     x13, x13, #32
        ror     x14, x14, #32
#endif
        ld1     {v18.4s}, [x8], #16                                  //load rk0

        ld1     {v19.4s}, [x8], #16                                  //load rk1

        ld1     {v20.4s}, [x8], #16                                  //load rk2

        lsr     x12, x11, #32
        ld1     {v21.4s}, [x8], #16                                  //load rk3
        orr     w11, w11, w11

        ld1     {v22.4s}, [x8], #16                                  //load rk4
        rev     w12, w12                               //rev_ctr32

        add     w12, w12, #1                           //increment rev_ctr32
        fmov    d3, x10                              //CTR block 3

        rev     w9, w12                                //CTR block 1
        add     w12, w12, #1                           //CTR block 1
        fmov    d1, x10                              //CTR block 1

        orr     x9, x11, x9, lsl #32           //CTR block 1
        ld1     { v0.16b}, [x16]                            //special case vector load initial counter so we can start first AES block as quickly as possible

        fmov    v1.d[1], x9                              //CTR block 1
        rev     w9, w12                                //CTR block 2
        add     w12, w12, #1                           //CTR block 2

        fmov    d2, x10                              //CTR block 2
        orr     x9, x11, x9, lsl #32           //CTR block 2

        fmov    v2.d[1], x9                              //CTR block 2
        rev     w9, w12                                //CTR block 3

        orr     x9, x11, x9, lsl #32           //CTR block 3
        ld1     {v23.4s}, [x8], #16                                  //load rk5

        fmov    v3.d[1], x9                              //CTR block 3

        ld1     {v24.4s}, [x8], #16                                  //load rk6

        ld1     {v25.4s}, [x8], #16                                  //load rk7

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b         //AES block 0 - round 0
        ld1     { v11.16b}, [x3]
        ext     v11.16b, v11.16b, v11.16b, #8
        rev64   v11.16b, v11.16b

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b         //AES block 3 - round 0
        ld1     {v26.4s}, [x8], #16                                  //load rk8

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b         //AES block 1 - round 0
        ldr     q15, [x3, #112]                       //load h4l | h4h
#ifndef __AARCH64EB__
        ext     v15.16b, v15.16b, v15.16b, #8
#endif
        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b         //AES block 2 - round 0
        ld1     {v27.4s}, [x8], #16                                  //load rk9

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b         //AES block 0 - round 1
        ld1     {v28.4s}, [x8], #16                              //load rk10

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b         //AES block 1 - round 1
        ldr     q12, [x3, #32]                        //load h1l | h1h
#ifndef __AARCH64EB__
        ext     v12.16b, v12.16b, v12.16b, #8
#endif
        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b         //AES block 2 - round 1
        ld1     {v29.4s}, [x8], #16                              //load rk11

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b         //AES block 3 - round 1
        ldr     q14, [x3, #80]                        //load h3l | h3h
#ifndef __AARCH64EB__
        ext     v14.16b, v14.16b, v14.16b, #8
#endif
        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b         //AES block 0 - round 2

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b         //AES block 2 - round 2

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b         //AES block 3 - round 2

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b         //AES block 0 - round 3
        trn1    v9.2d, v14.2d,    v15.2d                     //h4h | h3h

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b         //AES block 2 - round 3

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b         //AES block 1 - round 2
        trn2    v17.2d,  v14.2d,    v15.2d                     //h4l | h3l

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b         //AES block 0 - round 4

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b         //AES block 3 - round 3

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b         //AES block 1 - round 3

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b         //AES block 0 - round 5

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b         //AES block 2 - round 4

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b         //AES block 1 - round 4

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b         //AES block 0 - round 6

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b         //AES block 3 - round 4

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b         //AES block 2 - round 5

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b         //AES block 1 - round 5

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b         //AES block 3 - round 5

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b         //AES block 2 - round 6
        ldr     q13, [x3, #64]                        //load h2l | h2h
#ifndef __AARCH64EB__
        ext     v13.16b, v13.16b, v13.16b, #8
#endif
        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b         //AES block 1 - round 6

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b         //AES block 3 - round 6

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b         //AES block 0 - round 7

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b         //AES block 1 - round 7
        trn2    v16.2d,  v12.2d,    v13.2d                     //h2l | h1l

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b         //AES block 3 - round 7

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b         //AES block 0 - round 8

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b         //AES block 2 - round 7
        trn1    v8.2d,    v12.2d,    v13.2d                     //h2h | h1h

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b         //AES block 1 - round 8

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b         //AES block 3 - round 8

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b         //AES block 2 - round 8

        aese    v0.16b, v27.16b
        aesmc   v0.16b, v0.16b         //AES block 0 - round 9

        aese    v3.16b, v27.16b
        aesmc   v3.16b, v3.16b         //AES block 3 - round 9

        aese    v2.16b, v27.16b
        aesmc   v2.16b, v2.16b         //AES block 2 - round 9

        aese    v1.16b, v27.16b
        aesmc   v1.16b, v1.16b         //AES block 1 - round 9

        aese    v0.16b, v28.16b
        aesmc   v0.16b, v0.16b         //AES block 0 - round 10

        aese    v2.16b, v28.16b
        aesmc   v2.16b, v2.16b         //AES block 2 - round 10

        aese    v1.16b, v28.16b
        aesmc   v1.16b, v1.16b         //AES block 1 - round 10
        lsr     x5, x1, #3             //byte_len
        mov     x15, x5

        aese    v3.16b, v28.16b
        aesmc   v3.16b, v3.16b         //AES block 3 - round 10
        sub     x5, x5, #1     //byte_len - 1

        eor     v16.16b, v16.16b, v8.16b                    //h2k | h1k
        and     x5, x5, #0xffffffffffffffc0   //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

        eor     v17.16b, v17.16b, v9.16b                 //h4k | h3k

        aese    v2.16b, v29.16b                                    //AES block 2 - round 11
        add     x4, x0, x1, lsr #3  //end_input_ptr
        add     x5, x5, x0

        aese    v1.16b, v29.16b                                    //AES block 1 - round 11
        cmp     x0, x5                  //check if we have <= 4 blocks

        aese    v0.16b, v29.16b                                    //AES block 0 - round 11
        add     w12, w12, #1                           //CTR block 3

        aese    v3.16b, v29.16b                                    //AES block 3 - round 11
        b.ge    .L192_enc_tail                                   //handle tail

        rev     w9, w12                                //CTR block 4
        ldp     x6, x7, [x0, #0]           //AES block 0 - load plaintext
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        orr     x9, x11, x9, lsl #32           //CTR block 4
        ldp     x21, x22, [x0, #32]          //AES block 2 - load plaintext
#ifdef __AARCH64EB__
        rev     x21, x21
        rev     x22, x22
#endif
        ldp     x23, x24, [x0, #48]          //AES block 3 - load plaintext
#ifdef __AARCH64EB__
        rev     x23, x23
        rev     x24, x24
#endif
        ldp     x19, x20, [x0, #16]          //AES block 1 - load plaintext
#ifdef __AARCH64EB__
        rev     x19, x19
        rev     x20, x20
#endif
        add     x0, x0, #64                      //AES input_ptr update
        cmp     x0, x5                  //check if we have <= 8 blocks

        eor     x6, x6, x13                    //AES block 0 - round 12 low

        eor     x7, x7, x14                    //AES block 0 - round 12 high
        eor     x22, x22, x14                    //AES block 2 - round 12 high
        fmov    d4, x6                              //AES block 0 - mov low

        eor     x24, x24, x14                    //AES block 3 - round 12 high
        fmov    v4.d[1], x7                          //AES block 0 - mov high

        eor     x21, x21, x13                    //AES block 2 - round 12 low
        eor     x19, x19, x13                    //AES block 1 - round 12 low

        fmov    d5, x19                              //AES block 1 - mov low
        eor     x20, x20, x14                    //AES block 1 - round 12 high

        fmov    v5.d[1], x20                          //AES block 1 - mov high

        eor     x23, x23, x13                    //AES block 3 - round 12 low
        fmov    d6, x21                              //AES block 2 - mov low

        add     w12, w12, #1                           //CTR block 4
        eor     v4.16b, v4.16b, v0.16b                         //AES block 0 - result
        fmov    d0, x10                              //CTR block 4

        fmov    v0.d[1], x9                              //CTR block 4
        rev     w9, w12                                //CTR block 5

        orr     x9, x11, x9, lsl #32           //CTR block 5
        add     w12, w12, #1                           //CTR block 5

        fmov    d7, x23                              //AES block 3 - mov low
        st1     { v4.16b}, [x2], #16                    //AES block 0 - store result

        fmov    v6.d[1], x22                          //AES block 2 - mov high

        eor     v5.16b, v5.16b, v1.16b                         //AES block 1 - result
        fmov    d1, x10                              //CTR block 5
        st1     { v5.16b}, [x2], #16                    //AES block 1 - store result

        fmov    v7.d[1], x24                          //AES block 3 - mov high

        fmov    v1.d[1], x9                              //CTR block 5
        rev     w9, w12                                //CTR block 6

        orr     x9, x11, x9, lsl #32           //CTR block 6

        add     w12, w12, #1                           //CTR block 6
        eor     v6.16b, v6.16b, v2.16b                         //AES block 2 - result
        fmov    d2, x10                              //CTR block 6

        fmov    v2.d[1], x9                              //CTR block 6
        rev     w9, w12                                //CTR block 7

        orr     x9, x11, x9, lsl #32           //CTR block 7
        st1     { v6.16b}, [x2], #16                    //AES block 2 - store result

        eor     v7.16b, v7.16b, v3.16b                         //AES block 3 - result
        st1     { v7.16b}, [x2], #16                    //AES block 3 - store result
        b.ge    .L192_enc_prepretail                             //do prepretail

.L192_enc_main_loop:    //main  loop start
        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 0
        rev64   v5.16b, v5.16b                                   //GHASH block 4k+1 (t0 and t1 free)

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 0
        ldp     x19, x20, [x0, #16]          //AES block 4k+5 - load plaintext
#ifdef __AARCH64EB__
        rev     x19, x19
        rev     x20, x20
#endif
        ext     v11.16b, v11.16b, v11.16b, #8                    //PRE 0
        fmov    d3, x10                              //CTR block 4k+3
        rev64   v4.16b, v4.16b                                   //GHASH block 4k (only t0 is free)

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 1
        fmov    v3.d[1], x9                              //CTR block 4k+3

        pmull2  v30.1q, v5.2d, v14.2d                         //GHASH block 4k+1 - high
        rev64   v7.16b, v7.16b                                   //GHASH block 4k+3 (t0, t1, t2 and t3 free)
        ldp     x21, x22, [x0, #32]          //AES block 4k+6 - load plaintext
#ifdef __AARCH64EB__
        rev     x21, x21
        rev     x22, x22
#endif
        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 0
        ldp     x23, x24, [x0, #48]          //AES block 4k+3 - load plaintext
#ifdef __AARCH64EB__
        rev     x23, x23
        rev     x24, x24
#endif
        pmull   v31.1q, v5.1d, v14.1d                         //GHASH block 4k+1 - low
        eor     v4.16b, v4.16b, v11.16b                          //PRE 1

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 1

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 1
        rev64   v6.16b, v6.16b                                   //GHASH block 4k+2 (t0, t1, and t2 free)

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 0
        eor     x24, x24, x14                    //AES block 4k+3 - round 12 high

        pmull   v11.1q, v4.1d, v15.1d                      //GHASH block 4k - low
        mov     d8, v4.d[1]                                 //GHASH block 4k - mid

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 2

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 1
        eor     x21, x21, x13                    //AES block 4k+6 - round 12 low

        eor     v8.8b, v8.8b, v4.8b                         //GHASH block 4k - mid
        eor     v11.16b, v11.16b, v31.16b                        //GHASH block 4k+1 - low

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 3
        eor     x19, x19, x13                    //AES block 4k+5 - round 12 low

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 2
        mov     d31, v6.d[1]                                 //GHASH block 4k+2 - mid

        pmull2  v9.1q, v4.2d, v15.2d                      //GHASH block 4k - high
        mov     d4, v5.d[1]                                 //GHASH block 4k+1 - mid

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 2

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 3

        mov     d10, v17.d[1]                              //GHASH block 4k - mid
        eor     v9.16b, v9.16b, v30.16b                        //GHASH block 4k+1 - high

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 2
        eor     v31.8b, v31.8b, v6.8b                         //GHASH block 4k+2 - mid

        pmull2  v30.1q, v6.2d, v13.2d                         //GHASH block 4k+2 - high

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 4
        eor     v4.8b, v4.8b, v5.8b                         //GHASH block 4k+1 - mid

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 3

        pmull2  v5.1q, v7.2d, v12.2d                         //GHASH block 4k+3 - high
        eor     x20, x20, x14                    //AES block 4k+5 - round 12 high
        ins     v31.d[1], v31.d[0]                               //GHASH block 4k+2 - mid

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 5
        add     w12, w12, #1                           //CTR block 4k+3

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 4
        eor     v9.16b, v9.16b, v30.16b                        //GHASH block 4k+2 - high

        pmull   v4.1q, v4.1d, v17.1d                         //GHASH block 4k+1 - mid
        eor     x22, x22, x14                    //AES block 4k+6 - round 12 high

        pmull2  v31.1q, v31.2d, v16.2d                         //GHASH block 4k+2 - mid
        eor     x23, x23, x13                    //AES block 4k+3 - round 12 low
        mov     d30, v7.d[1]                                 //GHASH block 4k+3 - mid

        pmull   v10.1q, v8.1d, v10.1d                     //GHASH block 4k - mid
        rev     w9, w12                                //CTR block 4k+8

        pmull   v8.1q, v6.1d, v13.1d                         //GHASH block 4k+2 - low
        orr     x9, x11, x9, lsl #32           //CTR block 4k+8

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 3
        eor     v30.8b, v30.8b, v7.8b                         //GHASH block 4k+3 - mid

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 4
        ldp     x6, x7, [x0, #0]           //AES block 4k+4 - load plaintext
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 6
        eor     v11.16b, v11.16b, v8.16b                        //GHASH block 4k+2 - low

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 4
        add     x0, x0, #64                      //AES input_ptr update

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 5
        movi    v8.8b, #0xc2

        pmull   v6.1q, v7.1d, v12.1d                         //GHASH block 4k+3 - low
        eor     x7, x7, x14                    //AES block 4k+4 - round 12 high
        eor     v10.16b, v10.16b, v4.16b                        //GHASH block 4k+1 - mid

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 5
        eor     x6, x6, x13                    //AES block 4k+4 - round 12 low

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 6
        shl     d8, d8, #56              //mod_constant

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 5
        eor     v9.16b, v9.16b, v5.16b                        //GHASH block 4k+3 - high

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 7
        fmov    d5, x19                              //AES block 4k+5 - mov low

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 7
        eor     v10.16b, v10.16b, v31.16b                        //GHASH block 4k+2 - mid

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 6
        fmov    v5.d[1], x20                          //AES block 4k+5 - mov high

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 8
        eor     v11.16b, v11.16b, v6.16b                        //GHASH block 4k+3 - low

        pmull   v30.1q, v30.1d, v16.1d                         //GHASH block 4k+3 - mid
        cmp     x0, x5                  //.LOOP CONTROL
        fmov    d4, x6                              //AES block 4k+4 - mov low

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 6
        fmov    v4.d[1], x7                          //AES block 4k+4 - mov high

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 8
        fmov    d7, x23                              //AES block 4k+3 - mov low

        eor     v10.16b, v10.16b, v30.16b                        //GHASH block 4k+3 - mid
        eor     v30.16b, v11.16b, v9.16b                        //MODULO - karatsuba tidy up
        add     w12, w12, #1                           //CTR block 4k+8

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 7
        fmov    v7.d[1], x24                          //AES block 4k+3 - mov high

        pmull   v31.1q, v9.1d, v8.1d           //MODULO - top 64b align with mid
        ext     v9.16b, v9.16b, v9.16b, #8                    //MODULO - other top alignment
        fmov    d6, x21                              //AES block 4k+6 - mov low

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 7

        aese    v0.16b, v27.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 9
        eor     v10.16b, v10.16b, v30.16b                        //MODULO - karatsuba tidy up

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 8

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 8

        aese    v1.16b, v27.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 9

        aese    v0.16b, v28.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 10
        eor     v10.16b, v10.16b, v31.16b                     //MODULO - fold into mid

        aese    v3.16b, v27.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 9

        aese    v2.16b, v27.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 9

        aese    v0.16b, v29.16b                                    //AES block 4k+4 - round 11

        aese    v1.16b, v28.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 10
        eor     v10.16b, v10.16b, v9.16b                        //MODULO - fold into mid

        aese    v2.16b, v28.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 10

        eor     v4.16b, v4.16b, v0.16b                         //AES block 4k+4 - result
        fmov    d0, x10                              //CTR block 4k+8

        aese    v1.16b, v29.16b                                    //AES block 4k+5 - round 11
        fmov    v0.d[1], x9                              //CTR block 4k+8
        rev     w9, w12                                //CTR block 4k+9

        pmull   v9.1q, v10.1d, v8.1d           //MODULO - mid 64b align with low
        fmov    v6.d[1], x22                          //AES block 4k+6 - mov high
        st1     { v4.16b}, [x2], #16                    //AES block 4k+4 - store result

        aese    v3.16b, v28.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 10
        orr     x9, x11, x9, lsl #32           //CTR block 4k+9

        eor     v5.16b, v5.16b, v1.16b                         //AES block 4k+5 - result
        add     w12, w12, #1                           //CTR block 4k+9
        fmov    d1, x10                              //CTR block 4k+9

        aese    v2.16b, v29.16b                                    //AES block 4k+6 - round 11
        fmov    v1.d[1], x9                              //CTR block 4k+9
        rev     w9, w12                                //CTR block 4k+10

        add     w12, w12, #1                           //CTR block 4k+10
        ext     v10.16b, v10.16b, v10.16b, #8                    //MODULO - other mid alignment
        orr     x9, x11, x9, lsl #32           //CTR block 4k+10

        st1     { v5.16b}, [x2], #16                    //AES block 4k+5 - store result
        eor     v11.16b, v11.16b, v9.16b                        //MODULO - fold into low

        aese    v3.16b, v29.16b                                    //AES block 4k+7 - round 11
        eor     v6.16b, v6.16b, v2.16b                         //AES block 4k+6 - result
        fmov    d2, x10                              //CTR block 4k+10

        st1     { v6.16b}, [x2], #16                    //AES block 4k+6 - store result
        fmov    v2.d[1], x9                              //CTR block 4k+10
        rev     w9, w12                                //CTR block 4k+11

        eor     v11.16b, v11.16b, v10.16b                        //MODULO - fold into low
        orr     x9, x11, x9, lsl #32           //CTR block 4k+11

        eor     v7.16b, v7.16b, v3.16b                         //AES block 4k+3 - result
        st1     { v7.16b}, [x2], #16                    //AES block 4k+3 - store result
        b.lt    .L192_enc_main_loop

.L192_enc_prepretail:   //PREPRETAIL
        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 0
        rev64   v4.16b, v4.16b                                   //GHASH block 4k (only t0 is free)

        fmov    d3, x10                              //CTR block 4k+3
        ext     v11.16b, v11.16b, v11.16b, #8                    //PRE 0
        add     w12, w12, #1                           //CTR block 4k+3

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 0
        rev64   v5.16b, v5.16b                                   //GHASH block 4k+1 (t0 and t1 free)

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 0

        fmov    v3.d[1], x9                              //CTR block 4k+3
        eor     v4.16b, v4.16b, v11.16b                          //PRE 1
        mov     d10, v17.d[1]                              //GHASH block 4k - mid

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 1
        rev64   v6.16b, v6.16b                                   //GHASH block 4k+2 (t0, t1, and t2 free)

        pmull2  v30.1q, v5.2d, v14.2d                         //GHASH block 4k+1 - high

        pmull   v11.1q, v4.1d, v15.1d                      //GHASH block 4k - low
        mov     d8, v4.d[1]                                 //GHASH block 4k - mid

        pmull   v31.1q, v5.1d, v14.1d                         //GHASH block 4k+1 - low
        rev64   v7.16b, v7.16b                                   //GHASH block 4k+3 (t0, t1, t2 and t3 free)

        pmull2  v9.1q, v4.2d, v15.2d                      //GHASH block 4k - high

        eor     v8.8b, v8.8b, v4.8b                         //GHASH block 4k - mid
        mov     d4, v5.d[1]                                 //GHASH block 4k+1 - mid

        eor     v11.16b, v11.16b, v31.16b                        //GHASH block 4k+1 - low
        mov     d31, v6.d[1]                                 //GHASH block 4k+2 - mid

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 0
        eor     v9.16b, v9.16b, v30.16b                        //GHASH block 4k+1 - high

        pmull2  v30.1q, v6.2d, v13.2d                         //GHASH block 4k+2 - high

        eor     v4.8b, v4.8b, v5.8b                         //GHASH block 4k+1 - mid
        eor     v31.8b, v31.8b, v6.8b                         //GHASH block 4k+2 - mid

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 1

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 1
        eor     v9.16b, v9.16b, v30.16b                        //GHASH block 4k+2 - high

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 1

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 2
        mov     d30, v7.d[1]                                 //GHASH block 4k+3 - mid

        pmull2  v5.1q, v7.2d, v12.2d                         //GHASH block 4k+3 - high
        ins     v31.d[1], v31.d[0]                               //GHASH block 4k+2 - mid

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 2

        pmull   v10.1q, v8.1d, v10.1d                     //GHASH block 4k - mid
        eor     v30.8b, v30.8b, v7.8b                         //GHASH block 4k+3 - mid

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 3

        pmull2  v31.1q, v31.2d, v16.2d                         //GHASH block 4k+2 - mid

        pmull   v4.1q, v4.1d, v17.1d                         //GHASH block 4k+1 - mid

        pmull   v30.1q, v30.1d, v16.1d                         //GHASH block 4k+3 - mid
        eor     v9.16b, v9.16b, v5.16b                        //GHASH block 4k+3 - high

        pmull   v8.1q, v6.1d, v13.1d                         //GHASH block 4k+2 - low

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 3
        eor     v10.16b, v10.16b, v4.16b                        //GHASH block 4k+1 - mid

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 2

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 2
        eor     v11.16b, v11.16b, v8.16b                        //GHASH block 4k+2 - low

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 4

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 3
        eor     v10.16b, v10.16b, v31.16b                        //GHASH block 4k+2 - mid

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 3

        pmull   v6.1q, v7.1d, v12.1d                         //GHASH block 4k+3 - low
        movi    v8.8b, #0xc2

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 4

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 4

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 4
        eor     v10.16b, v10.16b, v30.16b                        //GHASH block 4k+3 - mid

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 5

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 5

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 5
        eor     v11.16b, v11.16b, v6.16b                        //GHASH block 4k+3 - low

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 5

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 6
        eor     v10.16b, v10.16b, v9.16b                        //karatsuba tidy up

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 6

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 6
        shl     d8, d8, #56              //mod_constant

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 7

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 7
        eor     v10.16b, v10.16b, v11.16b

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 7

        pmull   v30.1q, v9.1d, v8.1d

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 6
        ext     v9.16b, v9.16b, v9.16b, #8

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 8

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 8
        eor     v10.16b, v10.16b, v30.16b

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 7

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 8

        aese    v0.16b, v27.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 9

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 8
        eor     v10.16b, v10.16b, v9.16b

        aese    v3.16b, v27.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 9

        aese    v1.16b, v27.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 9

        aese    v2.16b, v27.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 9

        pmull   v30.1q, v10.1d, v8.1d

        ext     v10.16b, v10.16b, v10.16b, #8

        aese    v3.16b, v28.16b
        aesmc   v3.16b, v3.16b         //AES block 4k+7 - round 10

        aese    v0.16b, v28.16b
        aesmc   v0.16b, v0.16b         //AES block 4k+4 - round 10

        aese    v2.16b, v28.16b
        aesmc   v2.16b, v2.16b         //AES block 4k+6 - round 10

        aese    v1.16b, v28.16b
        aesmc   v1.16b, v1.16b         //AES block 4k+5 - round 10
        eor     v11.16b, v11.16b, v30.16b

        aese    v0.16b, v29.16b                                    //AES block 4k+4 - round 11

        aese    v3.16b, v29.16b                                    //AES block 4k+7 - round 11

        aese    v2.16b, v29.16b                                    //AES block 4k+6 - round 11

        aese    v1.16b, v29.16b                                    //AES block 4k+5 - round 11
        eor     v11.16b, v11.16b, v10.16b
.L192_enc_tail: //TAIL

        sub     x5, x4, x0  //main_end_input_ptr is number of bytes left to process
        ldp     x6, x7, [x0], #16          //AES block 4k+4 - load plaintext
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        eor     x6, x6, x13                    //AES block 4k+4 - round 12 low
        eor     x7, x7, x14                    //AES block 4k+4 - round 12 high

        fmov    d4, x6                              //AES block 4k+4 - mov low

        fmov    v4.d[1], x7                          //AES block 4k+4 - mov high
        cmp     x5, #48

        eor     v5.16b, v4.16b, v0.16b                         //AES block 4k+4 - result

        ext     v8.16b, v11.16b, v11.16b, #8                    //prepare final partial tag
        b.gt    .L192_enc_blocks_more_than_3

        sub     w12, w12, #1
        movi    v10.8b, #0

        mov     v3.16b, v2.16b
        movi    v9.8b, #0
        cmp     x5, #32

        mov     v2.16b, v1.16b
        movi    v11.8b, #0
        b.gt    .L192_enc_blocks_more_than_2

        sub     w12, w12, #1

        mov     v3.16b, v1.16b
        cmp     x5, #16
        b.gt    .L192_enc_blocks_more_than_1

        sub     w12, w12, #1
        b       .L192_enc_blocks_less_than_1
.L192_enc_blocks_more_than_3:   //blocks        left >  3
        st1     { v5.16b}, [x2], #16                    //AES final-3 block  - store result

        ldp     x6, x7, [x0], #16          //AES final-2 block - load input low & high
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        rev64   v4.16b, v5.16b                                   //GHASH final-3 block

        eor     x6, x6, x13                    //AES final-2 block - round 12 low
        eor     v4.16b, v4.16b, v8.16b                          //feed in partial tag

        eor     x7, x7, x14                    //AES final-2 block - round 12 high
        fmov    d5, x6                                //AES final-2 block - mov low

        fmov    v5.d[1], x7                            //AES final-2 block - mov high

        mov     d22, v4.d[1]                                //GHASH final-3 block - mid

        pmull   v11.1q, v4.1d, v15.1d                      //GHASH final-3 block - low

        mov     d10, v17.d[1]                              //GHASH final-3 block - mid

        eor     v22.8b, v22.8b, v4.8b                     //GHASH final-3 block - mid

        movi    v8.8b, #0                                       //suppress further partial tag feed in

        pmull2  v9.1q, v4.2d, v15.2d                      //GHASH final-3 block - high

        pmull   v10.1q, v22.1d, v10.1d                   //GHASH final-3 block - mid
        eor     v5.16b, v5.16b, v1.16b                           //AES final-2 block - result
.L192_enc_blocks_more_than_2:   //blocks        left >  2

        st1     { v5.16b}, [x2], #16                    //AES final-2 block - store result

        rev64   v4.16b, v5.16b                                   //GHASH final-2 block
        ldp     x6, x7, [x0], #16          //AES final-1 block - load input low & high
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        eor     v4.16b, v4.16b, v8.16b                          //feed in partial tag

        eor     x7, x7, x14                    //AES final-1 block - round 12 high

        pmull2  v20.1q, v4.2d, v14.2d                         //GHASH final-2 block - high
        mov     d22, v4.d[1]                                //GHASH final-2 block - mid

        pmull   v21.1q, v4.1d, v14.1d                         //GHASH final-2 block - low
        eor     x6, x6, x13                    //AES final-1 block - round 12 low

        fmov    d5, x6                                //AES final-1 block - mov low

        fmov    v5.d[1], x7                            //AES final-1 block - mov high
        eor     v9.16b, v9.16b, v20.16b                           //GHASH final-2 block - high
        eor     v22.8b, v22.8b, v4.8b                     //GHASH final-2 block - mid

        eor     v11.16b, v11.16b, v21.16b                           //GHASH final-2 block - low

        pmull   v22.1q, v22.1d, v17.1d                     //GHASH final-2 block - mid

        movi    v8.8b, #0                                       //suppress further partial tag feed in

        eor     v5.16b, v5.16b, v2.16b                           //AES final-1 block - result

        eor     v10.16b, v10.16b, v22.16b                      //GHASH final-2 block - mid
.L192_enc_blocks_more_than_1:   //blocks        left >  1

        st1     { v5.16b}, [x2], #16                    //AES final-1 block - store result

        ldp     x6, x7, [x0], #16          //AES final block - load input low & high
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        rev64   v4.16b, v5.16b                                   //GHASH final-1 block

        eor     x6, x6, x13                    //AES final block - round 12 low
        eor     v4.16b, v4.16b, v8.16b                          //feed in partial tag
        movi    v8.8b, #0                                       //suppress further partial tag feed in

        mov     d22, v4.d[1]                                //GHASH final-1 block - mid

        eor     v22.8b, v22.8b, v4.8b                     //GHASH final-1 block - mid
        eor     x7, x7, x14                    //AES final block - round 12 high
        fmov    d5, x6                                //AES final block - mov low

        pmull2  v20.1q, v4.2d, v13.2d                         //GHASH final-1 block - high
        fmov    v5.d[1], x7                            //AES final block - mov high

        ins     v22.d[1], v22.d[0]                           //GHASH final-1 block - mid

        eor     v9.16b, v9.16b, v20.16b                           //GHASH final-1 block - high

        pmull   v21.1q, v4.1d, v13.1d                         //GHASH final-1 block - low

        pmull2  v22.1q, v22.2d, v16.2d                     //GHASH final-1 block - mid

        eor     v5.16b, v5.16b, v3.16b                           //AES final block - result

        eor     v11.16b, v11.16b, v21.16b                           //GHASH final-1 block - low

        eor     v10.16b, v10.16b, v22.16b                      //GHASH final-1 block - mid
.L192_enc_blocks_less_than_1:   //blocks        left <= 1

        ld1     { v18.16b}, [x2]                           //load existing bytes where the possibly partial last block is to be stored
#ifndef __AARCH64EB__
        rev     w9, w12
#else
        mov     w9, w12
#endif
        and     x1, x1, #127                   //bit_length %= 128

        sub     x1, x1, #128                   //bit_length -= 128
        mvn     x14, xzr                                     //rk12_h = 0xffffffffffffffff

        neg     x1, x1                         //bit_length = 128 - #bits in input (in range [1,128])
        mvn     x13, xzr                                     //rk12_l = 0xffffffffffffffff

        and     x1, x1, #127                   //bit_length %= 128

        lsr     x14, x14, x1                    //rk12_h is mask for top 64b of last block
        cmp     x1, #64

        csel    x6, x13, x14, lt
        csel    x7, x14, xzr, lt

        fmov    d0, x6                                //ctr0b is mask for last block

        fmov    v0.d[1], x7

        and     v5.16b, v5.16b, v0.16b                           //possibly partial last block has zeroes in highest bits

        rev64   v4.16b, v5.16b                                   //GHASH final block

        eor     v4.16b, v4.16b, v8.16b                          //feed in partial tag

        mov     d8, v4.d[1]                                 //GHASH final block - mid

        pmull   v21.1q, v4.1d, v12.1d                         //GHASH final block - low

        pmull2  v20.1q, v4.2d, v12.2d                         //GHASH final block - high

        eor     v8.8b, v8.8b, v4.8b                         //GHASH final block - mid

        eor     v11.16b, v11.16b, v21.16b                           //GHASH final block - low

        eor     v9.16b, v9.16b, v20.16b                           //GHASH final block - high

        pmull   v8.1q, v8.1d, v16.1d                         //GHASH final block - mid

        eor     v10.16b, v10.16b, v8.16b                        //GHASH final block - mid
        movi    v8.8b, #0xc2

        eor     v30.16b, v11.16b, v9.16b                        //MODULO - karatsuba tidy up

        shl     d8, d8, #56              //mod_constant

        bif     v5.16b, v18.16b, v0.16b                             //insert existing bytes in top end of result before storing

        eor     v10.16b, v10.16b, v30.16b                        //MODULO - karatsuba tidy up

        pmull   v31.1q, v9.1d, v8.1d           //MODULO - top 64b align with mid

        ext     v9.16b, v9.16b, v9.16b, #8                    //MODULO - other top alignment

        eor     v10.16b, v10.16b, v31.16b                     //MODULO - fold into mid

        eor     v10.16b, v10.16b, v9.16b                        //MODULO - fold into mid

        pmull   v9.1q, v10.1d, v8.1d           //MODULO - mid 64b align with low

        ext     v10.16b, v10.16b, v10.16b, #8                    //MODULO - other mid alignment

        eor     v11.16b, v11.16b, v9.16b                        //MODULO - fold into low
        str     w9, [x16, #12]                         //store the updated counter

        st1     { v5.16b}, [x2]                         //store all 16B

        eor     v11.16b, v11.16b, v10.16b                        //MODULO - fold into low
        ext     v11.16b, v11.16b, v11.16b, #8
        rev64   v11.16b, v11.16b
        mov     x0, x15
        st1     { v11.16b }, [x3]

        ldp     x21, x22, [sp, #16]
        ldp     x23, x24, [sp, #32]
        ldp     d8, d9, [sp, #48]
        ldp     d10, d11, [sp, #64]
        ldp     d12, d13, [sp, #80]
        ldp     d14, d15, [sp, #96]
        ldp     x19, x20, [sp], #112
        ret

.L192_enc_ret:
        mov     w0, #0x0
        ret
.size   aes_gcm_enc_192_kernel,.-aes_gcm_enc_192_kernel
.globl  aes_gcm_dec_192_kernel
.type   aes_gcm_dec_192_kernel,%function
.align  4
aes_gcm_dec_192_kernel:
        AARCH64_VALID_CALL_TARGET
        cbz     x1, .L192_dec_ret
        stp     x19, x20, [sp, #-112]!
        mov     x16, x4
        mov     x8, x5
        stp     x21, x22, [sp, #16]
        stp     x23, x24, [sp, #32]
        stp     d8, d9, [sp, #48]
        stp     d10, d11, [sp, #64]
        stp     d12, d13, [sp, #80]
        stp     d14, d15, [sp, #96]

        add     x4, x0, x1, lsr #3   //end_input_ptr
        ldp     x10, x11, [x16]              //ctr96_b64, ctr96_t32
#ifdef __AARCH64EB__
        rev     x10, x10
        rev     x11, x11
#endif
        ldp     x13, x14, [x8, #192]                     //load rk12
#ifdef __AARCH64EB__
        ror     x13, x13, #32
        ror     x14, x14, #32
#endif
        ld1     { v0.16b}, [x16]                             //special case vector load initial counter so we can start first AES block as quickly as possible

        ld1     {v18.4s}, [x8], #16                                  //load rk0

        lsr     x5, x1, #3              //byte_len
        mov     x15, x5
        ld1     {v19.4s}, [x8], #16                               //load rk1

        lsr     x12, x11, #32
        orr     w11, w11, w11
        fmov    d3, x10                               //CTR block 3

        rev     w12, w12                                //rev_ctr32
        fmov    d1, x10                               //CTR block 1

        add     w12, w12, #1                            //increment rev_ctr32
        ld1     {v20.4s}, [x8], #16                               //load rk2

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 0
        rev     w9, w12                                 //CTR block 1

        add     w12, w12, #1                            //CTR block 1
        orr     x9, x11, x9, lsl #32            //CTR block 1
        ld1     {v21.4s}, [x8], #16                               //load rk3

        fmov    v1.d[1], x9                               //CTR block 1
        rev     w9, w12                                 //CTR block 2
        add     w12, w12, #1                            //CTR block 2

        fmov    d2, x10                               //CTR block 2
        orr     x9, x11, x9, lsl #32            //CTR block 2

        fmov    v2.d[1], x9                               //CTR block 2
        rev     w9, w12                                 //CTR block 3

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 1
        orr     x9, x11, x9, lsl #32            //CTR block 3

        fmov    v3.d[1], x9                               //CTR block 3

        ld1     {v22.4s}, [x8], #16                               //load rk4

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 2

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 0
        ld1     {v23.4s}, [x8], #16                               //load rk5

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 0
        ldr     q15, [x3, #112]                        //load h4l | h4h
#ifndef __AARCH64EB__
        ext     v15.16b, v15.16b, v15.16b, #8
#endif
        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 0
        ldr     q13, [x3, #64]                         //load h2l | h2h
#ifndef __AARCH64EB__
        ext     v13.16b, v13.16b, v13.16b, #8
#endif
        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 1
        ldr     q14, [x3, #80]                         //load h3l | h3h
#ifndef __AARCH64EB__
        ext     v14.16b, v14.16b, v14.16b, #8
#endif
        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 1

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 1
        ldr     q12, [x3, #32]                         //load h1l | h1h
#ifndef __AARCH64EB__
        ext     v12.16b, v12.16b, v12.16b, #8
#endif
        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 2
        ld1     {v24.4s}, [x8], #16                               //load rk6

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 3
        ld1     {v25.4s}, [x8], #16                               //load rk7

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 2
        ld1     {v26.4s}, [x8], #16                               //load rk8

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 2
        ld1     {v27.4s}, [x8], #16                               //load rk9

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 3
        ld1     { v11.16b}, [x3]
        ext     v11.16b, v11.16b, v11.16b, #8
        rev64   v11.16b, v11.16b

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 3
        add     w12, w12, #1                            //CTR block 3

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 3
        trn1    v9.2d, v14.2d,    v15.2d                      //h4h | h3h

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 4
        ld1     {v28.4s}, [x8], #16                              //load rk10

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 4
        trn2    v17.2d,  v14.2d,    v15.2d                      //h4l | h3l

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 4

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 4
        trn2    v16.2d,  v12.2d,    v13.2d                      //h2l | h1l

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 5
        ld1     {v29.4s}, [x8], #16                              //load rk11

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 5

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 5

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 5

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 6

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 6

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 6

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 7

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 7

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 7

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 6

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 8

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 8

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 7

        aese    v2.16b, v27.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 9

        aese    v3.16b, v27.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 9

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 8
        sub     x5, x5, #1      //byte_len - 1

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 8
        and     x5, x5, #0xffffffffffffffc0    //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

        aese    v3.16b, v28.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 10
        add     x5, x5, x0

        aese    v1.16b, v27.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 9
        cmp     x0, x5                   //check if we have <= 4 blocks

        aese    v0.16b, v27.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 9
        trn1    v8.2d,    v12.2d,    v13.2d                      //h2h | h1h

        aese    v3.16b, v29.16b                                     //AES block 3 - round 11

        aese    v2.16b, v28.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 10

        aese    v1.16b, v28.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 10

        aese    v0.16b, v28.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 10
        eor     v16.16b, v16.16b, v8.16b                     //h2k | h1k

        aese    v2.16b, v29.16b                                     //AES block 2 - round 11

        aese    v1.16b, v29.16b                                     //AES block 1 - round 11
        eor     v17.16b, v17.16b, v9.16b                  //h4k | h3k

        aese    v0.16b, v29.16b                                     //AES block 0 - round 11
        b.ge    .L192_dec_tail                                    //handle tail

        ld1     {v4.16b, v5.16b}, [x0], #32               //AES block 0,1 - load ciphertext

        eor     v1.16b, v5.16b, v1.16b                            //AES block 1 - result

        eor     v0.16b, v4.16b, v0.16b                            //AES block 0 - result
        rev     w9, w12                                 //CTR block 4
        ld1     {v6.16b, v7.16b}, [x0], #32               //AES block 2,3 - load ciphertext

        mov     x19, v1.d[0]                            //AES block 1 - mov low

        mov     x20, v1.d[1]                            //AES block 1 - mov high

        mov     x6, v0.d[0]                            //AES block 0 - mov low
        orr     x9, x11, x9, lsl #32            //CTR block 4
        add     w12, w12, #1                            //CTR block 4

        mov     x7, v0.d[1]                            //AES block 0 - mov high
        rev64   v4.16b, v4.16b                                    //GHASH block 0

        fmov    d0, x10                               //CTR block 4
        rev64   v5.16b, v5.16b                                    //GHASH block 1
        cmp     x0, x5                   //check if we have <= 8 blocks

        eor     x19, x19, x13                   //AES block 1 - round 12 low
#ifdef __AARCH64EB__
        rev     x19, x19
#endif
        fmov    v0.d[1], x9                               //CTR block 4
        rev     w9, w12                                 //CTR block 5

        orr     x9, x11, x9, lsl #32            //CTR block 5
        fmov    d1, x10                               //CTR block 5
        eor     x20, x20, x14                   //AES block 1 - round 12 high
#ifdef __AARCH64EB__
        rev     x20, x20
#endif
        add     w12, w12, #1                            //CTR block 5
        fmov    v1.d[1], x9                               //CTR block 5
        eor     x6, x6, x13                   //AES block 0 - round 12 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        rev     w9, w12                                 //CTR block 6
        eor     x7, x7, x14                   //AES block 0 - round 12 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        stp     x6, x7, [x2], #16        //AES block 0 - store result
        orr     x9, x11, x9, lsl #32            //CTR block 6

        stp     x19, x20, [x2], #16        //AES block 1 - store result

        add     w12, w12, #1                            //CTR block 6
        eor     v2.16b, v6.16b, v2.16b                            //AES block 2 - result
        b.ge    .L192_dec_prepretail                              //do prepretail

.L192_dec_main_loop:    //main  loop start
        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 0
        ext     v11.16b, v11.16b, v11.16b, #8                     //PRE 0

        pmull   v31.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low
        mov     x21, v2.d[0]                            //AES block 4k+2 - mov low

        mov     x22, v2.d[1]                            //AES block 4k+2 - mov high
        eor     v3.16b, v7.16b, v3.16b                            //AES block 4k+3 - result
        rev64   v7.16b, v7.16b                                    //GHASH block 4k+3

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 1
        fmov    d2, x10                               //CTR block 4k+6

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 0
        eor     v4.16b, v4.16b, v11.16b                           //PRE 1

        pmull2  v30.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high
        fmov    v2.d[1], x9                               //CTR block 4k+6

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 2
        mov     x24, v3.d[1]                            //AES block 4k+3 - mov high

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 1
        mov     x23, v3.d[0]                            //AES block 4k+3 - mov low

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
        fmov    d3, x10                               //CTR block 4k+7
        mov     d8, v4.d[1]                                  //GHASH block 4k - mid

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
        mov     d10, v17.d[1]                               //GHASH block 4k - mid
        rev     w9, w12                                 //CTR block 4k+7

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 0
        orr     x9, x11, x9, lsl #32            //CTR block 4k+7

        fmov    v3.d[1], x9                               //CTR block 4k+7
        eor     v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid
        mov     d4, v5.d[1]                                  //GHASH block 4k+1 - mid

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 3

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 2
        eor     x22, x22, x14                   //AES block 4k+2 - round 12 high
#ifdef __AARCH64EB__
        rev     x22, x22
#endif
        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 1
        eor     v4.8b, v4.8b, v5.8b                          //GHASH block 4k+1 - mid

        pmull   v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 0
        rev64   v6.16b, v6.16b                                    //GHASH block 4k+2

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 2

        pmull   v4.1q, v4.1d, v17.1d                          //GHASH block 4k+1 - mid
        eor     v11.16b, v11.16b, v31.16b                         //GHASH block 4k+1 - low
        eor     x21, x21, x13                   //AES block 4k+2 - round 12 low
#ifdef __AARCH64EB__
        rev     x21, x21
#endif
        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 4

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 3

        eor     v10.16b, v10.16b, v4.16b                         //GHASH block 4k+1 - mid
        mov     d31, v6.d[1]                                  //GHASH block 4k+2 - mid

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 1
        eor     v9.16b, v9.16b, v30.16b                         //GHASH block 4k+1 - high

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 4

        pmull2  v30.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high
        eor     v31.8b, v31.8b, v6.8b                          //GHASH block 4k+2 - mid

        pmull   v8.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 5

        eor     v9.16b, v9.16b, v30.16b                         //GHASH block 4k+2 - high
        mov     d30, v7.d[1]                                  //GHASH block 4k+3 - mid

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 5

        pmull2  v5.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 2
        eor     v30.8b, v30.8b, v7.8b                          //GHASH block 4k+3 - mid

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 6

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 6
        ins     v31.d[1], v31.d[0]                                //GHASH block 4k+2 - mid

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 3

        pmull   v30.1q, v30.1d, v16.1d                          //GHASH block 4k+3 - mid
        eor     v11.16b, v11.16b, v8.16b                         //GHASH block 4k+2 - low

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 7

        pmull2  v31.1q, v31.2d, v16.2d                          //GHASH block 4k+2 - mid
        eor     v9.16b, v9.16b, v5.16b                         //GHASH block 4k+3 - high

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 7

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 8
        movi    v8.8b, #0xc2

        pmull   v6.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 8
        eor     v10.16b, v10.16b, v31.16b                         //GHASH block 4k+2 - mid

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 3

        aese    v0.16b, v27.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 9
        eor     v11.16b, v11.16b, v6.16b                         //GHASH block 4k+3 - low

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 4

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 4
        eor     v10.16b, v10.16b, v30.16b                         //GHASH block 4k+3 - mid

        aese    v0.16b, v28.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 10

        aese    v1.16b, v27.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 9
        eor     v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 5

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 5
        shl     d8, d8, #56               //mod_constant

        aese    v1.16b, v28.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 10

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 6
        ld1     {v4.16b}, [x0], #16                       //AES block 4k+4 - load ciphertext

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 6
        eor     v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

        pmull   v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
        ld1     {v5.16b}, [x0], #16                       //AES block 4k+5 - load ciphertext
        eor     x23, x23, x13                   //AES block 4k+3 - round 12 low
#ifdef __AARCH64EB__
        rev     x23, x23
#endif
        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 7
        ext     v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

        aese    v0.16b, v29.16b                                     //AES block 4k+4 - round 11
        add     w12, w12, #1                            //CTR block 4k+7

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 7
        eor     v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 8
        ld1     {v6.16b}, [x0], #16                       //AES block 4k+6 - load ciphertext

        aese    v1.16b, v29.16b                                     //AES block 4k+5 - round 11
        ld1     {v7.16b}, [x0], #16                       //AES block 4k+7 - load ciphertext
        rev     w9, w12                                 //CTR block 4k+8

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 8
        stp     x21, x22, [x2], #16        //AES block 4k+2 - store result

        aese    v2.16b, v27.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 9
        eor     v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

        cmp     x0, x5                   //.LOOP CONTROL

        eor     v0.16b, v4.16b, v0.16b                            //AES block 4k+4 - result
        eor     x24, x24, x14                   //AES block 4k+3 - round 12 high
#ifdef __AARCH64EB__
        rev     x24, x24
#endif
        eor     v1.16b, v5.16b, v1.16b                            //AES block 4k+5 - result

        aese    v2.16b, v28.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 10
        orr     x9, x11, x9, lsl #32            //CTR block 4k+8

        aese    v3.16b, v27.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 9

        pmull   v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low
        mov     x19, v1.d[0]                            //AES block 4k+5 - mov low

        mov     x6, v0.d[0]                            //AES block 4k+4 - mov low
        stp     x23, x24, [x2], #16        //AES block 4k+3 - store result
        rev64   v5.16b, v5.16b                                    //GHASH block 4k+5

        aese    v2.16b, v29.16b                                     //AES block 4k+6 - round 11
        mov     x7, v0.d[1]                            //AES block 4k+4 - mov high

        aese    v3.16b, v28.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 10
        mov     x20, v1.d[1]                            //AES block 4k+5 - mov high

        fmov    d0, x10                               //CTR block 4k+8
        add     w12, w12, #1                            //CTR block 4k+8
        ext     v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

        eor     v2.16b, v6.16b, v2.16b                            //AES block 4k+6 - result
        fmov    v0.d[1], x9                               //CTR block 4k+8
        rev     w9, w12                                 //CTR block 4k+9

        eor     x6, x6, x13                   //AES block 4k+4 - round 12 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        orr     x9, x11, x9, lsl #32            //CTR block 4k+9
        eor     v11.16b, v11.16b, v8.16b               //MODULO - fold into low

        fmov    d1, x10                               //CTR block 4k+9
        add     w12, w12, #1                            //CTR block 4k+9
        eor     x19, x19, x13                   //AES block 4k+5 - round 12 low
#ifdef __AARCH64EB__
        rev     x19, x19
#endif
        fmov    v1.d[1], x9                               //CTR block 4k+9
        rev     w9, w12                                 //CTR block 4k+10
        eor     x20, x20, x14                   //AES block 4k+5 - round 12 high
#ifdef __AARCH64EB__
        rev     x20, x20
#endif
        eor     x7, x7, x14                   //AES block 4k+4 - round 12 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        stp     x6, x7, [x2], #16        //AES block 4k+4 - store result
        eor     v11.16b, v11.16b, v10.16b                         //MODULO - fold into low

        add     w12, w12, #1                            //CTR block 4k+10
        rev64   v4.16b, v4.16b                                    //GHASH block 4k+4
        orr     x9, x11, x9, lsl #32            //CTR block 4k+10

        aese    v3.16b, v29.16b                                     //AES block 4k+7 - round 11
        stp     x19, x20, [x2], #16        //AES block 4k+5 - store result
        b.lt    .L192_dec_main_loop

.L192_dec_prepretail:   //PREPRETAIL
        mov     x22, v2.d[1]                            //AES block 4k+2 - mov high
        ext     v11.16b, v11.16b, v11.16b, #8                     //PRE 0
        eor     v3.16b, v7.16b, v3.16b                            //AES block 4k+3 - result

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 0
        mov     x21, v2.d[0]                            //AES block 4k+2 - mov low

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 0
        mov     d10, v17.d[1]                               //GHASH block 4k - mid

        eor     v4.16b, v4.16b, v11.16b                           //PRE 1
        fmov    d2, x10                               //CTR block 4k+6

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 1
        mov     x23, v3.d[0]                            //AES block 4k+3 - mov low

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 1
        mov     x24, v3.d[1]                            //AES block 4k+3 - mov high

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
        mov     d8, v4.d[1]                                  //GHASH block 4k - mid
        fmov    d3, x10                               //CTR block 4k+7

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 2
        rev64   v6.16b, v6.16b                                    //GHASH block 4k+2

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
        fmov    v2.d[1], x9                               //CTR block 4k+6
        rev     w9, w12                                 //CTR block 4k+7

        orr     x9, x11, x9, lsl #32            //CTR block 4k+7
        eor     v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid
        mov     d4, v5.d[1]                                  //GHASH block 4k+1 - mid

        pmull   v31.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low
        eor     x24, x24, x14                   //AES block 4k+3 - round 12 high
#ifdef __AARCH64EB__
        rev     x24, x24
#endif
        fmov    v3.d[1], x9                               //CTR block 4k+7

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 2
        eor     x21, x21, x13                   //AES block 4k+2 - round 12 low
#ifdef __AARCH64EB__
        rev     x21, x21
#endif
        pmull2  v30.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high
        eor     x22, x22, x14                   //AES block 4k+2 - round 12 high
#ifdef __AARCH64EB__
        rev     x22, x22
#endif
        eor     v4.8b, v4.8b, v5.8b                          //GHASH block 4k+1 - mid

        pmull   v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid
        eor     x23, x23, x13                   //AES block 4k+3 - round 12 low
#ifdef __AARCH64EB__
        rev     x23, x23
#endif
        stp     x21, x22, [x2], #16        //AES block 4k+2 - store result

        rev64   v7.16b, v7.16b                                    //GHASH block 4k+3
        stp     x23, x24, [x2], #16        //AES block 4k+3 - store result

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 0
        eor     v9.16b, v9.16b, v30.16b                         //GHASH block 4k+1 - high

        pmull   v4.1q, v4.1d, v17.1d                          //GHASH block 4k+1 - mid
        add     w12, w12, #1                            //CTR block 4k+7

        pmull2  v30.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high
        eor     v11.16b, v11.16b, v31.16b                         //GHASH block 4k+1 - low

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 0

        eor     v10.16b, v10.16b, v4.16b                         //GHASH block 4k+1 - mid
        mov     d31, v6.d[1]                                  //GHASH block 4k+2 - mid

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 1

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 1
        eor     v9.16b, v9.16b, v30.16b                         //GHASH block 4k+2 - high

        eor     v31.8b, v31.8b, v6.8b                          //GHASH block 4k+2 - mid

        pmull   v8.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 2
        mov     d30, v7.d[1]                                  //GHASH block 4k+3 - mid

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 2
        ins     v31.d[1], v31.d[0]                                //GHASH block 4k+2 - mid

        pmull   v6.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 3
        eor     v30.8b, v30.8b, v7.8b                          //GHASH block 4k+3 - mid

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 3

        pmull2  v31.1q, v31.2d, v16.2d                          //GHASH block 4k+2 - mid
        eor     v11.16b, v11.16b, v8.16b                         //GHASH block 4k+2 - low

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 4

        pmull2  v5.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high
        movi    v8.8b, #0xc2

        pmull   v30.1q, v30.1d, v16.1d                          //GHASH block 4k+3 - mid

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 3

        shl     d8, d8, #56               //mod_constant
        eor     v9.16b, v9.16b, v5.16b                         //GHASH block 4k+3 - high

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 5
        eor     v10.16b, v10.16b, v31.16b                         //GHASH block 4k+2 - mid

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 4

        pmull   v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
        eor     v11.16b, v11.16b, v6.16b                         //GHASH block 4k+3 - low

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 6

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 3
        eor     v10.16b, v10.16b, v30.16b                         //GHASH block 4k+3 - mid

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 5

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 7
        eor     v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 4

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 6
        ext     v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 8

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 5
        eor     v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 4

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 7

        aese    v0.16b, v27.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 9

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 5

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 6
        eor     v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

        aese    v0.16b, v28.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 10

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 6

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 7

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 8
        eor     v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 7

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 8

        aese    v2.16b, v27.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 9

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 8

        aese    v3.16b, v27.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 9

        pmull   v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low

        aese    v1.16b, v27.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 9

        aese    v2.16b, v28.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 10

        aese    v3.16b, v28.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 10
        ext     v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

        aese    v1.16b, v28.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 10

        aese    v0.16b, v29.16b
        eor     v11.16b, v11.16b, v8.16b               //MODULO - fold into low

        aese    v2.16b, v29.16b

        aese    v1.16b, v29.16b

        aese    v3.16b, v29.16b

        eor     v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
.L192_dec_tail: //TAIL

        sub     x5, x4, x0   //main_end_input_ptr is number of bytes left to process
        ld1     { v5.16b}, [x0], #16                      //AES block 4k+4 - load ciphertext

        eor     v0.16b, v5.16b, v0.16b                            //AES block 4k+4 - result

        mov     x7, v0.d[1]                            //AES block 4k+4 - mov high

        mov     x6, v0.d[0]                            //AES block 4k+4 - mov low

        ext     v8.16b, v11.16b, v11.16b, #8                     //prepare final partial tag

        cmp     x5, #48

        eor     x7, x7, x14                   //AES block 4k+4 - round 12 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        eor     x6, x6, x13                   //AES block 4k+4 - round 12 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        b.gt    .L192_dec_blocks_more_than_3

        movi    v11.8b, #0
        movi    v9.8b, #0

        mov     v3.16b, v2.16b
        mov     v2.16b, v1.16b
        sub     w12, w12, #1

        movi    v10.8b, #0
        cmp     x5, #32
        b.gt    .L192_dec_blocks_more_than_2

        mov     v3.16b, v1.16b
        cmp     x5, #16
        sub     w12, w12, #1

        b.gt    .L192_dec_blocks_more_than_1

        sub     w12, w12, #1
        b       .L192_dec_blocks_less_than_1
.L192_dec_blocks_more_than_3:   //blocks        left >  3
        rev64   v4.16b, v5.16b                                    //GHASH final-3 block
        ld1     { v5.16b}, [x0], #16                      //AES final-2 block - load ciphertext

        stp     x6, x7, [x2], #16        //AES final-3 block  - store result

        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag

        eor     v0.16b, v5.16b, v1.16b                            //AES final-2 block - result

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH final-3 block - low
        mov     x6, v0.d[0]                            //AES final-2 block - mov low
        mov     d22, v4.d[1]                                 //GHASH final-3 block - mid

        mov     x7, v0.d[1]                            //AES final-2 block - mov high

        mov     d10, v17.d[1]                               //GHASH final-3 block - mid
        eor     v22.8b, v22.8b, v4.8b                      //GHASH final-3 block - mid

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH final-3 block - high

        eor     x6, x6, x13                   //AES final-2 block - round 12 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        movi    v8.8b, #0                                        //suppress further partial tag feed in

        pmull   v10.1q, v22.1d, v10.1d                    //GHASH final-3 block - mid
        eor     x7, x7, x14                   //AES final-2 block - round 12 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
.L192_dec_blocks_more_than_2:   //blocks        left >  2

        rev64   v4.16b, v5.16b                                    //GHASH final-2 block
        ld1     { v5.16b}, [x0], #16                      //AES final-1 block - load ciphertext

        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag

        movi    v8.8b, #0                                        //suppress further partial tag feed in

        eor     v0.16b, v5.16b, v2.16b                            //AES final-1 block - result

        mov     d22, v4.d[1]                                 //GHASH final-2 block - mid

        pmull   v21.1q, v4.1d, v14.1d                          //GHASH final-2 block - low

        stp     x6, x7, [x2], #16        //AES final-2 block  - store result

        eor     v22.8b, v22.8b, v4.8b                      //GHASH final-2 block - mid
        mov     x7, v0.d[1]                            //AES final-1 block - mov high

        eor     v11.16b, v11.16b, v21.16b                            //GHASH final-2 block - low
        mov     x6, v0.d[0]                            //AES final-1 block - mov low

        pmull2  v20.1q, v4.2d, v14.2d                          //GHASH final-2 block - high

        pmull   v22.1q, v22.1d, v17.1d                      //GHASH final-2 block - mid

        eor     v9.16b, v9.16b, v20.16b                            //GHASH final-2 block - high
        eor     x7, x7, x14                   //AES final-1 block - round 12 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        eor     x6, x6, x13                   //AES final-1 block - round 12 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        eor     v10.16b, v10.16b, v22.16b                       //GHASH final-2 block - mid
.L192_dec_blocks_more_than_1:   //blocks        left >  1

        rev64   v4.16b, v5.16b                                    //GHASH final-1 block

        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag
        ld1     { v5.16b}, [x0], #16                      //AES final block - load ciphertext

        mov     d22, v4.d[1]                                 //GHASH final-1 block - mid

        pmull2  v20.1q, v4.2d, v13.2d                          //GHASH final-1 block - high

        eor     v0.16b, v5.16b, v3.16b                            //AES final block - result
        stp     x6, x7, [x2], #16        //AES final-1 block  - store result

        eor     v22.8b, v22.8b, v4.8b                      //GHASH final-1 block - mid

        eor     v9.16b, v9.16b, v20.16b                            //GHASH final-1 block - high

        pmull   v21.1q, v4.1d, v13.1d                          //GHASH final-1 block - low
        mov     x7, v0.d[1]                            //AES final block - mov high

        ins     v22.d[1], v22.d[0]                            //GHASH final-1 block - mid
        mov     x6, v0.d[0]                            //AES final block - mov low

        pmull2  v22.1q, v22.2d, v16.2d                      //GHASH final-1 block - mid

        movi    v8.8b, #0                                        //suppress further partial tag feed in
        eor     v11.16b, v11.16b, v21.16b                            //GHASH final-1 block - low
        eor     x7, x7, x14                   //AES final block - round 12 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        eor     x6, x6, x13                   //AES final block - round 12 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        eor     v10.16b, v10.16b, v22.16b                       //GHASH final-1 block - mid
.L192_dec_blocks_less_than_1:   //blocks        left <= 1

        mvn     x13, xzr                                      //rk12_l = 0xffffffffffffffff
        ldp     x4, x5, [x2]  //load existing bytes we need to not overwrite
        and     x1, x1, #127                    //bit_length %= 128

        sub     x1, x1, #128                    //bit_length -= 128

        neg     x1, x1                          //bit_length = 128 - #bits in input (in range [1,128])

        and     x1, x1, #127                    //bit_length %= 128
        mvn     x14, xzr                                      //rk12_h = 0xffffffffffffffff

        lsr     x14, x14, x1                     //rk12_h is mask for top 64b of last block
        cmp     x1, #64

        csel    x9, x13, x14, lt
        csel    x10, x14, xzr, lt

        fmov    d0, x9                                   //ctr0b is mask for last block
        and     x6, x6, x9
        bic     x4, x4, x9           //mask out low existing bytes

        orr     x6, x6, x4
        mov     v0.d[1], x10
#ifndef __AARCH64EB__
        rev     w9, w12
#else
        mov     w9, w12
#endif

        and     v5.16b, v5.16b, v0.16b                            //possibly partial last block has zeroes in highest bits
        str     w9, [x16, #12]                          //store the updated counter

        rev64   v4.16b, v5.16b                                    //GHASH final block

        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag
        bic     x5, x5, x10 //mask out high existing bytes

        and     x7, x7, x10

        pmull2  v20.1q, v4.2d, v12.2d                          //GHASH final block - high
        mov     d8, v4.d[1]                                  //GHASH final block - mid

        pmull   v21.1q, v4.1d, v12.1d                          //GHASH final block - low

        eor     v8.8b, v8.8b, v4.8b                          //GHASH final block - mid

        eor     v9.16b, v9.16b, v20.16b                            //GHASH final block - high

        pmull   v8.1q, v8.1d, v16.1d                          //GHASH final block - mid

        eor     v11.16b, v11.16b, v21.16b                            //GHASH final block - low

        eor     v10.16b, v10.16b, v8.16b                         //GHASH final block - mid
        movi    v8.8b, #0xc2

        eor     v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

        shl     d8, d8, #56               //mod_constant

        eor     v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

        pmull   v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
        orr     x7, x7, x5
        stp     x6, x7, [x2]

        ext     v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

        eor     v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

        eor     v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

        pmull   v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low

        eor     v11.16b, v11.16b, v8.16b               //MODULO - fold into low

        ext     v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

        eor     v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
        ext     v11.16b, v11.16b, v11.16b, #8
        rev64   v11.16b, v11.16b
        mov     x0, x15
        st1     { v11.16b }, [x3]

        ldp     x21, x22, [sp, #16]
        ldp     x23, x24, [sp, #32]
        ldp     d8, d9, [sp, #48]
        ldp     d10, d11, [sp, #64]
        ldp     d12, d13, [sp, #80]
        ldp     d14, d15, [sp, #96]
        ldp     x19, x20, [sp], #112
        ret

.L192_dec_ret:
        mov     w0, #0x0
        ret
.size   aes_gcm_dec_192_kernel,.-aes_gcm_dec_192_kernel
.globl  aes_gcm_enc_256_kernel
.type   aes_gcm_enc_256_kernel,%function
.align  4
aes_gcm_enc_256_kernel:
        AARCH64_VALID_CALL_TARGET
        cbz     x1, .L256_enc_ret
        stp     x19, x20, [sp, #-112]!
        mov     x16, x4
        mov     x8, x5
        stp     x21, x22, [sp, #16]
        stp     x23, x24, [sp, #32]
        stp     d8, d9, [sp, #48]
        stp     d10, d11, [sp, #64]
        stp     d12, d13, [sp, #80]
        stp     d14, d15, [sp, #96]

        add     x4, x0, x1, lsr #3   //end_input_ptr
        lsr     x5, x1, #3              //byte_len
        mov     x15, x5
        ldp     x10, x11, [x16]              //ctr96_b64, ctr96_t32
#ifdef __AARCH64EB__
        rev     x10, x10
        rev     x11, x11
#endif
        ldp     x13, x14, [x8, #224]                     //load rk14
#ifdef __AARCH64EB__
        ror     x13, x13, #32
        ror     x14, x14, #32
#endif
        ld1     { v0.16b}, [x16]                             //special case vector load initial counter so we can start first AES block as quickly as possible
        sub     x5, x5, #1      //byte_len - 1

        ld1     {v18.4s}, [x8], #16                               //load rk0
        and     x5, x5, #0xffffffffffffffc0 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

        ld1     {v19.4s}, [x8], #16                               //load rk1
        add     x5, x5, x0

        lsr     x12, x11, #32
        fmov    d2, x10                               //CTR block 2
        orr     w11, w11, w11

        rev     w12, w12                                //rev_ctr32
        cmp     x0, x5                   //check if we have <= 4 blocks
        fmov    d1, x10                               //CTR block 1

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 0
        add     w12, w12, #1                            //increment rev_ctr32

        rev     w9, w12                                 //CTR block 1
        fmov    d3, x10                               //CTR block 3

        orr     x9, x11, x9, lsl #32            //CTR block 1
        add     w12, w12, #1                            //CTR block 1
        ld1     {v20.4s}, [x8], #16                               //load rk2

        fmov    v1.d[1], x9                               //CTR block 1
        rev     w9, w12                                 //CTR block 2
        add     w12, w12, #1                            //CTR block 2

        orr     x9, x11, x9, lsl #32            //CTR block 2
        ld1     {v21.4s}, [x8], #16                               //load rk3

        fmov    v2.d[1], x9                               //CTR block 2
        rev     w9, w12                                 //CTR block 3

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 1
        orr     x9, x11, x9, lsl #32            //CTR block 3

        fmov    v3.d[1], x9                               //CTR block 3

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 0
        ld1     {v22.4s}, [x8], #16                               //load rk4

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 2
        ld1     {v23.4s}, [x8], #16                               //load rk5

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 0
        ld1     {v24.4s}, [x8], #16                               //load rk6

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 1
        ldr     q14, [x3, #80]                         //load h3l | h3h
#ifndef __AARCH64EB__
        ext     v14.16b, v14.16b, v14.16b, #8
#endif
        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 0
        ld1     {v25.4s}, [x8], #16                               //load rk7

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 1
        ld1     {v26.4s}, [x8], #16                               //load rk8

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 2
        ldr     q13, [x3, #64]                         //load h2l | h2h
#ifndef __AARCH64EB__
        ext     v13.16b, v13.16b, v13.16b, #8
#endif
        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 1
        ld1     {v27.4s}, [x8], #16                               //load rk9

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 2
        ldr     q15, [x3, #112]                        //load h4l | h4h
#ifndef __AARCH64EB__
        ext     v15.16b, v15.16b, v15.16b, #8
#endif
        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 3
        ld1     {v28.4s}, [x8], #16                              //load rk10

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 2
        ld1     {v29.4s}, [x8], #16                              //load rk11

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 3
        add     w12, w12, #1                            //CTR block 3

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 3

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 3
        ld1     { v11.16b}, [x3]
        ext     v11.16b, v11.16b, v11.16b, #8
        rev64   v11.16b, v11.16b

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 4

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 4

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 4

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 4

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 5

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 5

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 5

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 5

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 6
        trn2    v17.2d,  v14.2d,    v15.2d                      //h4l | h3l

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 6
        ld1     {v30.4s}, [x8], #16                              //load rk12

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 6
        ldr     q12, [x3, #32]                         //load h1l | h1h
#ifndef __AARCH64EB__
        ext     v12.16b, v12.16b, v12.16b, #8
#endif
        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 6
        ld1     {v31.4s}, [x8], #16                              //load rk13

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 7
        trn1    v9.2d, v14.2d,    v15.2d                      //h4h | h3h

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 7

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 7

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 7
        trn2    v16.2d,  v12.2d,    v13.2d                      //h2l | h1l

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 8

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 8

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 8

        aese    v1.16b, v27.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 9

        aese    v2.16b, v27.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 9

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 8

        aese    v1.16b, v28.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 10

        aese    v3.16b, v27.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 9

        aese    v0.16b, v27.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 9

        aese    v2.16b, v28.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 10

        aese    v3.16b, v28.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 10

        aese    v1.16b, v29.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 11

        aese    v2.16b, v29.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 11

        aese    v0.16b, v28.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 10

        aese    v1.16b, v30.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 12

        aese    v2.16b, v30.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 12

        aese    v0.16b, v29.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 11
        eor     v17.16b, v17.16b, v9.16b                  //h4k | h3k

        aese    v3.16b, v29.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 11

        aese    v2.16b, v31.16b                                     //AES block 2 - round 13
        trn1    v8.2d,    v12.2d,    v13.2d                      //h2h | h1h

        aese    v0.16b, v30.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 12

        aese    v3.16b, v30.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 12

        aese    v1.16b, v31.16b                                     //AES block 1 - round 13

        aese    v0.16b, v31.16b                                     //AES block 0 - round 13

        aese    v3.16b, v31.16b                                     //AES block 3 - round 13
        eor     v16.16b, v16.16b, v8.16b                     //h2k | h1k
        b.ge    .L256_enc_tail                                    //handle tail

        ldp     x19, x20, [x0, #16]           //AES block 1 - load plaintext
#ifdef __AARCH64EB__
        rev     x19, x19
        rev     x20, x20
#endif
        rev     w9, w12                                 //CTR block 4
        ldp     x6, x7, [x0, #0]            //AES block 0 - load plaintext
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        ldp     x23, x24, [x0, #48]           //AES block 3 - load plaintext
#ifdef __AARCH64EB__
        rev     x23, x23
        rev     x24, x24
#endif
        ldp     x21, x22, [x0, #32]           //AES block 2 - load plaintext
#ifdef __AARCH64EB__
        rev     x21, x21
        rev     x22, x22
#endif
        add     x0, x0, #64                       //AES input_ptr update

        eor     x19, x19, x13                     //AES block 1 - round 14 low
        eor     x20, x20, x14                     //AES block 1 - round 14 high

        fmov    d5, x19                               //AES block 1 - mov low
        eor     x6, x6, x13                     //AES block 0 - round 14 low

        eor     x7, x7, x14                     //AES block 0 - round 14 high
        eor     x24, x24, x14                     //AES block 3 - round 14 high
        fmov    d4, x6                               //AES block 0 - mov low

        cmp     x0, x5                   //check if we have <= 8 blocks
        fmov    v4.d[1], x7                           //AES block 0 - mov high
        eor     x23, x23, x13                     //AES block 3 - round 14 low

        eor     x21, x21, x13                     //AES block 2 - round 14 low
        fmov    v5.d[1], x20                           //AES block 1 - mov high

        fmov    d6, x21                               //AES block 2 - mov low
        add     w12, w12, #1                            //CTR block 4

        orr     x9, x11, x9, lsl #32            //CTR block 4
        fmov    d7, x23                               //AES block 3 - mov low
        eor     x22, x22, x14                     //AES block 2 - round 14 high

        fmov    v6.d[1], x22                           //AES block 2 - mov high

        eor     v4.16b, v4.16b, v0.16b                          //AES block 0 - result
        fmov    d0, x10                               //CTR block 4

        fmov    v0.d[1], x9                               //CTR block 4
        rev     w9, w12                                 //CTR block 5
        add     w12, w12, #1                            //CTR block 5

        eor     v5.16b, v5.16b, v1.16b                          //AES block 1 - result
        fmov    d1, x10                               //CTR block 5
        orr     x9, x11, x9, lsl #32            //CTR block 5

        fmov    v1.d[1], x9                               //CTR block 5
        rev     w9, w12                                 //CTR block 6
        st1     { v4.16b}, [x2], #16                     //AES block 0 - store result

        fmov    v7.d[1], x24                           //AES block 3 - mov high
        orr     x9, x11, x9, lsl #32            //CTR block 6
        eor     v6.16b, v6.16b, v2.16b                          //AES block 2 - result

        st1     { v5.16b}, [x2], #16                     //AES block 1 - store result

        add     w12, w12, #1                            //CTR block 6
        fmov    d2, x10                               //CTR block 6

        fmov    v2.d[1], x9                               //CTR block 6
        st1     { v6.16b}, [x2], #16                     //AES block 2 - store result
        rev     w9, w12                                 //CTR block 7

        orr     x9, x11, x9, lsl #32            //CTR block 7

        eor     v7.16b, v7.16b, v3.16b                          //AES block 3 - result
        st1     { v7.16b}, [x2], #16                     //AES block 3 - store result
        b.ge    .L256_enc_prepretail                               //do prepretail

.L256_enc_main_loop:    //main  loop start
        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 0
        rev64   v4.16b, v4.16b                                    //GHASH block 4k (only t0 is free)

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 0
        fmov    d3, x10                               //CTR block 4k+3

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 0
        ext     v11.16b, v11.16b, v11.16b, #8                     //PRE 0

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 1
        fmov    v3.d[1], x9                               //CTR block 4k+3

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 1
        ldp     x23, x24, [x0, #48]           //AES block 4k+7 - load plaintext
#ifdef __AARCH64EB__
        rev     x23, x23
        rev     x24, x24
#endif
        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 1
        ldp     x21, x22, [x0, #32]           //AES block 4k+6 - load plaintext
#ifdef __AARCH64EB__
        rev     x21, x21
        rev     x22, x22
#endif
        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 2
        eor     v4.16b, v4.16b, v11.16b                           //PRE 1

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 2

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 0
        eor     x23, x23, x13                     //AES block 4k+7 - round 14 low

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 3
        mov     d10, v17.d[1]                               //GHASH block 4k - mid

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
        eor     x22, x22, x14                     //AES block 4k+6 - round 14 high
        mov     d8, v4.d[1]                                  //GHASH block 4k - mid

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 1
        rev64   v5.16b, v5.16b                                    //GHASH block 4k+1 (t0 and t1 free)

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 4

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
        eor     v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 2

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 5
        rev64   v7.16b, v7.16b                                    //GHASH block 4k+3 (t0, t1, t2 and t3 free)

        pmull2  v4.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high

        pmull   v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid
        rev64   v6.16b, v6.16b                                    //GHASH block 4k+2 (t0, t1, and t2 free)

        pmull   v8.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low

        eor     v9.16b, v9.16b, v4.16b                         //GHASH block 4k+1 - high
        mov     d4, v5.d[1]                                  //GHASH block 4k+1 - mid

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 3

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 2
        eor     v11.16b, v11.16b, v8.16b                         //GHASH block 4k+1 - low

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 3

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 4
        mov     d8, v6.d[1]                                  //GHASH block 4k+2 - mid

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 3
        eor     v4.8b, v4.8b, v5.8b                          //GHASH block 4k+1 - mid

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 4

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 6
        eor     v8.8b, v8.8b, v6.8b                          //GHASH block 4k+2 - mid

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 4

        pmull   v4.1q, v4.1d, v17.1d                          //GHASH block 4k+1 - mid

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 7

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 5
        ins     v8.d[1], v8.d[0]                                //GHASH block 4k+2 - mid

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 5

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 8

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 5

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 6
        eor     v10.16b, v10.16b, v4.16b                         //GHASH block 4k+1 - mid

        pmull2  v4.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high

        pmull   v5.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 7

        pmull   v6.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low
        eor     v9.16b, v9.16b, v4.16b                         //GHASH block 4k+2 - high

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 6
        ldp     x19, x20, [x0, #16]           //AES block 4k+5 - load plaintext
#ifdef __AARCH64EB__
        rev     x19, x19
        rev     x20, x20
#endif
        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 8
        mov     d4, v7.d[1]                                  //GHASH block 4k+3 - mid

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 6
        eor     v11.16b, v11.16b, v5.16b                         //GHASH block 4k+2 - low

        pmull2  v8.1q, v8.2d, v16.2d                          //GHASH block 4k+2 - mid

        pmull2  v5.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high
        eor     v4.8b, v4.8b, v7.8b                          //GHASH block 4k+3 - mid

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 7
        eor     x19, x19, x13                     //AES block 4k+5 - round 14 low

        aese    v1.16b, v27.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 9
        eor     v10.16b, v10.16b, v8.16b                         //GHASH block 4k+2 - mid

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 7
        eor     x21, x21, x13                     //AES block 4k+6 - round 14 low

        aese    v0.16b, v27.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 9
        movi    v8.8b, #0xc2

        pmull   v4.1q, v4.1d, v16.1d                          //GHASH block 4k+3 - mid
        eor     v9.16b, v9.16b, v5.16b                         //GHASH block 4k+3 - high
        fmov    d5, x19                               //AES block 4k+5 - mov low

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 8
        ldp     x6, x7, [x0, #0]            //AES block 4k+4 - load plaintext
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        aese    v0.16b, v28.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 10
        shl     d8, d8, #56               //mod_constant

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 8
        eor     v11.16b, v11.16b, v6.16b                         //GHASH block 4k+3 - low

        aese    v2.16b, v27.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 9

        aese    v1.16b, v28.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 10
        eor     v10.16b, v10.16b, v4.16b                         //GHASH block 4k+3 - mid

        aese    v3.16b, v27.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 9
        add     w12, w12, #1                            //CTR block 4k+3

        aese    v0.16b, v29.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 11
        eor     v4.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

        aese    v1.16b, v29.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 11
        add     x0, x0, #64                       //AES input_ptr update

        pmull   v7.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
        rev     w9, w12                                 //CTR block 4k+8
        ext     v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

        aese    v2.16b, v28.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 10
        eor     x6, x6, x13                     //AES block 4k+4 - round 14 low

        aese    v1.16b, v30.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 12
        eor     v10.16b, v10.16b, v4.16b                         //MODULO - karatsuba tidy up

        aese    v3.16b, v28.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 10
        eor     x7, x7, x14                     //AES block 4k+4 - round 14 high

        fmov    d4, x6                               //AES block 4k+4 - mov low
        orr     x9, x11, x9, lsl #32            //CTR block 4k+8
        eor     v7.16b, v9.16b, v7.16b                   //MODULO - fold into mid

        aese    v0.16b, v30.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 12
        eor     x20, x20, x14                     //AES block 4k+5 - round 14 high

        aese    v2.16b, v29.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 11
        eor     x24, x24, x14                     //AES block 4k+7 - round 14 high

        aese    v3.16b, v29.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 11
        add     w12, w12, #1                            //CTR block 4k+8

        aese    v0.16b, v31.16b                                     //AES block 4k+4 - round 13
        fmov    v4.d[1], x7                           //AES block 4k+4 - mov high
        eor     v10.16b, v10.16b, v7.16b                      //MODULO - fold into mid

        aese    v2.16b, v30.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 12
        fmov    d7, x23                               //AES block 4k+7 - mov low

        aese    v1.16b, v31.16b                                     //AES block 4k+5 - round 13
        fmov    v5.d[1], x20                           //AES block 4k+5 - mov high

        fmov    d6, x21                               //AES block 4k+6 - mov low
        cmp     x0, x5                   //.LOOP CONTROL

        fmov    v6.d[1], x22                           //AES block 4k+6 - mov high

        pmull   v9.1q, v10.1d, v8.1d            //MODULO - mid 64b align with low
        eor     v4.16b, v4.16b, v0.16b                          //AES block 4k+4 - result
        fmov    d0, x10                               //CTR block 4k+8

        fmov    v0.d[1], x9                               //CTR block 4k+8
        rev     w9, w12                                 //CTR block 4k+9
        add     w12, w12, #1                            //CTR block 4k+9

        eor     v5.16b, v5.16b, v1.16b                          //AES block 4k+5 - result
        fmov    d1, x10                               //CTR block 4k+9
        orr     x9, x11, x9, lsl #32            //CTR block 4k+9

        aese    v3.16b, v30.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 12
        fmov    v1.d[1], x9                               //CTR block 4k+9

        aese    v2.16b, v31.16b                                     //AES block 4k+6 - round 13
        rev     w9, w12                                 //CTR block 4k+10
        st1     { v4.16b}, [x2], #16                     //AES block 4k+4 - store result

        orr     x9, x11, x9, lsl #32            //CTR block 4k+10
        eor     v11.16b, v11.16b, v9.16b                         //MODULO - fold into low
        fmov    v7.d[1], x24                           //AES block 4k+7 - mov high

        ext     v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment
        st1     { v5.16b}, [x2], #16                     //AES block 4k+5 - store result
        add     w12, w12, #1                            //CTR block 4k+10

        aese    v3.16b, v31.16b                                     //AES block 4k+7 - round 13
        eor     v6.16b, v6.16b, v2.16b                          //AES block 4k+6 - result
        fmov    d2, x10                               //CTR block 4k+10

        st1     { v6.16b}, [x2], #16                     //AES block 4k+6 - store result
        fmov    v2.d[1], x9                               //CTR block 4k+10
        rev     w9, w12                                 //CTR block 4k+11

        eor     v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
        orr     x9, x11, x9, lsl #32            //CTR block 4k+11

        eor     v7.16b, v7.16b, v3.16b                          //AES block 4k+7 - result
        st1     { v7.16b}, [x2], #16                     //AES block 4k+7 - store result
        b.lt    .L256_enc_main_loop

.L256_enc_prepretail:   //PREPRETAIL
        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 0
        rev64   v6.16b, v6.16b                                    //GHASH block 4k+2 (t0, t1, and t2 free)

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 0
        fmov    d3, x10                               //CTR block 4k+3

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 0
        rev64   v4.16b, v4.16b                                    //GHASH block 4k (only t0 is free)

        fmov    v3.d[1], x9                               //CTR block 4k+3
        ext     v11.16b, v11.16b, v11.16b, #8                     //PRE 0

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 1

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 1

        eor     v4.16b, v4.16b, v11.16b                           //PRE 1
        rev64   v5.16b, v5.16b                                    //GHASH block 4k+1 (t0 and t1 free)

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 2

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 0
        mov     d10, v17.d[1]                               //GHASH block 4k - mid

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 1

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
        mov     d8, v4.d[1]                                  //GHASH block 4k - mid

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 3

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 2
        eor     v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 2

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 1

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 3

        pmull   v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid

        pmull2  v4.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high

        pmull   v8.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 2

        eor     v9.16b, v9.16b, v4.16b                         //GHASH block 4k+1 - high
        mov     d4, v5.d[1]                                  //GHASH block 4k+1 - mid

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 3
        eor     v11.16b, v11.16b, v8.16b                         //GHASH block 4k+1 - low

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 3

        eor     v4.8b, v4.8b, v5.8b                          //GHASH block 4k+1 - mid
        mov     d8, v6.d[1]                                  //GHASH block 4k+2 - mid

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 4
        rev64   v7.16b, v7.16b                                    //GHASH block 4k+3 (t0, t1, t2 and t3 free)

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 4

        pmull   v4.1q, v4.1d, v17.1d                          //GHASH block 4k+1 - mid
        eor     v8.8b, v8.8b, v6.8b                          //GHASH block 4k+2 - mid
        add     w12, w12, #1                            //CTR block 4k+3

        pmull   v5.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 5

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 4
        eor     v10.16b, v10.16b, v4.16b                         //GHASH block 4k+1 - mid

        pmull2  v4.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high

        eor     v11.16b, v11.16b, v5.16b                         //GHASH block 4k+2 - low
        ins     v8.d[1], v8.d[0]                                //GHASH block 4k+2 - mid

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 5

        eor     v9.16b, v9.16b, v4.16b                         //GHASH block 4k+2 - high
        mov     d4, v7.d[1]                                  //GHASH block 4k+3 - mid

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 4

        pmull2  v8.1q, v8.2d, v16.2d                          //GHASH block 4k+2 - mid

        eor     v4.8b, v4.8b, v7.8b                          //GHASH block 4k+3 - mid

        pmull2  v5.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 5

        pmull   v4.1q, v4.1d, v16.1d                          //GHASH block 4k+3 - mid
        eor     v10.16b, v10.16b, v8.16b                         //GHASH block 4k+2 - mid

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 5

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 6

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 6

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 6
        movi    v8.8b, #0xc2

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 6

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 7
        eor     v9.16b, v9.16b, v5.16b                         //GHASH block 4k+3 - high

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 7

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 7
        shl     d8, d8, #56               //mod_constant

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 8
        eor     v10.16b, v10.16b, v4.16b                         //GHASH block 4k+3 - mid

        pmull   v6.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 8

        aese    v1.16b, v27.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 9

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 8
        eor     v11.16b, v11.16b, v6.16b                         //GHASH block 4k+3 - low

        aese    v3.16b, v27.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 9

        eor     v10.16b, v10.16b, v9.16b                         //karatsuba tidy up

        pmull   v4.1q, v9.1d, v8.1d
        ext     v9.16b, v9.16b, v9.16b, #8

        aese    v3.16b, v28.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 10

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 7
        eor     v10.16b, v10.16b, v11.16b

        aese    v1.16b, v28.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 10

        aese    v0.16b, v27.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 9

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 8

        aese    v1.16b, v29.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 11
        eor     v10.16b, v10.16b, v4.16b

        aese    v0.16b, v28.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 10

        aese    v2.16b, v27.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 9

        aese    v1.16b, v30.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 12

        aese    v0.16b, v29.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 11
        eor     v10.16b, v10.16b, v9.16b

        aese    v3.16b, v29.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 11

        aese    v2.16b, v28.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 10

        aese    v0.16b, v30.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 12

        pmull   v4.1q, v10.1d, v8.1d

        aese    v2.16b, v29.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 11
        ext     v10.16b, v10.16b, v10.16b, #8

        aese    v3.16b, v30.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 12

        aese    v1.16b, v31.16b                                     //AES block 4k+5 - round 13
        eor     v11.16b, v11.16b, v4.16b

        aese    v2.16b, v30.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 12

        aese    v3.16b, v31.16b                                     //AES block 4k+7 - round 13

        aese    v0.16b, v31.16b                                     //AES block 4k+4 - round 13

        aese    v2.16b, v31.16b                                     //AES block 4k+6 - round 13
        eor     v11.16b, v11.16b, v10.16b
.L256_enc_tail: //TAIL

        ext     v8.16b, v11.16b, v11.16b, #8                     //prepare final partial tag
        sub     x5, x4, x0   //main_end_input_ptr is number of bytes left to process
        ldp     x6, x7, [x0], #16           //AES block 4k+4 - load plaintext
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        eor     x6, x6, x13                     //AES block 4k+4 - round 14 low
        eor     x7, x7, x14                     //AES block 4k+4 - round 14 high

        cmp     x5, #48
        fmov    d4, x6                               //AES block 4k+4 - mov low

        fmov    v4.d[1], x7                           //AES block 4k+4 - mov high

        eor     v5.16b, v4.16b, v0.16b                          //AES block 4k+4 - result
        b.gt    .L256_enc_blocks_more_than_3

        cmp     x5, #32
        mov     v3.16b, v2.16b
        movi    v11.8b, #0

        movi    v9.8b, #0
        sub     w12, w12, #1

        mov     v2.16b, v1.16b
        movi    v10.8b, #0
        b.gt    .L256_enc_blocks_more_than_2

        mov     v3.16b, v1.16b
        sub     w12, w12, #1
        cmp     x5, #16

        b.gt    .L256_enc_blocks_more_than_1

        sub     w12, w12, #1
        b       .L256_enc_blocks_less_than_1
.L256_enc_blocks_more_than_3:   //blocks        left >  3
        st1     { v5.16b}, [x2], #16                    //AES final-3 block  - store result

        ldp     x6, x7, [x0], #16          //AES final-2 block - load input low & high
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        rev64   v4.16b, v5.16b                                   //GHASH final-3 block

        eor     x6, x6, x13                    //AES final-2 block - round 14 low
        eor     v4.16b, v4.16b, v8.16b                          //feed in partial tag

        eor     x7, x7, x14                    //AES final-2 block - round 14 high

        mov     d22, v4.d[1]                                //GHASH final-3 block - mid
        fmov    d5, x6                                //AES final-2 block - mov low

        fmov    v5.d[1], x7                            //AES final-2 block - mov high

        eor     v22.8b, v22.8b, v4.8b                     //GHASH final-3 block - mid
        movi    v8.8b, #0                                       //suppress further partial tag feed in

        mov     d10, v17.d[1]                              //GHASH final-3 block - mid

        pmull   v11.1q, v4.1d, v15.1d                      //GHASH final-3 block - low

        pmull2  v9.1q, v4.2d, v15.2d                      //GHASH final-3 block - high

        pmull   v10.1q, v22.1d, v10.1d                   //GHASH final-3 block - mid
        eor     v5.16b, v5.16b, v1.16b                           //AES final-2 block - result
.L256_enc_blocks_more_than_2:   //blocks        left >  2

        st1     { v5.16b}, [x2], #16                    //AES final-2 block - store result

        ldp     x6, x7, [x0], #16          //AES final-1 block - load input low & high
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        rev64   v4.16b, v5.16b                                   //GHASH final-2 block

        eor     x6, x6, x13                    //AES final-1 block - round 14 low
        eor     v4.16b, v4.16b, v8.16b                          //feed in partial tag

        fmov    d5, x6                                //AES final-1 block - mov low
        eor     x7, x7, x14                    //AES final-1 block - round 14 high

        fmov    v5.d[1], x7                            //AES final-1 block - mov high

        movi    v8.8b, #0                                       //suppress further partial tag feed in

        pmull2  v20.1q, v4.2d, v14.2d                         //GHASH final-2 block - high
        mov     d22, v4.d[1]                                //GHASH final-2 block - mid

        pmull   v21.1q, v4.1d, v14.1d                         //GHASH final-2 block - low

        eor     v22.8b, v22.8b, v4.8b                     //GHASH final-2 block - mid

        eor     v5.16b, v5.16b, v2.16b                           //AES final-1 block - result

        eor     v9.16b, v9.16b, v20.16b                           //GHASH final-2 block - high

        pmull   v22.1q, v22.1d, v17.1d                     //GHASH final-2 block - mid

        eor     v11.16b, v11.16b, v21.16b                           //GHASH final-2 block - low

        eor     v10.16b, v10.16b, v22.16b                      //GHASH final-2 block - mid
.L256_enc_blocks_more_than_1:   //blocks        left >  1

        st1     { v5.16b}, [x2], #16                    //AES final-1 block - store result

        rev64   v4.16b, v5.16b                                   //GHASH final-1 block

        ldp     x6, x7, [x0], #16          //AES final block - load input low & high
#ifdef __AARCH64EB__
        rev     x6, x6
        rev     x7, x7
#endif
        eor     v4.16b, v4.16b, v8.16b                          //feed in partial tag

        movi    v8.8b, #0                                       //suppress further partial tag feed in

        eor     x6, x6, x13                    //AES final block - round 14 low
        mov     d22, v4.d[1]                                //GHASH final-1 block - mid

        pmull2  v20.1q, v4.2d, v13.2d                         //GHASH final-1 block - high
        eor     x7, x7, x14                    //AES final block - round 14 high

        eor     v22.8b, v22.8b, v4.8b                     //GHASH final-1 block - mid

        eor     v9.16b, v9.16b, v20.16b                           //GHASH final-1 block - high

        ins     v22.d[1], v22.d[0]                           //GHASH final-1 block - mid
        fmov    d5, x6                                //AES final block - mov low

        fmov    v5.d[1], x7                            //AES final block - mov high

        pmull2  v22.1q, v22.2d, v16.2d                     //GHASH final-1 block - mid

        pmull   v21.1q, v4.1d, v13.1d                         //GHASH final-1 block - low

        eor     v5.16b, v5.16b, v3.16b                           //AES final block - result
        eor     v10.16b, v10.16b, v22.16b                      //GHASH final-1 block - mid

        eor     v11.16b, v11.16b, v21.16b                           //GHASH final-1 block - low
.L256_enc_blocks_less_than_1:   //blocks        left <= 1

        and     x1, x1, #127                   //bit_length %= 128

        mvn     x13, xzr                                     //rk14_l = 0xffffffffffffffff
        sub     x1, x1, #128                   //bit_length -= 128

        neg     x1, x1                         //bit_length = 128 - #bits in input (in range [1,128])
        ld1     { v18.16b}, [x2]                           //load existing bytes where the possibly partial last block is to be stored

        mvn     x14, xzr                                     //rk14_h = 0xffffffffffffffff
        and     x1, x1, #127                   //bit_length %= 128

        lsr     x14, x14, x1                    //rk14_h is mask for top 64b of last block
        cmp     x1, #64

        csel    x6, x13, x14, lt
        csel    x7, x14, xzr, lt

        fmov    d0, x6                                //ctr0b is mask for last block

        fmov    v0.d[1], x7

        and     v5.16b, v5.16b, v0.16b                           //possibly partial last block has zeroes in highest bits

        rev64   v4.16b, v5.16b                                   //GHASH final block

        eor     v4.16b, v4.16b, v8.16b                          //feed in partial tag

        bif     v5.16b, v18.16b, v0.16b                             //insert existing bytes in top end of result before storing

        pmull2  v20.1q, v4.2d, v12.2d                         //GHASH final block - high
        mov     d8, v4.d[1]                                 //GHASH final block - mid
#ifndef __AARCH64EB__
        rev     w9, w12
#else
        mov     w9, w12
#endif

        pmull   v21.1q, v4.1d, v12.1d                         //GHASH final block - low

        eor     v9.16b, v9.16b, v20.16b                           //GHASH final block - high
        eor     v8.8b, v8.8b, v4.8b                         //GHASH final block - mid

        pmull   v8.1q, v8.1d, v16.1d                         //GHASH final block - mid

        eor     v11.16b, v11.16b, v21.16b                           //GHASH final block - low

        eor     v10.16b, v10.16b, v8.16b                        //GHASH final block - mid
        movi    v8.8b, #0xc2

        eor     v4.16b, v11.16b, v9.16b                        //MODULO - karatsuba tidy up

        shl     d8, d8, #56              //mod_constant

        eor     v10.16b, v10.16b, v4.16b                        //MODULO - karatsuba tidy up

        pmull   v7.1q, v9.1d, v8.1d           //MODULO - top 64b align with mid

        ext     v9.16b, v9.16b, v9.16b, #8                    //MODULO - other top alignment

        eor     v10.16b, v10.16b, v7.16b                     //MODULO - fold into mid

        eor     v10.16b, v10.16b, v9.16b                        //MODULO - fold into mid

        pmull   v9.1q, v10.1d, v8.1d           //MODULO - mid 64b align with low

        ext     v10.16b, v10.16b, v10.16b, #8                    //MODULO - other mid alignment

        str     w9, [x16, #12]                         //store the updated counter

        st1     { v5.16b}, [x2]                         //store all 16B
        eor     v11.16b, v11.16b, v9.16b                        //MODULO - fold into low

        eor     v11.16b, v11.16b, v10.16b                        //MODULO - fold into low
        ext     v11.16b, v11.16b, v11.16b, #8
        rev64   v11.16b, v11.16b
        mov     x0, x15
        st1     { v11.16b }, [x3]

        ldp     x21, x22, [sp, #16]
        ldp     x23, x24, [sp, #32]
        ldp     d8, d9, [sp, #48]
        ldp     d10, d11, [sp, #64]
        ldp     d12, d13, [sp, #80]
        ldp     d14, d15, [sp, #96]
        ldp     x19, x20, [sp], #112
        ret

.L256_enc_ret:
        mov     w0, #0x0
        ret
.size   aes_gcm_enc_256_kernel,.-aes_gcm_enc_256_kernel
.globl  aes_gcm_dec_256_kernel
.type   aes_gcm_dec_256_kernel,%function
.align  4
aes_gcm_dec_256_kernel:
        AARCH64_VALID_CALL_TARGET
        cbz     x1, .L256_dec_ret
        stp     x19, x20, [sp, #-112]!
        mov     x16, x4
        mov     x8, x5
        stp     x21, x22, [sp, #16]
        stp     x23, x24, [sp, #32]
        stp     d8, d9, [sp, #48]
        stp     d10, d11, [sp, #64]
        stp     d12, d13, [sp, #80]
        stp     d14, d15, [sp, #96]

        lsr     x5, x1, #3              //byte_len
        mov     x15, x5
        ldp     x10, x11, [x16]              //ctr96_b64, ctr96_t32
#ifdef __AARCH64EB__
        rev     x10, x10
        rev     x11, x11
#endif
        ldp     x13, x14, [x8, #224]                     //load rk14
#ifdef __AARCH64EB__
        ror     x14, x14, #32
        ror     x13, x13, #32
#endif
        ld1     {v18.4s}, [x8], #16                               //load rk0
        sub     x5, x5, #1      //byte_len - 1

        ld1     {v19.4s}, [x8], #16                               //load rk1
        and     x5, x5, #0xffffffffffffffc0 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

        add     x4, x0, x1, lsr #3   //end_input_ptr
        ld1     {v20.4s}, [x8], #16                               //load rk2

        lsr     x12, x11, #32
        ld1     {v21.4s}, [x8], #16                               //load rk3
        orr     w11, w11, w11

        ld1     {v22.4s}, [x8], #16                               //load rk4
        add     x5, x5, x0
        rev     w12, w12                                //rev_ctr32

        add     w12, w12, #1                            //increment rev_ctr32
        fmov    d3, x10                               //CTR block 3

        rev     w9, w12                                 //CTR block 1
        add     w12, w12, #1                            //CTR block 1
        fmov    d1, x10                               //CTR block 1

        orr     x9, x11, x9, lsl #32            //CTR block 1
        ld1     { v0.16b}, [x16]                             //special case vector load initial counter so we can start first AES block as quickly as possible

        fmov    v1.d[1], x9                               //CTR block 1
        rev     w9, w12                                 //CTR block 2
        add     w12, w12, #1                            //CTR block 2

        fmov    d2, x10                               //CTR block 2
        orr     x9, x11, x9, lsl #32            //CTR block 2

        fmov    v2.d[1], x9                               //CTR block 2
        rev     w9, w12                                 //CTR block 3

        orr     x9, x11, x9, lsl #32            //CTR block 3
        ld1     {v23.4s}, [x8], #16                               //load rk5

        fmov    v3.d[1], x9                               //CTR block 3
        add     w12, w12, #1                            //CTR block 3

        ld1     {v24.4s}, [x8], #16                               //load rk6

        ld1     {v25.4s}, [x8], #16                               //load rk7

        ld1     {v26.4s}, [x8], #16                               //load rk8

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 0
        ldr     q14, [x3, #80]                         //load h3l | h3h
#ifndef __AARCH64EB__
        ext     v14.16b, v14.16b, v14.16b, #8
#endif

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 0
        ldr     q15, [x3, #112]                        //load h4l | h4h
#ifndef __AARCH64EB__
        ext     v15.16b, v15.16b, v15.16b, #8
#endif

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 0
        ldr     q13, [x3, #64]                         //load h2l | h2h
#ifndef __AARCH64EB__
        ext     v13.16b, v13.16b, v13.16b, #8
#endif

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 0
        ld1     {v27.4s}, [x8], #16                                 //load rk9

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 1

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 1
        ld1     { v11.16b}, [x3]
        ext     v11.16b, v11.16b, v11.16b, #8
        rev64   v11.16b, v11.16b

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 1
        ld1     {v28.4s}, [x8], #16                              //load rk10

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 1
        ld1     {v29.4s}, [x8], #16                              //load rk11

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 2
        ldr     q12, [x3, #32]                         //load h1l | h1h
#ifndef __AARCH64EB__
        ext     v12.16b, v12.16b, v12.16b, #8
#endif
        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 2
        ld1     {v30.4s}, [x8], #16                              //load rk12

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 2

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 3

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 2

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 3

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 4
        cmp     x0, x5                   //check if we have <= 4 blocks

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 3

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 3

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 4

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 4

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 4

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 5

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 5

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 5

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 5

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 6

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 6

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 6

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 6

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 7

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 7

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 7

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 8

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 7

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 8

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 8

        aese    v0.16b, v27.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 9

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 8
        ld1     {v31.4s}, [x8], #16                             //load rk13

        aese    v1.16b, v27.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 9

        aese    v0.16b, v28.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 10

        aese    v3.16b, v27.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 9

        aese    v1.16b, v28.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 10

        aese    v2.16b, v27.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 9

        aese    v3.16b, v28.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 10

        aese    v0.16b, v29.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 11

        aese    v2.16b, v28.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 10

        aese    v3.16b, v29.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 11

        aese    v1.16b, v29.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 11

        aese    v2.16b, v29.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 11

        trn1    v9.2d, v14.2d,    v15.2d                      //h4h | h3h

        trn2    v17.2d,  v14.2d,    v15.2d                      //h4l | h3l

        trn1    v8.2d,    v12.2d,    v13.2d                      //h2h | h1h
        trn2    v16.2d,  v12.2d,    v13.2d                      //h2l | h1l

        aese    v1.16b, v30.16b
        aesmc   v1.16b, v1.16b          //AES block 1 - round 12

        aese    v0.16b, v30.16b
        aesmc   v0.16b, v0.16b          //AES block 0 - round 12

        aese    v2.16b, v30.16b
        aesmc   v2.16b, v2.16b          //AES block 2 - round 12

        aese    v3.16b, v30.16b
        aesmc   v3.16b, v3.16b          //AES block 3 - round 12
        eor     v17.16b, v17.16b, v9.16b                  //h4k | h3k

        aese    v1.16b, v31.16b                                     //AES block 1 - round 13

        aese    v2.16b, v31.16b                                     //AES block 2 - round 13
        eor     v16.16b, v16.16b, v8.16b                     //h2k | h1k

        aese    v3.16b, v31.16b                                     //AES block 3 - round 13

        aese    v0.16b, v31.16b                                     //AES block 0 - round 13
        b.ge    .L256_dec_tail                                    //handle tail

        ld1     {v4.16b, v5.16b}, [x0], #32               //AES block 0,1 - load ciphertext

        rev     w9, w12                                 //CTR block 4

        eor     v0.16b, v4.16b, v0.16b                            //AES block 0 - result

        eor     v1.16b, v5.16b, v1.16b                            //AES block 1 - result
        rev64   v5.16b, v5.16b                                    //GHASH block 1
        ld1     {v6.16b}, [x0], #16                       //AES block 2 - load ciphertext

        mov     x7, v0.d[1]                            //AES block 0 - mov high

        mov     x6, v0.d[0]                            //AES block 0 - mov low
        rev64   v4.16b, v4.16b                                    //GHASH block 0
        add     w12, w12, #1                            //CTR block 4

        fmov    d0, x10                               //CTR block 4
        orr     x9, x11, x9, lsl #32            //CTR block 4

        fmov    v0.d[1], x9                               //CTR block 4
        rev     w9, w12                                 //CTR block 5
        add     w12, w12, #1                            //CTR block 5

        mov     x19, v1.d[0]                            //AES block 1 - mov low

        orr     x9, x11, x9, lsl #32            //CTR block 5
        mov     x20, v1.d[1]                            //AES block 1 - mov high
        eor     x7, x7, x14                   //AES block 0 - round 14 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        eor     x6, x6, x13                   //AES block 0 - round 14 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        stp     x6, x7, [x2], #16        //AES block 0 - store result
        fmov    d1, x10                               //CTR block 5

        ld1     {v7.16b}, [x0], #16                       //AES block 3 - load ciphertext

        fmov    v1.d[1], x9                               //CTR block 5
        rev     w9, w12                                 //CTR block 6
        add     w12, w12, #1                            //CTR block 6

        eor     x19, x19, x13                   //AES block 1 - round 14 low
#ifdef __AARCH64EB__
        rev     x19, x19
#endif
        orr     x9, x11, x9, lsl #32            //CTR block 6

        eor     x20, x20, x14                   //AES block 1 - round 14 high
#ifdef __AARCH64EB__
        rev     x20, x20
#endif
        stp     x19, x20, [x2], #16        //AES block 1 - store result

        eor     v2.16b, v6.16b, v2.16b                            //AES block 2 - result
        cmp     x0, x5                   //check if we have <= 8 blocks
        b.ge    .L256_dec_prepretail                              //do prepretail

.L256_dec_main_loop:    //main  loop start
        mov     x21, v2.d[0]                            //AES block 4k+2 - mov low
        ext     v11.16b, v11.16b, v11.16b, #8                     //PRE 0
        eor     v3.16b, v7.16b, v3.16b                            //AES block 4k+3 - result

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 0
        mov     x22, v2.d[1]                            //AES block 4k+2 - mov high

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 0
        fmov    d2, x10                               //CTR block 4k+6

        fmov    v2.d[1], x9                               //CTR block 4k+6
        eor     v4.16b, v4.16b, v11.16b                           //PRE 1
        rev     w9, w12                                 //CTR block 4k+7

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 1
        mov     x24, v3.d[1]                            //AES block 4k+3 - mov high

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 1
        mov     x23, v3.d[0]                            //AES block 4k+3 - mov low

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
        mov     d8, v4.d[1]                                  //GHASH block 4k - mid
        fmov    d3, x10                               //CTR block 4k+7

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 2
        orr     x9, x11, x9, lsl #32            //CTR block 4k+7

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 0
        fmov    v3.d[1], x9                               //CTR block 4k+7

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 2
        eor     v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 3
        eor     x22, x22, x14                   //AES block 4k+2 - round 14 high
#ifdef __AARCH64EB__
        rev     x22, x22
#endif
        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 1
        mov     d10, v17.d[1]                               //GHASH block 4k - mid

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 3
        rev64   v6.16b, v6.16b                                    //GHASH block 4k+2

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 0
        eor     x21, x21, x13                   //AES block 4k+2 - round 14 low
#ifdef __AARCH64EB__
        rev     x21, x21
#endif
        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 2
        stp     x21, x22, [x2], #16        //AES block 4k+2 - store result

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low

        pmull2  v4.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 3
        rev64   v7.16b, v7.16b                                    //GHASH block 4k+3

        pmull   v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid
        eor     x23, x23, x13                   //AES block 4k+3 - round 14 low
#ifdef __AARCH64EB__
        rev     x23, x23
#endif
        pmull   v8.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low
        eor     x24, x24, x14                   //AES block 4k+3 - round 14 high
#ifdef __AARCH64EB__
        rev     x24, x24
#endif
        eor     v9.16b, v9.16b, v4.16b                         //GHASH block 4k+1 - high

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 4

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 1
        mov     d4, v5.d[1]                                  //GHASH block 4k+1 - mid

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 4
        eor     v11.16b, v11.16b, v8.16b                         //GHASH block 4k+1 - low

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 5
        add     w12, w12, #1                            //CTR block 4k+7

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 2
        mov     d8, v6.d[1]                                  //GHASH block 4k+2 - mid

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 4
        eor     v4.8b, v4.8b, v5.8b                          //GHASH block 4k+1 - mid

        pmull   v5.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 3
        eor     v8.8b, v8.8b, v6.8b                          //GHASH block 4k+2 - mid

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 5

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 5
        eor     v11.16b, v11.16b, v5.16b                         //GHASH block 4k+2 - low

        pmull   v4.1q, v4.1d, v17.1d                          //GHASH block 4k+1 - mid
        rev     w9, w12                                 //CTR block 4k+8

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 6
        ins     v8.d[1], v8.d[0]                                //GHASH block 4k+2 - mid

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 6
        add     w12, w12, #1                            //CTR block 4k+8

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 4

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 7
        eor     v10.16b, v10.16b, v4.16b                         //GHASH block 4k+1 - mid

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 7

        pmull2  v4.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high
        mov     d6, v7.d[1]                                  //GHASH block 4k+3 - mid

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 5

        pmull2  v8.1q, v8.2d, v16.2d                          //GHASH block 4k+2 - mid

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 8
        eor     v9.16b, v9.16b, v4.16b                         //GHASH block 4k+2 - high

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 6

        pmull   v4.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low
        orr     x9, x11, x9, lsl #32            //CTR block 4k+8
        eor     v10.16b, v10.16b, v8.16b                         //GHASH block 4k+2 - mid

        pmull2  v5.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

        aese    v0.16b, v27.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 9
        eor     v6.8b, v6.8b, v7.8b                          //GHASH block 4k+3 - mid

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 8

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 6
        eor     v9.16b, v9.16b, v5.16b                         //GHASH block 4k+3 - high

        aese    v0.16b, v28.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 10

        pmull   v6.1q, v6.1d, v16.1d                          //GHASH block 4k+3 - mid
        movi    v8.8b, #0xc2

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 7
        eor     v11.16b, v11.16b, v4.16b                         //GHASH block 4k+3 - low

        aese    v0.16b, v29.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 11

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 7
        shl     d8, d8, #56               //mod_constant

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 8
        eor     v10.16b, v10.16b, v6.16b                         //GHASH block 4k+3 - mid

        aese    v0.16b, v30.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 12

        pmull   v7.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
        eor     v6.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

        aese    v1.16b, v27.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 9
        ld1     {v4.16b}, [x0], #16                       //AES block 4k+4 - load ciphertext

        aese    v0.16b, v31.16b                                     //AES block 4k+4 - round 13
        ext     v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

        aese    v1.16b, v28.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 10
        eor     v10.16b, v10.16b, v6.16b                         //MODULO - karatsuba tidy up

        aese    v2.16b, v27.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 9
        ld1     {v5.16b}, [x0], #16                       //AES block 4k+5 - load ciphertext

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 8
        eor     v0.16b, v4.16b, v0.16b                            //AES block 4k+4 - result

        aese    v1.16b, v29.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 11
        stp     x23, x24, [x2], #16        //AES block 4k+3 - store result

        aese    v2.16b, v28.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 10
        eor     v10.16b, v10.16b, v7.16b                      //MODULO - fold into mid

        aese    v3.16b, v27.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 9
        ld1     {v6.16b}, [x0], #16                       //AES block 4k+6 - load ciphertext

        aese    v1.16b, v30.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 12
        ld1     {v7.16b}, [x0], #16                       //AES block 4k+7 - load ciphertext

        aese    v2.16b, v29.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 11
        mov     x7, v0.d[1]                            //AES block 4k+4 - mov high

        aese    v3.16b, v28.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 10
        eor     v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

        aese    v1.16b, v31.16b                                     //AES block 4k+5 - round 13
        mov     x6, v0.d[0]                            //AES block 4k+4 - mov low

        aese    v2.16b, v30.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 12
        fmov    d0, x10                               //CTR block 4k+8

        aese    v3.16b, v29.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 11
        fmov    v0.d[1], x9                               //CTR block 4k+8

        pmull   v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low
        eor     v1.16b, v5.16b, v1.16b                            //AES block 4k+5 - result
        rev     w9, w12                                 //CTR block 4k+9

        aese    v2.16b, v31.16b                                     //AES block 4k+6 - round 13
        orr     x9, x11, x9, lsl #32            //CTR block 4k+9
        cmp     x0, x5                   //.LOOP CONTROL

        add     w12, w12, #1                            //CTR block 4k+9

        eor     x6, x6, x13                   //AES block 4k+4 - round 14 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        eor     x7, x7, x14                   //AES block 4k+4 - round 14 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        mov     x20, v1.d[1]                            //AES block 4k+5 - mov high
        eor     v2.16b, v6.16b, v2.16b                            //AES block 4k+6 - result
        eor     v11.16b, v11.16b, v8.16b               //MODULO - fold into low

        aese    v3.16b, v30.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 12
        mov     x19, v1.d[0]                            //AES block 4k+5 - mov low

        fmov    d1, x10                               //CTR block 4k+9
        ext     v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

        fmov    v1.d[1], x9                               //CTR block 4k+9
        rev     w9, w12                                 //CTR block 4k+10
        add     w12, w12, #1                            //CTR block 4k+10

        aese    v3.16b, v31.16b                                     //AES block 4k+7 - round 13
        orr     x9, x11, x9, lsl #32            //CTR block 4k+10

        rev64   v5.16b, v5.16b                                    //GHASH block 4k+5
        eor     x20, x20, x14                   //AES block 4k+5 - round 14 high
#ifdef __AARCH64EB__
        rev     x20, x20
#endif
        stp     x6, x7, [x2], #16        //AES block 4k+4 - store result

        eor     x19, x19, x13                   //AES block 4k+5 - round 14 low
#ifdef __AARCH64EB__
        rev     x19, x19
#endif
        stp     x19, x20, [x2], #16        //AES block 4k+5 - store result

        rev64   v4.16b, v4.16b                                    //GHASH block 4k+4
        eor     v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
        b.lt    .L256_dec_main_loop


.L256_dec_prepretail:   //PREPRETAIL
        ext     v11.16b, v11.16b, v11.16b, #8                     //PRE 0
        mov     x21, v2.d[0]                            //AES block 4k+2 - mov low
        eor     v3.16b, v7.16b, v3.16b                            //AES block 4k+3 - result

        aese    v0.16b, v18.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 0
        mov     x22, v2.d[1]                            //AES block 4k+2 - mov high

        aese    v1.16b, v18.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 0
        fmov    d2, x10                               //CTR block 4k+6

        fmov    v2.d[1], x9                               //CTR block 4k+6
        rev     w9, w12                                 //CTR block 4k+7
        eor     v4.16b, v4.16b, v11.16b                           //PRE 1

        rev64   v6.16b, v6.16b                                    //GHASH block 4k+2
        orr     x9, x11, x9, lsl #32            //CTR block 4k+7
        mov     x23, v3.d[0]                            //AES block 4k+3 - mov low

        aese    v1.16b, v19.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 1
        mov     x24, v3.d[1]                            //AES block 4k+3 - mov high

        pmull   v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
        mov     d8, v4.d[1]                                  //GHASH block 4k - mid
        fmov    d3, x10                               //CTR block 4k+7

        pmull2  v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
        fmov    v3.d[1], x9                               //CTR block 4k+7

        aese    v2.16b, v18.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 0
        mov     d10, v17.d[1]                               //GHASH block 4k - mid

        aese    v0.16b, v19.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 1
        eor     v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

        pmull2  v4.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high

        aese    v2.16b, v19.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 1
        rev64   v7.16b, v7.16b                                    //GHASH block 4k+3

        aese    v3.16b, v18.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 0

        pmull   v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid
        eor     v9.16b, v9.16b, v4.16b                         //GHASH block 4k+1 - high

        pmull   v8.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low

        aese    v3.16b, v19.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 1
        mov     d4, v5.d[1]                                  //GHASH block 4k+1 - mid

        aese    v0.16b, v20.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 2

        aese    v1.16b, v20.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 2
        eor     v11.16b, v11.16b, v8.16b                         //GHASH block 4k+1 - low

        aese    v2.16b, v20.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 2

        aese    v0.16b, v21.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 3
        mov     d8, v6.d[1]                                  //GHASH block 4k+2 - mid

        aese    v3.16b, v20.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 2
        eor     v4.8b, v4.8b, v5.8b                          //GHASH block 4k+1 - mid

        pmull   v5.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

        aese    v0.16b, v22.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 4

        aese    v3.16b, v21.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 3
        eor     v8.8b, v8.8b, v6.8b                          //GHASH block 4k+2 - mid

        pmull   v4.1q, v4.1d, v17.1d                          //GHASH block 4k+1 - mid

        aese    v0.16b, v23.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 5
        eor     v11.16b, v11.16b, v5.16b                         //GHASH block 4k+2 - low

        aese    v3.16b, v22.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 4

        pmull2  v5.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high
        eor     v10.16b, v10.16b, v4.16b                         //GHASH block 4k+1 - mid

        pmull2  v4.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high

        aese    v3.16b, v23.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 5
        ins     v8.d[1], v8.d[0]                                //GHASH block 4k+2 - mid

        aese    v2.16b, v21.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 3

        aese    v1.16b, v21.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 3
        eor     v9.16b, v9.16b, v4.16b                         //GHASH block 4k+2 - high

        pmull   v4.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low

        aese    v2.16b, v22.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 4
        mov     d6, v7.d[1]                                  //GHASH block 4k+3 - mid

        aese    v1.16b, v22.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 4

        pmull2  v8.1q, v8.2d, v16.2d                          //GHASH block 4k+2 - mid

        aese    v2.16b, v23.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 5
        eor     v6.8b, v6.8b, v7.8b                          //GHASH block 4k+3 - mid

        aese    v1.16b, v23.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 5

        aese    v3.16b, v24.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 6
        eor     v10.16b, v10.16b, v8.16b                         //GHASH block 4k+2 - mid

        aese    v2.16b, v24.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 6

        aese    v0.16b, v24.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 6
        movi    v8.8b, #0xc2

        aese    v1.16b, v24.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 6
        eor     v11.16b, v11.16b, v4.16b                         //GHASH block 4k+3 - low

        pmull   v6.1q, v6.1d, v16.1d                          //GHASH block 4k+3 - mid

        aese    v3.16b, v25.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 7
        eor     v9.16b, v9.16b, v5.16b                         //GHASH block 4k+3 - high

        aese    v1.16b, v25.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 7

        aese    v0.16b, v25.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 7
        eor     v10.16b, v10.16b, v6.16b                         //GHASH block 4k+3 - mid

        aese    v3.16b, v26.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 8

        aese    v2.16b, v25.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 7
        eor     v6.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

        aese    v1.16b, v26.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 8

        aese    v0.16b, v26.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 8
        shl     d8, d8, #56               //mod_constant

        aese    v2.16b, v26.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 8

        aese    v1.16b, v27.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 9
        eor     v10.16b, v10.16b, v6.16b                         //MODULO - karatsuba tidy up

        pmull   v7.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid

        aese    v2.16b, v27.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 9
        ext     v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

        aese    v3.16b, v27.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 9

        aese    v0.16b, v27.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 9
        eor     v10.16b, v10.16b, v7.16b                      //MODULO - fold into mid

        aese    v2.16b, v28.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 10

        aese    v3.16b, v28.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 10

        aese    v0.16b, v28.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 10
        eor     x22, x22, x14                   //AES block 4k+2 - round 14 high
#ifdef __AARCH64EB__
        rev     x22, x22
#endif
        aese    v1.16b, v28.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 10
        eor     x23, x23, x13                   //AES block 4k+3 - round 14 low
#ifdef __AARCH64EB__
        rev     x23, x23
#endif
        aese    v2.16b, v29.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 11
        eor     v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

        aese    v0.16b, v29.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 11
        add     w12, w12, #1                            //CTR block 4k+7

        aese    v1.16b, v29.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 11
        eor     x21, x21, x13                   //AES block 4k+2 - round 14 low
#ifdef __AARCH64EB__
        rev     x21, x21
#endif

        aese    v2.16b, v30.16b
        aesmc   v2.16b, v2.16b          //AES block 4k+6 - round 12

        pmull   v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low
        eor     x24, x24, x14                   //AES block 4k+3 - round 14 high
#ifdef __AARCH64EB__
        rev     x24, x24
#endif

        aese    v3.16b, v29.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 11
        stp     x21, x22, [x2], #16        //AES block 4k+2 - store result

        aese    v1.16b, v30.16b
        aesmc   v1.16b, v1.16b          //AES block 4k+5 - round 12
        ext     v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

        aese    v0.16b, v30.16b
        aesmc   v0.16b, v0.16b          //AES block 4k+4 - round 12
        stp     x23, x24, [x2], #16        //AES block 4k+3 - store result

        aese    v3.16b, v30.16b
        aesmc   v3.16b, v3.16b          //AES block 4k+7 - round 12
        eor     v11.16b, v11.16b, v8.16b               //MODULO - fold into low

        aese    v1.16b, v31.16b                                     //AES block 4k+5 - round 13

        aese    v0.16b, v31.16b                                     //AES block 4k+4 - round 13

        aese    v3.16b, v31.16b                                     //AES block 4k+7 - round 13

        aese    v2.16b, v31.16b                                     //AES block 4k+6 - round 13
        eor     v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
.L256_dec_tail: //TAIL

        sub     x5, x4, x0   //main_end_input_ptr is number of bytes left to process
        ld1     { v5.16b}, [x0], #16                      //AES block 4k+4 - load ciphertext

        eor     v0.16b, v5.16b, v0.16b                            //AES block 4k+4 - result

        mov     x6, v0.d[0]                            //AES block 4k+4 - mov low

        mov     x7, v0.d[1]                            //AES block 4k+4 - mov high
        ext     v8.16b, v11.16b, v11.16b, #8                     //prepare final partial tag

        cmp     x5, #48

        eor     x6, x6, x13                   //AES block 4k+4 - round 14 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif

        eor     x7, x7, x14                   //AES block 4k+4 - round 14 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
        b.gt    .L256_dec_blocks_more_than_3

        sub     w12, w12, #1
        mov     v3.16b, v2.16b
        movi    v10.8b, #0

        movi    v11.8b, #0
        cmp     x5, #32

        movi    v9.8b, #0
        mov     v2.16b, v1.16b
        b.gt    .L256_dec_blocks_more_than_2

        sub     w12, w12, #1

        mov     v3.16b, v1.16b
        cmp     x5, #16
        b.gt    .L256_dec_blocks_more_than_1

        sub     w12, w12, #1
        b       .L256_dec_blocks_less_than_1
.L256_dec_blocks_more_than_3:   //blocks        left >  3
        rev64   v4.16b, v5.16b                                   //GHASH final-3 block
        ld1     { v5.16b}, [x0], #16                     //AES final-2 block - load ciphertext

        stp     x6, x7, [x2], #16       //AES final-3 block  - store result

        mov     d10, v17.d[1]                              //GHASH final-3 block - mid

        eor     v4.16b, v4.16b, v8.16b                          //feed in partial tag

        eor     v0.16b, v5.16b, v1.16b                           //AES final-2 block - result

        mov     d22, v4.d[1]                                //GHASH final-3 block - mid

        mov     x6, v0.d[0]                           //AES final-2 block - mov low

        mov     x7, v0.d[1]                           //AES final-2 block - mov high

        eor     v22.8b, v22.8b, v4.8b                     //GHASH final-3 block - mid

        movi    v8.8b, #0                                       //suppress further partial tag feed in

        pmull2  v9.1q, v4.2d, v15.2d                      //GHASH final-3 block - high

        pmull   v10.1q, v22.1d, v10.1d                   //GHASH final-3 block - mid
        eor     x6, x6, x13                  //AES final-2 block - round 14 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif

        pmull   v11.1q, v4.1d, v15.1d                      //GHASH final-3 block - low
        eor     x7, x7, x14                  //AES final-2 block - round 14 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
.L256_dec_blocks_more_than_2:   //blocks        left >  2

        rev64   v4.16b, v5.16b                                   //GHASH final-2 block
        ld1     { v5.16b}, [x0], #16                     //AES final-1 block - load ciphertext

        eor     v4.16b, v4.16b, v8.16b                          //feed in partial tag
        stp     x6, x7, [x2], #16       //AES final-2 block  - store result

        eor     v0.16b, v5.16b, v2.16b                           //AES final-1 block - result

        mov     d22, v4.d[1]                                //GHASH final-2 block - mid

        pmull   v21.1q, v4.1d, v14.1d                         //GHASH final-2 block - low

        pmull2  v20.1q, v4.2d, v14.2d                         //GHASH final-2 block - high

        eor     v22.8b, v22.8b, v4.8b                     //GHASH final-2 block - mid
        mov     x6, v0.d[0]                           //AES final-1 block - mov low

        mov     x7, v0.d[1]                           //AES final-1 block - mov high
        eor     v11.16b, v11.16b, v21.16b                           //GHASH final-2 block - low
        movi    v8.8b, #0                                       //suppress further partial tag feed in

        pmull   v22.1q, v22.1d, v17.1d                     //GHASH final-2 block - mid

        eor     v9.16b, v9.16b, v20.16b                           //GHASH final-2 block - high
        eor     x6, x6, x13                  //AES final-1 block - round 14 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif

        eor     v10.16b, v10.16b, v22.16b                      //GHASH final-2 block - mid
        eor     x7, x7, x14                  //AES final-1 block - round 14 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
.L256_dec_blocks_more_than_1:   //blocks        left >  1

        stp     x6, x7, [x2], #16       //AES final-1 block  - store result
        rev64   v4.16b, v5.16b                                   //GHASH final-1 block

        ld1     { v5.16b}, [x0], #16                     //AES final block - load ciphertext

        eor     v4.16b, v4.16b, v8.16b                          //feed in partial tag
        movi    v8.8b, #0                                       //suppress further partial tag feed in

        mov     d22, v4.d[1]                                //GHASH final-1 block - mid

        eor     v0.16b, v5.16b, v3.16b                           //AES final block - result

        pmull2  v20.1q, v4.2d, v13.2d                         //GHASH final-1 block - high

        eor     v22.8b, v22.8b, v4.8b                     //GHASH final-1 block - mid

        pmull   v21.1q, v4.1d, v13.1d                         //GHASH final-1 block - low
        mov     x6, v0.d[0]                           //AES final block - mov low

        ins     v22.d[1], v22.d[0]                           //GHASH final-1 block - mid

        mov     x7, v0.d[1]                           //AES final block - mov high

        pmull2  v22.1q, v22.2d, v16.2d                     //GHASH final-1 block - mid
        eor     x6, x6, x13                  //AES final block - round 14 low
#ifdef __AARCH64EB__
        rev     x6, x6
#endif
        eor     v11.16b, v11.16b, v21.16b                           //GHASH final-1 block - low

        eor     v9.16b, v9.16b, v20.16b                           //GHASH final-1 block - high

        eor     v10.16b, v10.16b, v22.16b                      //GHASH final-1 block - mid
        eor     x7, x7, x14                  //AES final block - round 14 high
#ifdef __AARCH64EB__
        rev     x7, x7
#endif
.L256_dec_blocks_less_than_1:   //blocks        left <= 1

        and     x1, x1, #127                   //bit_length %= 128
        mvn     x14, xzr                                     //rk14_h = 0xffffffffffffffff

        sub     x1, x1, #128                   //bit_length -= 128
        mvn     x13, xzr                                     //rk14_l = 0xffffffffffffffff

        ldp     x4, x5, [x2] //load existing bytes we need to not overwrite
        neg     x1, x1                         //bit_length = 128 - #bits in input (in range [1,128])

        and     x1, x1, #127                   //bit_length %= 128

        lsr     x14, x14, x1                    //rk14_h is mask for top 64b of last block
        cmp     x1, #64

        csel    x9, x13, x14, lt
        csel    x10, x14, xzr, lt

        fmov    d0, x9                                  //ctr0b is mask for last block
        and     x6, x6, x9

        mov     v0.d[1], x10
        bic     x4, x4, x9          //mask out low existing bytes

#ifndef __AARCH64EB__
        rev     w9, w12
#else
        mov     w9, w12
#endif

        bic     x5, x5, x10      //mask out high existing bytes

        orr     x6, x6, x4

        and     x7, x7, x10

        orr     x7, x7, x5

        and     v5.16b, v5.16b, v0.16b                            //possibly partial last block has zeroes in highest bits

        rev64   v4.16b, v5.16b                                    //GHASH final block

        eor     v4.16b, v4.16b, v8.16b                           //feed in partial tag

        pmull   v21.1q, v4.1d, v12.1d                          //GHASH final block - low

        mov     d8, v4.d[1]                                  //GHASH final block - mid

        eor     v8.8b, v8.8b, v4.8b                          //GHASH final block - mid

        pmull2  v20.1q, v4.2d, v12.2d                          //GHASH final block - high

        pmull   v8.1q, v8.1d, v16.1d                          //GHASH final block - mid

        eor     v9.16b, v9.16b, v20.16b                            //GHASH final block - high

        eor     v11.16b, v11.16b, v21.16b                            //GHASH final block - low

        eor     v10.16b, v10.16b, v8.16b                         //GHASH final block - mid
        movi    v8.8b, #0xc2

        eor     v6.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

        shl     d8, d8, #56               //mod_constant

        eor     v10.16b, v10.16b, v6.16b                         //MODULO - karatsuba tidy up

        pmull   v7.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid

        ext     v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

        eor     v10.16b, v10.16b, v7.16b                      //MODULO - fold into mid

        eor     v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

        pmull   v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low

        ext     v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

        eor     v11.16b, v11.16b, v8.16b               //MODULO - fold into low

        stp     x6, x7, [x2]

        str     w9, [x16, #12]                          //store the updated counter

        eor     v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
        ext     v11.16b, v11.16b, v11.16b, #8
        rev64   v11.16b, v11.16b
        mov     x0, x15
        st1     { v11.16b }, [x3]

        ldp     x21, x22, [sp, #16]
        ldp     x23, x24, [sp, #32]
        ldp     d8, d9, [sp, #48]
        ldp     d10, d11, [sp, #64]
        ldp     d12, d13, [sp, #80]
        ldp     d14, d15, [sp, #96]
        ldp     x19, x20, [sp], #112
        ret

.L256_dec_ret:
        mov     w0, #0x0
        ret
.size   aes_gcm_dec_256_kernel,.-aes_gcm_dec_256_kernel
.section        .rodata
.byte   71,72,65,83,72,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
.align  2
.align  2
#endif