// ====================================================================
// Written by Andy Polyakov, @dot-asm, initially for use in the OpenSSL
// project.
// ====================================================================
//
// Poly1305 for Itanium.
//
// January 2019
//
// Performance was reported to be ~2.1 cycles per byte on Itanium 2.
// With exception for processors in 95xx family, which have higher
// floating-point instructions' latencies and deliver ~2.6 cpb.
// Comparison to compiler-generated code is not exactly fair, because
// of different radixes. But just for reference, it was observed to be
// >3x faster. Originally it was argued that floating-point base 2^32
// implementation would be optimal. Upon closer look estimate for below
// integer base 2^64 implementation turned to be approximately same on
// Itanium 2. But floating-point code would be larger, and have higher
// overhead, which would negatively affect small-block performance...

#if defined(_HPUX_SOURCE)
# if !defined(_LP64)
#  define ADDP  addp4
# else
#  define ADDP  add
# endif
# define RUM    rum
# define SUM    sum
#else
# define ADDP   add
# define RUM    nop
# define SUM    nop
#endif

.text
.explicit

.global poly1305_init#
.proc   poly1305_init#
.align  64
poly1305_init:
        .prologue
        .save           ar.pfs,r2
{ .mmi; alloc           r2=ar.pfs,2,0,0,0
        cmp.eq          p6,p7=0,r33             }       // key == NULL?
{ .mmi; ADDP            r9=8,r32
        ADDP            r10=16,r32
        ADDP            r32=0,r32               };;
        .body
{ .mmi; st8             [r32]=r0,24                     // ctx->h0 = 0
        st8             [r9]=r0                         // ctx->h1 = 0
(p7)    ADDP            r8=0,r33                }
{ .mib; st8             [r10]=r0                        // ctx->h2 = 0
(p6)    mov             r8=0
(p6)    br.ret.spnt     b0                      };;

{ .mmi; ADDP            r9=1,r33
        ADDP            r10=2,r33
        ADDP            r11=3,r33               };;
{ .mmi; ld1             r16=[r8],4                      // load key, little-endian
        ld1             r17=[r9],4              }
{ .mmi; ld1             r18=[r10],4
        ld1             r19=[r11],4             };;
{ .mmi; ld1             r20=[r8],4
        ld1             r21=[r9],4              }
{ .mmi; ld1             r22=[r10],4
        ld1             r23=[r11],4
        and             r19=15,r19              };;
{ .mmi; ld1             r24=[r8],4
        ld1             r25=[r9],4
        and             r20=-4,r20              }
{ .mmi; ld1             r26=[r10],4
        ld1             r27=[r11],4
        and             r23=15,r23              };;
{ .mmi; ld1             r28=[r8],4
        ld1             r29=[r9],4
        and             r24=-4,r24              }
{ .mmi; ld1             r30=[r10],4
        ld1             r31=[r11],4
        and             r27=15,r27              };;

{ .mii; and             r28=-4,r28
        dep             r16=r17,r16,8,8
        dep             r18=r19,r18,8,8         };;
{ .mii; and             r31=15,r31
        dep             r16=r18,r16,16,16
        dep             r20=r21,r20,8,8         };;
{ .mii; dep             r16=r20,r16,32,16
        dep             r22=r23,r22,8,8         };;
{ .mii; dep             r16=r22,r16,48,16
        dep             r24=r25,r24,8,8         };;
{ .mii; dep             r26=r27,r26,8,8
        dep             r28=r29,r28,8,8         };;
{ .mii; dep             r24=r26,r24,16,16
        dep             r30=r31,r30,8,8         };;
{ .mii; st8             [r32]=r16,8                     // ctx->r0
        dep             r24=r28,r24,32,16;;
        dep             r24=r30,r24,48,16       };;
{ .mii; st8             [r32]=r24,8                     // ctx->r1
        shr.u           r25=r24,2;;
        add             r25=r25,r24             };;
{ .mib; st8             [r32]=r25                       // ctx->s1
        mov             r8=0
        br.ret.sptk     b0                      };;
.endp   poly1305_init#

h0=r17;  h1=r18;  h2=r19;
i0=r20;  i1=r21;
HF0=f8;  HF1=f9;  HF2=f10;
RF0=f11; RF1=f12; SF1=f13;

.global poly1305_blocks#
.proc   poly1305_blocks#
.align  64
poly1305_blocks:
        .prologue
        .save           ar.pfs,r2
{ .mii; alloc           r2=ar.pfs,4,1,0,0
        .save           ar.lc,r3
        mov             r3=ar.lc
        .save           pr,r36
        mov             r36=pr                  }

        .body
{ .mmi; ADDP            r8=0,r32
        ADDP            r9=8,r32
        and             r29=7,r33               };;
{ .mmi; ld8             h0=[r8],16
        ld8             h1=[r9],16
        and             r33=-8,r33              };;
{ .mmi; ld8             h2=[r8],16
        ldf8            RF0=[r9],16
        shr.u           r34=r34,4               };;
{ .mmi; ldf8            RF1=[r8],-32
        ldf8            SF1=[r9],-32
        cmp.ltu         p16,p17=1,r34           };;
{ .mmi;
(p16)   add             r34=-2,r34
(p17)   mov             r34=0
        ADDP            r10=0,r33               }
{ .mii; ADDP            r11=8,r33
(p16)   mov             ar.ec=2
(p17)   mov             ar.ec=1                 };;
{ .mib; RUM             1<<1                            // go little-endian
        mov             ar.lc=r34
        brp.loop.imp    .Loop,.Lcend-16         }

{ .mmi; cmp.eq          p8,p7=0,r29
        cmp.eq          p9,p0=1,r29
        cmp.eq          p10,p0=2,r29            }
{ .mmi; cmp.eq          p11,p0=3,r29
        cmp.eq          p12,p0=4,r29
        cmp.eq          p13,p0=5,r29            }
{ .mmi; cmp.eq          p14,p0=6,r29
        cmp.eq          p15,p0=7,r29
        add             r16=16,r10              };;

{ .mmb;
(p8)    ld8             i0=[r10],16                     // aligned input
(p8)    ld8             i1=[r11],16
(p8)    br.cond.sptk    .Loop                   };;

        // align first block
        .pred.rel       "mutex",p8,p9,p10,p11,p12,p13,p14,p15
{ .mmi; (p7)    ld8             r14=[r10],24
        (p7)    ld8             r15=[r11],24            }

{ .mii; (p7)    ld8             r16=[r16]
                nop.i           0;;
        (p15)   shrp            i0=r15,r14,56           }
{ .mii; (p15)   shrp            i1=r16,r15,56
        (p14)   shrp            i0=r15,r14,48           }
{ .mii; (p14)   shrp            i1=r16,r15,48
        (p13)   shrp            i0=r15,r14,40           }
{ .mii; (p13)   shrp            i1=r16,r15,40
        (p12)   shrp            i0=r15,r14,32           }
{ .mii; (p12)   shrp            i1=r16,r15,32
        (p11)   shrp            i0=r15,r14,24           }
{ .mii; (p11)   shrp            i1=r16,r15,24
        (p10)   shrp            i0=r15,r14,16           }
{ .mii; (p10)   shrp            i1=r16,r15,16
        (p9)    shrp            i0=r15,r14,8            }
{ .mii; (p9)    shrp            i1=r16,r15,8
                mov             r14=r16                 };;

.Loop:
                .pred.rel       "mutex",p8,p9,p10,p11,p12,p13,p14,p15
{ .mmi;         add             h0=h0,i0
                add             h1=h1,i1
                add             h2=h2,r35               };;
{ .mmi;         setf.sig        HF0=h0
                cmp.ltu         p6,p0=h0,i0
                cmp.ltu         p7,p0=h1,i1             };;
{ .mmi; (p6)    add             h1=1,h1;;
                setf.sig        HF1=h1
        (p6)    cmp.eq.or       p7,p0=0,h1              };;
{ .mmi; (p7)    add             h2=1,h2;;
                setf.sig        HF2=h2                  };;

{ .mfi; (p16)   ld8             r15=[r10],16
                xmpy.lu         f32=HF0,RF0             }
{ .mfi; (p16)   ld8             r16=[r11],16
                xmpy.hu         f33=HF0,RF0             }
{ .mfi;         xmpy.lu         f36=HF0,RF1             }
{ .mfi;         xmpy.hu         f37=HF0,RF1             };;
{ .mfi;         xmpy.lu         f34=HF1,SF1
        (p15)   shrp            i0=r15,r14,56           }
{ .mfi;         xmpy.hu         f35=HF1,SF1             }
{ .mfi;         xmpy.lu         f38=HF1,RF0
        (p15)   shrp            i1=r16,r15,56           }
{ .mfi;         xmpy.hu         f39=HF1,RF0             }
{ .mfi;         xmpy.lu         f40=HF2,SF1
        (p14)   shrp            i0=r15,r14,48           }
{ .mfi;         xmpy.lu         f41=HF2,RF0             };;

{ .mmi;         getf.sig        r22=f32
                getf.sig        r23=f33
        (p14)   shrp            i1=r16,r15,48           }
{ .mmi;         getf.sig        r24=f34
                getf.sig        r25=f35
        (p13)   shrp            i0=r15,r14,40           }
{ .mmi;         getf.sig        r26=f36
                getf.sig        r27=f37
        (p13)   shrp            i1=r16,r15,40           }
{ .mmi;         getf.sig        r28=f38
                getf.sig        r29=f39
        (p12)   shrp            i0=r15,r14,32           }
{ .mmi;         getf.sig        r30=f40
                getf.sig        r31=f41                 };;

{ .mmi;         add             h0=r22,r24
                add             r23=r23,r25
        (p12)   shrp            i1=r16,r15,32           }
{ .mmi;         add             h1=r26,r28
                add             r27=r27,r29
        (p11)   shrp            i0=r15,r14,24           };;
{ .mmi;         cmp.ltu         p6,p0=h0,r24
                cmp.ltu         p7,p0=h1,r28
                add             r23=r23,r30             };;
{ .mmi; (p6)    add             r23=1,r23
        (p7)    add             r27=1,r27
        (p11)   shrp            i1=r16,r15,24           };;
{ .mmi;         add             h1=h1,r23;;
                cmp.ltu         p6,p7=h1,r23
        (p10)   shrp            i0=r15,r14,16           };;
{ .mmi; (p6)    add             h2=r31,r27,1
        (p7)    add             h2=r31,r27
        (p10)   shrp            i1=r16,r15,16           };;

{ .mmi; (p8)    mov             i0=r15
                and             r22=-4,h2
                shr.u           r23=h2,2                };;
{ .mmi;         add             r22=r22,r23
                and             h2=3,h2
        (p9)    shrp            i0=r15,r14,8            };;

{ .mmi;         add             h0=h0,r22;;
                cmp.ltu         p6,p0=h0,r22
        (p9)    shrp            i1=r16,r15,8            };;
{ .mmi; (p8)    mov             i1=r16
        (p6)    cmp.eq.unc      p7,p0=-1,h1
        (p6)    add             h1=1,h1                 };;
{ .mmb; (p7)    add             h2=1,h2
                mov             r14=r16
                br.ctop.sptk    .Loop                   };;
.Lcend:

{ .mii; SUM             1<<1                            // back to big-endian
        mov             ar.lc=r3                };;

{ .mmi; st8             [r8]=h0,16
        st8             [r9]=h1
        mov             pr=r36,0x1ffff          };;
{ .mmb; st8             [r8]=h2
        rum             1<<5
        br.ret.sptk     b0                      };;
.endp   poly1305_blocks#

.global poly1305_emit#
.proc   poly1305_emit#
.align  64
poly1305_emit:
        .prologue
        .save           ar.pfs,r2
{ .mmi; alloc           r2=ar.pfs,3,0,0,0
        ADDP            r8=0,r32
        ADDP            r9=8,r32                };;

        .body
{ .mmi; ld8             r16=[r8],16                     // load hash
        ld8             r17=[r9]
        ADDP            r10=0,r34               };;
{ .mmi; ld8             r18=[r8]
        ld4             r24=[r10],8                     // load nonce
        ADDP            r11=4,r34               };;

{ .mmi; ld4             r25=[r11],8
        ld4             r26=[r10]
        add             r20=5,r16               };;

{ .mmi; ld4             r27=[r11]
        cmp.ltu         p6,p7=r20,r16
        shl             r25=r25,32              };;
{ .mmi;
(p6)    add             r21=1,r17
(p7)    add             r21=0,r17
(p6)    cmp.eq.or.andcm p6,p7=-1,r17            };;
{ .mmi;
(p6)    add             r22=1,r18
(p7)    add             r22=0,r18
        shl             r27=r27,32              };;
{ .mmi; or              r24=r24,r25
        or              r26=r26,r27
        cmp.leu         p6,p7=4,r22             };;
{ .mmi;
(p6)    add             r16=r20,r24
(p7)    add             r16=r16,r24
(p6)    add             r17=r21,r26             };;
{ .mii;
(p7)    add             r17=r17,r26
        cmp.ltu         p6,p7=r16,r24;;
(p6)    add             r17=1,r17               };;

{ .mmi; ADDP            r8=0,r33
        ADDP            r9=4,r33
        shr.u           r20=r16,32              }
{ .mmi; ADDP            r10=8,r33
        ADDP            r11=12,r33
        shr.u           r21=r17,32              };;

{ .mmi; st1             [r8]=r16,1                      // write mac, little-endian
        st1             [r9]=r20,1
        shr.u           r16=r16,8               }
{ .mii; st1             [r10]=r17,1
        shr.u           r20=r20,8
        shr.u           r17=r17,8               }
{ .mmi; st1             [r11]=r21,1
        shr.u           r21=r21,8               };;

{ .mmi; st1             [r8]=r16,1
        st1             [r9]=r20,1
        shr.u           r16=r16,8               }
{ .mii; st1             [r10]=r17,1
        shr.u           r20=r20,8
        shr.u           r17=r17,8               }
{ .mmi; st1             [r11]=r21,1
        shr.u           r21=r21,8               };;

{ .mmi; st1             [r8]=r16,1
        st1             [r9]=r20,1
        shr.u           r16=r16,8               }
{ .mii; st1             [r10]=r17,1
        shr.u           r20=r20,8
        shr.u           r17=r17,8               }
{ .mmi; st1             [r11]=r21,1
        shr.u           r21=r21,8               };;

{ .mmi; st1             [r8]=r16
        st1             [r9]=r20                }
{ .mmb; st1             [r10]=r17
        st1             [r11]=r21
        br.ret.sptk     b0                      };;
.endp   poly1305_emit#

stringz "Poly1305 for IA64, CRYPTOGAMS by \@dot-asm"