diff --git a/pasta_curves/CHANGELOG.md b/pasta_curves/CHANGELOG.md index b626fccd..2752cacd 100644 --- a/pasta_curves/CHANGELOG.md +++ b/pasta_curves/CHANGELOG.md @@ -32,3 +32,6 @@ and this project adheres to Rust's notion of the following multiplication. The sequence of field operations (and thus the variable-time profile, which depends only on the exponent) is unchanged. +- The `x86_64-asm` backend now uses a dedicated squaring schedule for `Fp` and + `Fq`, instead of passing identical operands to the general multiplication + routine. diff --git a/pasta_curves/src/asm/pasta_mulx-x86_64-coff.s b/pasta_curves/src/asm/pasta_mulx-x86_64-coff.s index 8cfe2906..dfdc543b 100644 --- a/pasta_curves/src/asm/pasta_mulx-x86_64-coff.s +++ b/pasta_curves/src/asm/pasta_mulx-x86_64-coff.s @@ -6,8 +6,9 @@ # 13ffc78074a6fbec44a4fd12b7f585a0bc1dc154: # https://github.com/supranational/semolina # -# Generated from pasta_mulx-x86_64.pl with only Montgomery multiplication, -# squaring, and their shared helper retained. Symbols are crate-prefixed. +# Montgomery multiplication and its helper are generated from +# pasta_mulx-x86_64.pl. The specialized square is a direct x86-64 translation +# of pasta_mul-armv8.S. Symbols are crate-prefixed. # The routines have no secret-dependent branches or memory accesses. Their # reduction specializes the shared high limbs of the two Pasta moduli. @@ -109,37 +110,198 @@ pasta_curves_sqrx_mont: pushq %r15 - subq $8,%rsp + subq $40,%rsp .LSEH_body_pasta_curves_sqrx_mont: + movq %rdi,%rbx + movq %rdx,%rbp - movq %rsi,%rbx - movq %rcx,%r8 - movq %rdx,%rcx + # Form the six off-diagonal products once. movq 0(%rsi),%rdx - movq 8(%rsi),%r15 - movq 16(%rsi),%rbp - movq 24(%rsi),%r9 - leaq -128(%rbx),%rsi - leaq -128(%rcx),%rcx - - mulxq %rdx,%rax,%r11 - call __pasta_curves_mulx_mont - - movq 8(%rsp),%r15 - - movq 16(%rsp),%r14 - - movq 24(%rsp),%r13 - - movq 32(%rsp),%r12 + mulxq 8(%rsi),%r9,%rax + mulxq 16(%rsi),%r10,%rdi + addq %rax,%r10 + adcq $0,%rdi + mulxq 24(%rsi),%r11,%r12 + addq %rdi,%r11 + adcq $0,%r12 - movq 40(%rsp),%rbx + movq 8(%rsi),%rdx + mulxq 16(%rsi),%rax,%rdi + addq %rax,%r11 + adcq $0,%rdi + mulxq 24(%rsi),%rax,%r13 + addq %r12,%rax + adcq $0,%r13 + addq %rdi,%rax + adcq $0,%r13 + movq %rax,%r12 - movq 48(%rsp),%rbp + movq 16(%rsi),%rdx + mulxq 24(%rsi),%rax,%r14 + addq %r13,%rax + adcq $0,%r14 + movq %rax,%r13 - leaq 56(%rsp),%rsp + # Double the off-diagonal half of the square. + xorq %r15,%r15 + addq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %r12,%r12 + adcq %r13,%r13 + adcq %r14,%r14 + adcq %r15,%r15 + + # Add the four diagonal products. + movq 0(%rsi),%rdx + mulxq %rdx,%r8,%rax + addq %rax,%r9 + movq 8(%rsi),%rdx + mulxq %rdx,%rax,%rdi + adcq %r10,%rax + adcq $0,%rdi + movq %rax,%r10 + addq %rdi,%r11 + movq 16(%rsi),%rdx + mulxq %rdx,%rax,%rdi + adcq %r12,%rax + adcq $0,%rdi + movq %rax,%r12 + addq %rdi,%r13 + movq 24(%rsi),%rdx + mulxq %rdx,%rax,%rdi + adcq %r14,%rax + adcq $0,%rdi + movq %rax,%r14 + addq %rdi,%r15 + + # Preserve the upper half while reducing the lower half by R. + movq %r12,0(%rsp) + movq %r13,8(%rsp) + movq %r14,16(%rsp) + movq %r15,24(%rsp) + movq 0(%rbp),%r12 + movq 8(%rbp),%r13 + + # Montgomery cancellation 0. The p[2] product is zero and the + # p[3] = 2^62 product is formed with shifts. + movq %r8,%rdx + imulq %rcx,%rdx + mulxq %r13,%r15,%rdi + mulxq %r12,%rax,%r14 + movq %rdx,%rax + shlq $62,%rax + shrq $2,%rdx + negq %r8 + adcq %r15,%r9 + adcq $0,%r10 + adcq %rax,%r11 + movq $0,%r8 + adcq $0,%r8 + addq %r14,%r9 + adcq %rdi,%r10 + adcq $0,%r11 + adcq %rdx,%r8 + + # Montgomery cancellation 1. + movq %r9,%rdx + imulq %rcx,%rdx + mulxq %r13,%r15,%rdi + mulxq %r12,%rax,%r14 + movq %rdx,%rax + shlq $62,%rax + shrq $2,%rdx + negq %r9 + adcq %r15,%r10 + adcq $0,%r11 + adcq %rax,%r8 + movq $0,%r9 + adcq $0,%r9 + addq %r14,%r10 + adcq %rdi,%r11 + adcq $0,%r8 + adcq %rdx,%r9 + + # Montgomery cancellation 2. + movq %r10,%rdx + imulq %rcx,%rdx + mulxq %r13,%r15,%rdi + mulxq %r12,%rax,%r14 + movq %rdx,%rax + shlq $62,%rax + shrq $2,%rdx + negq %r10 + adcq %r15,%r11 + adcq $0,%r8 + adcq %rax,%r9 + movq $0,%r10 + adcq $0,%r10 + addq %r14,%r11 + adcq %rdi,%r8 + adcq $0,%r9 + adcq %rdx,%r10 + + # Montgomery cancellation 3. + movq %r11,%rdx + imulq %rcx,%rdx + mulxq %r13,%r15,%rdi + mulxq %r12,%rax,%r14 + movq %rdx,%rax + shlq $62,%rax + shrq $2,%rdx + negq %r11 + adcq %r15,%r8 + adcq $0,%r9 + adcq %rax,%r10 + movq $0,%r11 + adcq $0,%r11 + addq %r14,%r8 + adcq %rdi,%r9 + adcq $0,%r10 + adcq %rdx,%r11 + + # Add the untouched upper half and capture a possible 257th bit. + addq 0(%rsp),%r8 + adcq 8(%rsp),%r9 + adcq 16(%rsp),%r10 + adcq 24(%rsp),%r11 + movq $0,%rsi + adcq $0,%rsi + + # Canonicalize with one constant-time conditional subtraction. + movq %r8,%r14 + movq %r9,%r15 + movq %r10,%rax + movq %r11,%rdi + subq %r12,%r8 + sbbq %r13,%r9 + sbbq $0,%r10 + sbbq 24(%rbp),%r11 + sbbq $0,%rsi + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + cmovcq %rdi,%r11 + movq %r8,0(%rbx) + movq %r9,8(%rbx) + movq %r10,16(%rbx) + movq %r11,24(%rbx) + + movq 40(%rsp),%r15 + + movq 48(%rsp),%r14 + + movq 56(%rsp),%r13 + + movq 64(%rsp),%r12 + + movq 72(%rsp),%rbx + + movq 80(%rsp),%rbp + + leaq 88(%rsp),%rsp .LSEH_epilogue_pasta_curves_sqrx_mont: mov 8(%rsp),%rdi @@ -395,15 +557,15 @@ __pasta_curves_mulx_mont: .byte 0,0 .LSEH_info_pasta_curves_sqrx_mont_body: .byte 1,0,17,0 -.byte 0x00,0xf4,0x01,0x00 -.byte 0x00,0xe4,0x02,0x00 -.byte 0x00,0xd4,0x03,0x00 -.byte 0x00,0xc4,0x04,0x00 -.byte 0x00,0x34,0x05,0x00 -.byte 0x00,0x54,0x06,0x00 -.byte 0x00,0x74,0x08,0x00 -.byte 0x00,0x64,0x09,0x00 -.byte 0x00,0x62 +.byte 0x00,0xf4,0x05,0x00 +.byte 0x00,0xe4,0x06,0x00 +.byte 0x00,0xd4,0x07,0x00 +.byte 0x00,0xc4,0x08,0x00 +.byte 0x00,0x34,0x09,0x00 +.byte 0x00,0x54,0x0a,0x00 +.byte 0x00,0x74,0x0c,0x00 +.byte 0x00,0x64,0x0d,0x00 +.byte 0x00,0xa2 .byte 0x00,0x00 .LSEH_info_pasta_curves_sqrx_mont_epilogue: .byte 1,0,4,0 diff --git a/pasta_curves/src/asm/pasta_mulx-x86_64-elf.s b/pasta_curves/src/asm/pasta_mulx-x86_64-elf.s index f723325d..079a9729 100644 --- a/pasta_curves/src/asm/pasta_mulx-x86_64-elf.s +++ b/pasta_curves/src/asm/pasta_mulx-x86_64-elf.s @@ -6,8 +6,9 @@ # 13ffc78074a6fbec44a4fd12b7f585a0bc1dc154: # https://github.com/supranational/semolina # -# Generated from pasta_mulx-x86_64.pl with only Montgomery multiplication, -# squaring, and their shared helper retained. Symbols are crate-prefixed. +# Montgomery multiplication and its helper are generated from +# pasta_mulx-x86_64.pl. The specialized square is a direct x86-64 translation +# of pasta_mul-armv8.S. Symbols are crate-prefixed. # The routines have no secret-dependent branches or memory accesses. Their # reduction specializes the shared high limbs of the two Pasta moduli. @@ -83,7 +84,6 @@ pasta_curves_sqrx_mont: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa - pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -102,37 +102,198 @@ pasta_curves_sqrx_mont: pushq %r15 .cfi_adjust_cfa_offset 8 .cfi_offset %r15,-56 - subq $8,%rsp -.cfi_adjust_cfa_offset 8 + subq $40,%rsp +.cfi_adjust_cfa_offset 40 + movq %rdi,%rbx + movq %rdx,%rbp - movq %rsi,%rbx - movq %rcx,%r8 - movq %rdx,%rcx + # Form the six off-diagonal products once. movq 0(%rsi),%rdx - movq 8(%rsi),%r15 - movq 16(%rsi),%rbp - movq 24(%rsi),%r9 - leaq -128(%rbx),%rsi - leaq -128(%rcx),%rcx + mulxq 8(%rsi),%r9,%rax + mulxq 16(%rsi),%r10,%rdi + addq %rax,%r10 + adcq $0,%rdi + mulxq 24(%rsi),%r11,%r12 + addq %rdi,%r11 + adcq $0,%r12 - mulxq %rdx,%rax,%r11 - call __pasta_curves_mulx_mont + movq 8(%rsi),%rdx + mulxq 16(%rsi),%rax,%rdi + addq %rax,%r11 + adcq $0,%rdi + mulxq 24(%rsi),%rax,%r13 + addq %r12,%rax + adcq $0,%r13 + addq %rdi,%rax + adcq $0,%r13 + movq %rax,%r12 - movq 8(%rsp),%r15 + movq 16(%rsi),%rdx + mulxq 24(%rsi),%rax,%r14 + addq %r13,%rax + adcq $0,%r14 + movq %rax,%r13 + + # Double the off-diagonal half of the square. + xorq %r15,%r15 + addq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %r12,%r12 + adcq %r13,%r13 + adcq %r14,%r14 + adcq %r15,%r15 + + # Add the four diagonal products. + movq 0(%rsi),%rdx + mulxq %rdx,%r8,%rax + addq %rax,%r9 + movq 8(%rsi),%rdx + mulxq %rdx,%rax,%rdi + adcq %r10,%rax + adcq $0,%rdi + movq %rax,%r10 + addq %rdi,%r11 + movq 16(%rsi),%rdx + mulxq %rdx,%rax,%rdi + adcq %r12,%rax + adcq $0,%rdi + movq %rax,%r12 + addq %rdi,%r13 + movq 24(%rsi),%rdx + mulxq %rdx,%rax,%rdi + adcq %r14,%rax + adcq $0,%rdi + movq %rax,%r14 + addq %rdi,%r15 + + # Preserve the upper half while reducing the lower half by R. + movq %r12,0(%rsp) + movq %r13,8(%rsp) + movq %r14,16(%rsp) + movq %r15,24(%rsp) + movq 0(%rbp),%r12 + movq 8(%rbp),%r13 + + # Montgomery cancellation 0. The p[2] product is zero and the + # p[3] = 2^62 product is formed with shifts. + movq %r8,%rdx + imulq %rcx,%rdx + mulxq %r13,%r15,%rdi + mulxq %r12,%rax,%r14 + movq %rdx,%rax + shlq $62,%rax + shrq $2,%rdx + negq %r8 + adcq %r15,%r9 + adcq $0,%r10 + adcq %rax,%r11 + movq $0,%r8 + adcq $0,%r8 + addq %r14,%r9 + adcq %rdi,%r10 + adcq $0,%r11 + adcq %rdx,%r8 + + # Montgomery cancellation 1. + movq %r9,%rdx + imulq %rcx,%rdx + mulxq %r13,%r15,%rdi + mulxq %r12,%rax,%r14 + movq %rdx,%rax + shlq $62,%rax + shrq $2,%rdx + negq %r9 + adcq %r15,%r10 + adcq $0,%r11 + adcq %rax,%r8 + movq $0,%r9 + adcq $0,%r9 + addq %r14,%r10 + adcq %rdi,%r11 + adcq $0,%r8 + adcq %rdx,%r9 + + # Montgomery cancellation 2. + movq %r10,%rdx + imulq %rcx,%rdx + mulxq %r13,%r15,%rdi + mulxq %r12,%rax,%r14 + movq %rdx,%rax + shlq $62,%rax + shrq $2,%rdx + negq %r10 + adcq %r15,%r11 + adcq $0,%r8 + adcq %rax,%r9 + movq $0,%r10 + adcq $0,%r10 + addq %r14,%r11 + adcq %rdi,%r8 + adcq $0,%r9 + adcq %rdx,%r10 + + # Montgomery cancellation 3. + movq %r11,%rdx + imulq %rcx,%rdx + mulxq %r13,%r15,%rdi + mulxq %r12,%rax,%r14 + movq %rdx,%rax + shlq $62,%rax + shrq $2,%rdx + negq %r11 + adcq %r15,%r8 + adcq $0,%r9 + adcq %rax,%r10 + movq $0,%r11 + adcq $0,%r11 + addq %r14,%r8 + adcq %rdi,%r9 + adcq $0,%r10 + adcq %rdx,%r11 + + # Add the untouched upper half and capture a possible 257th bit. + addq 0(%rsp),%r8 + adcq 8(%rsp),%r9 + adcq 16(%rsp),%r10 + adcq 24(%rsp),%r11 + movq $0,%rsi + adcq $0,%rsi + + # Canonicalize with one constant-time conditional subtraction. + movq %r8,%r14 + movq %r9,%r15 + movq %r10,%rax + movq %r11,%rdi + subq %r12,%r8 + sbbq %r13,%r9 + sbbq $0,%r10 + sbbq 24(%rbp),%r11 + sbbq $0,%rsi + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + cmovcq %rdi,%r11 + movq %r8,0(%rbx) + movq %r9,8(%rbx) + movq %r10,16(%rbx) + movq %r11,24(%rbx) + + movq 40(%rsp),%r15 .cfi_restore %r15 - movq 16(%rsp),%r14 + movq 48(%rsp),%r14 .cfi_restore %r14 - movq 24(%rsp),%r13 + movq 56(%rsp),%r13 .cfi_restore %r13 - movq 32(%rsp),%r12 + movq 64(%rsp),%r12 .cfi_restore %r12 - movq 40(%rsp),%rbx + movq 72(%rsp),%rbx .cfi_restore %rbx - movq 48(%rsp),%rbp + movq 80(%rsp),%rbp .cfi_restore %rbp - leaq 56(%rsp),%rsp -.cfi_adjust_cfa_offset -56 + leaq 88(%rsp),%rsp +.cfi_adjust_cfa_offset -88 .byte 0xf3,0xc3 .cfi_endproc diff --git a/pasta_curves/src/asm/pasta_mulx-x86_64-macosx.s b/pasta_curves/src/asm/pasta_mulx-x86_64-macosx.s index e9d0be2e..8f9e8814 100644 --- a/pasta_curves/src/asm/pasta_mulx-x86_64-macosx.s +++ b/pasta_curves/src/asm/pasta_mulx-x86_64-macosx.s @@ -6,8 +6,9 @@ # 13ffc78074a6fbec44a4fd12b7f585a0bc1dc154: # https://github.com/supranational/semolina # -# Generated from pasta_mulx-x86_64.pl with only Montgomery multiplication, -# squaring, and their shared helper retained. Symbols are crate-prefixed. +# Montgomery multiplication and its helper are generated from +# pasta_mulx-x86_64.pl. The specialized square is a direct x86-64 translation +# of pasta_mul-armv8.S. Symbols are crate-prefixed. # The routines have no secret-dependent branches or memory accesses. Their # reduction specializes the shared high limbs of the two Pasta moduli. @@ -83,7 +84,6 @@ _pasta_curves_sqrx_mont: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa - pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -102,37 +102,198 @@ _pasta_curves_sqrx_mont: pushq %r15 .cfi_adjust_cfa_offset 8 .cfi_offset %r15,-56 - subq $8,%rsp -.cfi_adjust_cfa_offset 8 + subq $40,%rsp +.cfi_adjust_cfa_offset 40 + movq %rdi,%rbx + movq %rdx,%rbp - movq %rsi,%rbx - movq %rcx,%r8 - movq %rdx,%rcx + # Form the six off-diagonal products once. movq 0(%rsi),%rdx - movq 8(%rsi),%r15 - movq 16(%rsi),%rbp - movq 24(%rsi),%r9 - leaq -128(%rbx),%rsi - leaq -128(%rcx),%rcx + mulxq 8(%rsi),%r9,%rax + mulxq 16(%rsi),%r10,%rdi + addq %rax,%r10 + adcq $0,%rdi + mulxq 24(%rsi),%r11,%r12 + addq %rdi,%r11 + adcq $0,%r12 - mulxq %rdx,%rax,%r11 - call __pasta_curves_mulx_mont + movq 8(%rsi),%rdx + mulxq 16(%rsi),%rax,%rdi + addq %rax,%r11 + adcq $0,%rdi + mulxq 24(%rsi),%rax,%r13 + addq %r12,%rax + adcq $0,%r13 + addq %rdi,%rax + adcq $0,%r13 + movq %rax,%r12 - movq 8(%rsp),%r15 + movq 16(%rsi),%rdx + mulxq 24(%rsi),%rax,%r14 + addq %r13,%rax + adcq $0,%r14 + movq %rax,%r13 + + # Double the off-diagonal half of the square. + xorq %r15,%r15 + addq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %r12,%r12 + adcq %r13,%r13 + adcq %r14,%r14 + adcq %r15,%r15 + + # Add the four diagonal products. + movq 0(%rsi),%rdx + mulxq %rdx,%r8,%rax + addq %rax,%r9 + movq 8(%rsi),%rdx + mulxq %rdx,%rax,%rdi + adcq %r10,%rax + adcq $0,%rdi + movq %rax,%r10 + addq %rdi,%r11 + movq 16(%rsi),%rdx + mulxq %rdx,%rax,%rdi + adcq %r12,%rax + adcq $0,%rdi + movq %rax,%r12 + addq %rdi,%r13 + movq 24(%rsi),%rdx + mulxq %rdx,%rax,%rdi + adcq %r14,%rax + adcq $0,%rdi + movq %rax,%r14 + addq %rdi,%r15 + + # Preserve the upper half while reducing the lower half by R. + movq %r12,0(%rsp) + movq %r13,8(%rsp) + movq %r14,16(%rsp) + movq %r15,24(%rsp) + movq 0(%rbp),%r12 + movq 8(%rbp),%r13 + + # Montgomery cancellation 0. The p[2] product is zero and the + # p[3] = 2^62 product is formed with shifts. + movq %r8,%rdx + imulq %rcx,%rdx + mulxq %r13,%r15,%rdi + mulxq %r12,%rax,%r14 + movq %rdx,%rax + shlq $62,%rax + shrq $2,%rdx + negq %r8 + adcq %r15,%r9 + adcq $0,%r10 + adcq %rax,%r11 + movq $0,%r8 + adcq $0,%r8 + addq %r14,%r9 + adcq %rdi,%r10 + adcq $0,%r11 + adcq %rdx,%r8 + + # Montgomery cancellation 1. + movq %r9,%rdx + imulq %rcx,%rdx + mulxq %r13,%r15,%rdi + mulxq %r12,%rax,%r14 + movq %rdx,%rax + shlq $62,%rax + shrq $2,%rdx + negq %r9 + adcq %r15,%r10 + adcq $0,%r11 + adcq %rax,%r8 + movq $0,%r9 + adcq $0,%r9 + addq %r14,%r10 + adcq %rdi,%r11 + adcq $0,%r8 + adcq %rdx,%r9 + + # Montgomery cancellation 2. + movq %r10,%rdx + imulq %rcx,%rdx + mulxq %r13,%r15,%rdi + mulxq %r12,%rax,%r14 + movq %rdx,%rax + shlq $62,%rax + shrq $2,%rdx + negq %r10 + adcq %r15,%r11 + adcq $0,%r8 + adcq %rax,%r9 + movq $0,%r10 + adcq $0,%r10 + addq %r14,%r11 + adcq %rdi,%r8 + adcq $0,%r9 + adcq %rdx,%r10 + + # Montgomery cancellation 3. + movq %r11,%rdx + imulq %rcx,%rdx + mulxq %r13,%r15,%rdi + mulxq %r12,%rax,%r14 + movq %rdx,%rax + shlq $62,%rax + shrq $2,%rdx + negq %r11 + adcq %r15,%r8 + adcq $0,%r9 + adcq %rax,%r10 + movq $0,%r11 + adcq $0,%r11 + addq %r14,%r8 + adcq %rdi,%r9 + adcq $0,%r10 + adcq %rdx,%r11 + + # Add the untouched upper half and capture a possible 257th bit. + addq 0(%rsp),%r8 + adcq 8(%rsp),%r9 + adcq 16(%rsp),%r10 + adcq 24(%rsp),%r11 + movq $0,%rsi + adcq $0,%rsi + + # Canonicalize with one constant-time conditional subtraction. + movq %r8,%r14 + movq %r9,%r15 + movq %r10,%rax + movq %r11,%rdi + subq %r12,%r8 + sbbq %r13,%r9 + sbbq $0,%r10 + sbbq 24(%rbp),%r11 + sbbq $0,%rsi + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + cmovcq %rdi,%r11 + movq %r8,0(%rbx) + movq %r9,8(%rbx) + movq %r10,16(%rbx) + movq %r11,24(%rbx) + + movq 40(%rsp),%r15 .cfi_restore %r15 - movq 16(%rsp),%r14 + movq 48(%rsp),%r14 .cfi_restore %r14 - movq 24(%rsp),%r13 + movq 56(%rsp),%r13 .cfi_restore %r13 - movq 32(%rsp),%r12 + movq 64(%rsp),%r12 .cfi_restore %r12 - movq 40(%rsp),%rbx + movq 72(%rsp),%rbx .cfi_restore %rbx - movq 48(%rsp),%rbp + movq 80(%rsp),%rbp .cfi_restore %rbp - leaq 56(%rsp),%rsp -.cfi_adjust_cfa_offset -56 + leaq 88(%rsp),%rsp +.cfi_adjust_cfa_offset -88 .byte 0xf3,0xc3 .cfi_endproc diff --git a/pasta_curves/src/asm/pasta_mulx-x86_64-win64.asm b/pasta_curves/src/asm/pasta_mulx-x86_64-win64.asm index 1dea0d6b..465dab86 100644 --- a/pasta_curves/src/asm/pasta_mulx-x86_64-win64.asm +++ b/pasta_curves/src/asm/pasta_mulx-x86_64-win64.asm @@ -6,8 +6,9 @@ ; 13ffc78074a6fbec44a4fd12b7f585a0bc1dc154: ; https://github.com/supranational/semolina ; -; Generated from pasta_mulx-x86_64.pl with only Montgomery multiplication, -; squaring, and their shared helper retained. Symbols are crate-prefixed. +; Montgomery multiplication and its helper are generated from +; pasta_mulx-x86_64.pl. The specialized square is a direct x86-64 translation +; of pasta_mul-armv8.S. Symbols are crate-prefixed. ; The routines have no secret-dependent branches or memory accesses. Their ; reduction specializes the shared high limbs of the two Pasta moduli. @@ -113,37 +114,198 @@ $L$SEH_begin_pasta_curves_sqrx_mont:: push r15 - sub rsp,8 + sub rsp,40 $L$SEH_body_pasta_curves_sqrx_mont:: + mov rbx,rdi + mov rbp,rdx - mov rbx,rsi - mov r8,rcx - mov rcx,rdx + ; Form the six off-diagonal products once. mov rdx,QWORD PTR[rsi] - mov r15,QWORD PTR[8+rsi] - mov rbp,QWORD PTR[16+rsi] - mov r9,QWORD PTR[24+rsi] - lea rsi,QWORD PTR[((-128))+rbx] - lea rcx,QWORD PTR[((-128))+rcx] - - mulx r11,rax,rdx - call __pasta_curves_mulx_mont - - mov r15,QWORD PTR[8+rsp] - - mov r14,QWORD PTR[16+rsp] - - mov r13,QWORD PTR[24+rsp] - - mov r12,QWORD PTR[32+rsp] + mulx rax,r9,QWORD PTR[8+rsi] + mulx rdi,r10,QWORD PTR[16+rsi] + add r10,rax + adc rdi,0 + mulx r12,r11,QWORD PTR[24+rsi] + add r11,rdi + adc r12,0 - mov rbx,QWORD PTR[40+rsp] + mov rdx,QWORD PTR[8+rsi] + mulx rdi,rax,QWORD PTR[16+rsi] + add r11,rax + adc rdi,0 + mulx r13,rax,QWORD PTR[24+rsi] + add rax,r12 + adc r13,0 + add rax,rdi + adc r13,0 + mov r12,rax - mov rbp,QWORD PTR[48+rsp] + mov rdx,QWORD PTR[16+rsi] + mulx r14,rax,QWORD PTR[24+rsi] + add rax,r13 + adc r14,0 + mov r13,rax - lea rsp,QWORD PTR[56+rsp] + ; Double the off-diagonal half of the square. + xor r15,r15 + add r9,r9 + adc r10,r10 + adc r11,r11 + adc r12,r12 + adc r13,r13 + adc r14,r14 + adc r15,r15 + + ; Add the four diagonal products. + mov rdx,QWORD PTR[rsi] + mulx rax,r8,rdx + add r9,rax + mov rdx,QWORD PTR[8+rsi] + mulx rdi,rax,rdx + adc rax,r10 + adc rdi,0 + mov r10,rax + add r11,rdi + mov rdx,QWORD PTR[16+rsi] + mulx rdi,rax,rdx + adc rax,r12 + adc rdi,0 + mov r12,rax + add r13,rdi + mov rdx,QWORD PTR[24+rsi] + mulx rdi,rax,rdx + adc rax,r14 + adc rdi,0 + mov r14,rax + add r15,rdi + + ; Preserve the upper half while reducing the lower half by R. + mov QWORD PTR[rsp],r12 + mov QWORD PTR[8+rsp],r13 + mov QWORD PTR[16+rsp],r14 + mov QWORD PTR[24+rsp],r15 + mov r12,QWORD PTR[rbp] + mov r13,QWORD PTR[8+rbp] + + ; Montgomery cancellation 0. The p[2] product is zero and the + ; p[3] = 2^62 product is formed with shifts. + mov rdx,r8 + imul rdx,rcx + mulx rdi,r15,r13 + mulx r14,rax,r12 + mov rax,rdx + shl rax,62 + shr rdx,2 + neg r8 + adc r9,r15 + adc r10,0 + adc r11,rax + mov r8,0 + adc r8,0 + add r9,r14 + adc r10,rdi + adc r11,0 + adc r8,rdx + + ; Montgomery cancellation 1. + mov rdx,r9 + imul rdx,rcx + mulx rdi,r15,r13 + mulx r14,rax,r12 + mov rax,rdx + shl rax,62 + shr rdx,2 + neg r9 + adc r10,r15 + adc r11,0 + adc r8,rax + mov r9,0 + adc r9,0 + add r10,r14 + adc r11,rdi + adc r8,0 + adc r9,rdx + + ; Montgomery cancellation 2. + mov rdx,r10 + imul rdx,rcx + mulx rdi,r15,r13 + mulx r14,rax,r12 + mov rax,rdx + shl rax,62 + shr rdx,2 + neg r10 + adc r11,r15 + adc r8,0 + adc r9,rax + mov r10,0 + adc r10,0 + add r11,r14 + adc r8,rdi + adc r9,0 + adc r10,rdx + + ; Montgomery cancellation 3. + mov rdx,r11 + imul rdx,rcx + mulx rdi,r15,r13 + mulx r14,rax,r12 + mov rax,rdx + shl rax,62 + shr rdx,2 + neg r11 + adc r8,r15 + adc r9,0 + adc r10,rax + mov r11,0 + adc r11,0 + add r8,r14 + adc r9,rdi + adc r10,0 + adc r11,rdx + + ; Add the untouched upper half and capture a possible 257th bit. + add r8,QWORD PTR[rsp] + adc r9,QWORD PTR[8+rsp] + adc r10,QWORD PTR[16+rsp] + adc r11,QWORD PTR[24+rsp] + mov rsi,0 + adc rsi,0 + + ; Canonicalize with one constant-time conditional subtraction. + mov r14,r8 + mov r15,r9 + mov rax,r10 + mov rdi,r11 + sub r8,r12 + sbb r9,r13 + sbb r10,0 + sbb r11,QWORD PTR[24+rbp] + sbb rsi,0 + cmovc r8,r14 + cmovc r9,r15 + cmovc r10,rax + cmovc r11,rdi + mov QWORD PTR[rbx],r8 + mov QWORD PTR[8+rbx],r9 + mov QWORD PTR[16+rbx],r10 + mov QWORD PTR[24+rbx],r11 + + mov r15,QWORD PTR[40+rsp] + + mov r14,QWORD PTR[48+rsp] + + mov r13,QWORD PTR[56+rsp] + + mov r12,QWORD PTR[64+rsp] + + mov rbx,QWORD PTR[72+rsp] + + mov rbp,QWORD PTR[80+rsp] + + lea rsp,QWORD PTR[88+rsp] $L$SEH_epilogue_pasta_curves_sqrx_mont:: mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue @@ -401,15 +563,15 @@ DB 0,003h DB 0,0 $L$SEH_info_pasta_curves_sqrx_mont_body:: DB 1,0,17,0 -DB 000h,0f4h,001h,000h -DB 000h,0e4h,002h,000h -DB 000h,0d4h,003h,000h -DB 000h,0c4h,004h,000h -DB 000h,034h,005h,000h -DB 000h,054h,006h,000h -DB 000h,074h,008h,000h -DB 000h,064h,009h,000h -DB 000h,062h +DB 000h,0f4h,005h,000h +DB 000h,0e4h,006h,000h +DB 000h,0d4h,007h,000h +DB 000h,0c4h,008h,000h +DB 000h,034h,009h,000h +DB 000h,054h,00ah,000h +DB 000h,074h,00ch,000h +DB 000h,064h,00dh,000h +DB 000h,0a2h DB 000h,000h $L$SEH_info_pasta_curves_sqrx_mont_epilogue:: DB 1,0,4,0