diff --git a/src/deferred.rs b/src/deferred.rs index 990d393..2e254a0 100644 --- a/src/deferred.rs +++ b/src/deferred.rs @@ -1,12 +1,15 @@ //! Deferred normalization for field arithmetic. //! -//! This module provides the [`DeferredField`] trait, which enables accumulating -//! multiple unreduced products before performing a single expensive reduction. -//! This is useful for operations like inner products where many multiplications -//! feed into a sum. +//! This module provides the [`DeferredField`] trait and a wide [`Product`] +//! accumulator. Together they enable accumulating multiple unreduced +//! Montgomery products before performing a single expensive reduction. +//! This is useful for operations like inner products where many +//! multiplications feed into a sum. use core::fmt::Debug; +use crate::arithmetic::{adc, mac}; + /// A trait for fields that support deferred reduction of products. /// /// Instead of reducing each multiplication result immediately, callers @@ -28,6 +31,103 @@ pub trait DeferredField: ff::Field { fn reduce(acc: Self::Accumulator) -> Self; } +/// A wide accumulator for unreduced Montgomery products over field `F`. +/// +/// This stores a running sum of 512-bit products with a 64-bit carry for +/// overflow beyond 512 bits. Products are added internally by +/// [`DeferredField::mul_accumulate`] and [`DeferredField::square_accumulate`]. +/// +/// Call [`DeferredField::reduce`] to fold the carry back into range and +/// perform Montgomery reduction. +#[derive(Clone, Copy, Debug)] +pub struct Product { + limbs: [u64; 8], + carry: u64, + _marker: core::marker::PhantomData, +} + +impl Default for Product { + fn default() -> Self { + Self::ZERO + } +} + +impl Product { + /// The zero (additive identity) accumulator. + pub const ZERO: Self = Product { + limbs: [0; 8], + carry: 0, + _marker: core::marker::PhantomData, + }; + + /// Adds a raw 512-bit product (8 limbs) into this accumulator. + /// + /// Each call contributes at most 1 to `carry`; overflow of the 64-bit + /// carry requires 2^64 accumulated products (~590 exabytes of input). + #[inline] + pub(crate) fn accumulate(&mut self, product: [u64; 8]) { + let (d0, c) = adc(self.limbs[0], product[0], 0); + let (d1, c) = adc(self.limbs[1], product[1], c); + let (d2, c) = adc(self.limbs[2], product[2], c); + let (d3, c) = adc(self.limbs[3], product[3], c); + let (d4, c) = adc(self.limbs[4], product[4], c); + let (d5, c) = adc(self.limbs[5], product[5], c); + let (d6, c) = adc(self.limbs[6], product[6], c); + let (d7, c) = adc(self.limbs[7], product[7], c); + self.limbs = [d0, d1, d2, d3, d4, d5, d6, d7]; + let (carry, overflow) = self.carry.overflowing_add(c); + debug_assert!(!overflow, "carry overflow: too many accumulated products"); + self.carry = carry; + } + + /// Folds `carry` (bits 512+) and `limbs[7]` (bits 448–511) into the lower + /// 448 bits using precomputed residues of $2^{448}$ and $2^{512}$ modulo + /// the field prime. + /// + /// The result fits in 8 limbs with value $< 2^{449} < Rp$, safe for + /// Montgomery reduction. + #[cfg_attr(not(feature = "uninline-portable"), inline)] + pub(crate) fn partial_reduce(&self, b448: &[u64; 4], r2: &[u64; 4]) -> [u64; 8] { + let b7 = self.limbs[7]; + let b8 = self.carry; + + // Compute b7 * b448 (5 limbs) + let (t0, c) = mac(0, b7, b448[0], 0); + let (t1, c) = mac(0, b7, b448[1], c); + let (t2, c) = mac(0, b7, b448[2], c); + let (t3, c) = mac(0, b7, b448[3], c); + let t4 = c; + + // Accumulate b8 * r2 + let (t0, c) = mac(t0, b8, r2[0], 0); + let (t1, c) = mac(t1, b8, r2[1], c); + let (t2, c) = mac(t2, b8, r2[2], c); + let (t3, c) = mac(t3, b8, r2[3], c); + let (t4, t5) = adc(t4, 0, c); + debug_assert!( + t5 == 0, + "folding term overflow: t4 + carry does not fit in 64 bits" + ); + + // Add to lower 7 limbs + let (d0, c) = adc(self.limbs[0], t0, 0); + let (d1, c) = adc(self.limbs[1], t1, c); + let (d2, c) = adc(self.limbs[2], t2, c); + let (d3, c) = adc(self.limbs[3], t3, c); + let (d4, c) = adc(self.limbs[4], t4, c); + let (d5, c) = adc(self.limbs[5], 0, c); + let (d6, c) = adc(self.limbs[6], 0, c); + let (d7, _) = adc(0, 0, c); + + // B448 < 2^253 and r2 < 2^252, so the folding term + // b7 * B448 + b8 * r2 < 2^317 + 2^316 < 2^318. + // The full value is < 2^448 + 2^318 < 2^449, so d7 is at most 1. + debug_assert!(d7 <= 1); + + [d0, d1, d2, d3, d4, d5, d6, d7] + } +} + #[cfg(test)] mod tests { use super::DeferredField; diff --git a/src/fields/fp.rs b/src/fields/fp.rs index a613a5e..c475d98 100644 --- a/src/fields/fp.rs +++ b/src/fields/fp.rs @@ -13,7 +13,7 @@ use ff::{FieldBits, PrimeFieldBits}; use crate::arithmetic::{adc, mac, sbb, SqrtTableHelpers}; #[cfg(feature = "deferred")] -use crate::deferred::DeferredField; +use crate::deferred::{DeferredField, Product}; #[cfg(feature = "sqrt-table")] use crate::arithmetic::SqrtTables; @@ -481,21 +481,31 @@ impl Fp { #[cfg(feature = "deferred")] impl DeferredField for Fp { - type Accumulator = Fp; + type Accumulator = Product; - #[inline] - fn mul_accumulate(acc: &mut Fp, a: &Fp, b: &Fp) { - *acc += *a * *b; + #[cfg_attr(not(feature = "uninline-portable"), inline)] + fn mul_accumulate(acc: &mut Self::Accumulator, a: &Fp, b: &Fp) { + acc.accumulate(a.mul_unreduced(b)); } - #[inline] - fn square_accumulate(acc: &mut Fp, a: &Fp) { - *acc += a.square(); + #[cfg_attr(not(feature = "uninline-portable"), inline)] + fn square_accumulate(acc: &mut Self::Accumulator, a: &Fp) { + acc.accumulate(a.square_unreduced()); } - #[inline] - fn reduce(acc: Fp) -> Fp { - acc + #[cfg_attr(not(feature = "uninline-portable"), inline)] + fn reduce(acc: Self::Accumulator) -> Fp { + /// 2^448 mod p (little-endian limbs). + const B448: [u64; 4] = [ + 0x9b9858f294cf91ba, + 0x8635bd2c4252b065, + 0x496d41af7b9cb714, + 0x1b4b3c4bfffffffc, + ]; + let limbs = acc.partial_reduce(&B448, &R2.0); + Fp::montgomery_reduce( + limbs[0], limbs[1], limbs[2], limbs[3], limbs[4], limbs[5], limbs[6], limbs[7], + ) } } diff --git a/src/fields/fq.rs b/src/fields/fq.rs index 8b0e46f..8993055 100644 --- a/src/fields/fq.rs +++ b/src/fields/fq.rs @@ -13,7 +13,7 @@ use ff::{FieldBits, PrimeFieldBits}; use crate::arithmetic::{adc, mac, sbb, SqrtTableHelpers}; #[cfg(feature = "deferred")] -use crate::deferred::DeferredField; +use crate::deferred::{DeferredField, Product}; #[cfg(feature = "sqrt-table")] use crate::arithmetic::SqrtTables; @@ -481,21 +481,31 @@ impl Fq { #[cfg(feature = "deferred")] impl DeferredField for Fq { - type Accumulator = Fq; + type Accumulator = Product; - #[inline] - fn mul_accumulate(acc: &mut Fq, a: &Fq, b: &Fq) { - *acc += *a * *b; + #[cfg_attr(not(feature = "uninline-portable"), inline)] + fn mul_accumulate(acc: &mut Self::Accumulator, a: &Fq, b: &Fq) { + acc.accumulate(a.mul_unreduced(b)); } - #[inline] - fn square_accumulate(acc: &mut Fq, a: &Fq) { - *acc += a.square(); + #[cfg_attr(not(feature = "uninline-portable"), inline)] + fn square_accumulate(acc: &mut Self::Accumulator, a: &Fq) { + acc.accumulate(a.square_unreduced()); } - #[inline] - fn reduce(acc: Fq) -> Fq { - acc + #[cfg_attr(not(feature = "uninline-portable"), inline)] + fn reduce(acc: Self::Accumulator) -> Fq { + /// 2^448 mod q (little-endian limbs). + const B448: [u64; 4] = [ + 0xcc920bb9994a8dd9, + 0x87a7dcbe1ff6e0d7, + 0x496d41af7ccfdaa9, + 0x0ee4537bfffffffc, + ]; + let limbs = acc.partial_reduce(&B448, &R2.0); + Fq::montgomery_reduce( + limbs[0], limbs[1], limbs[2], limbs[3], limbs[4], limbs[5], limbs[6], limbs[7], + ) } }