Cache per-level FFT twiddle tables and batch evaluator slice math

crates/halo2_proofs/Cargo.toml

@@ -26,7 +26,7 @@ ff = "0.14"
 group = "0.14"
 halo2_legacy_pdqsort = { workspace = true, optional = true }
 maybe-rayon = { version = "0.1.0", default-features = false }
-pasta_curves = { workspace = true, features = ["default", "deferred", "glv"] }
+pasta_curves = { workspace = true, features = ["default", "deferred", "glv", "ifma"] }
 plotters = { version = "0.3.0", optional = true, default-features = false }
 rand = { version = "0.10", features = ["sys_rng"], optional = true, default-features = false }
 rand_core = { version = "0.10", default-features = false }

crates/halo2_proofs/src/arithmetic.rs

@@ -11,6 +11,95 @@ pub use pasta_curves::arithmetic::*;
 
 use crate::multicore::{self, TheBestReduce};
 
+// Batched slice arithmetic, dispatching to the vectorized Pasta
+// implementations when the field type matches. The unsafe code is contained
+// to slice casts whose element type is verified by `TypeId` equality.
+pub(crate) mod batch {
+    use core::any::TypeId;
+
+    use group::ff::Field;
+    use pasta_curves::{pallas, vesta};
+
+    #[allow(unsafe_code)]
+    fn downcast_mut<F: Field, T: 'static>(s: &mut [F]) -> Option<&mut [T]> {
+        (TypeId::of::<F>() == TypeId::of::<T>()).then(|| {
+            // SAFETY: `TypeId` equality guarantees `F` and `T` are the same
+            // type, so the layouts and element values are identical.
+            unsafe { core::slice::from_raw_parts_mut(s.as_mut_ptr().cast::<T>(), s.len()) }
+        })
+    }
+
+    #[allow(unsafe_code)]
+    fn downcast_ref<F: Field, T: 'static>(s: &[F]) -> Option<&[T]> {
+        (TypeId::of::<F>() == TypeId::of::<T>()).then(|| {
+            // SAFETY: `TypeId` equality guarantees `F` and `T` are the same
+            // type, so the layouts and element values are identical.
+            unsafe { core::slice::from_raw_parts(s.as_ptr().cast::<T>(), s.len()) }
+        })
+    }
+
+    /// Elementwise in-place product: `lhs[i] *= rhs[i]`.
+    pub(crate) fn mul_slice<F: Field>(lhs: &mut [F], rhs: &[F]) {
+        if let Some(l) = downcast_mut::<F, pallas::Base>(lhs) {
+            let r = downcast_ref::<F, pallas::Base>(rhs).expect("same field type");
+            pasta_curves::fp_mul_slice(l, r);
+        } else if let Some(l) = downcast_mut::<F, vesta::Base>(lhs) {
+            let r = downcast_ref::<F, vesta::Base>(rhs).expect("same field type");
+            pasta_curves::fq_mul_slice(l, r);
+        } else {
+            for (l, r) in lhs.iter_mut().zip(rhs.iter()) {
+                *l *= *r;
+            }
+        }
+    }
+
+    /// Elementwise in-place squaring: `x[i] = x[i]^2`.
+    pub(crate) fn sqr_slice<F: Field>(x: &mut [F]) {
+        if let Some(x) = downcast_mut::<F, pallas::Base>(x) {
+            pasta_curves::fp_sqr_slice(x);
+        } else if let Some(x) = downcast_mut::<F, vesta::Base>(x) {
+            pasta_curves::fq_sqr_slice(x);
+        } else {
+            for v in x.iter_mut() {
+                *v = v.square();
+            }
+        }
+    }
+
+    /// Deferred-reduction inner product `sum_i a[i] * b[i]`, when a
+    /// specialized implementation exists for `F`.
+    pub(crate) fn inner_product<F: Field>(a: &[F], b: &[F]) -> Option<F> {
+        if let Some(a) = downcast_ref::<F, pallas::Base>(a) {
+            let b = downcast_ref::<F, pallas::Base>(b).expect("same field type");
+            let out = pasta_curves::fp_inner_product(a, b);
+            downcast_ref::<pallas::Base, F>(core::slice::from_ref(&out)).map(|s| s[0])
+        } else if let Some(a) = downcast_ref::<F, vesta::Base>(a) {
+            let b = downcast_ref::<F, vesta::Base>(b).expect("same field type");
+            let out = pasta_curves::fq_inner_product(a, b);
+            downcast_ref::<vesta::Base, F>(core::slice::from_ref(&out)).map(|s| s[0])
+        } else {
+            None
+        }
+    }
+
+    /// Elementwise in-place scaling by one factor: `x[i] *= k`.
+    pub(crate) fn scale_slice<F: Field>(x: &mut [F], k: &F) {
+        if let Some(x) = downcast_mut::<F, pallas::Base>(x) {
+            let k =
+                downcast_ref::<F, pallas::Base>(core::slice::from_ref(k)).expect("same field type");
+            pasta_curves::fp_scale_slice(x, &k[0]);
+        } else if let Some(x) = downcast_mut::<F, vesta::Base>(x) {
+            let k =
+                downcast_ref::<F, vesta::Base>(core::slice::from_ref(k)).expect("same field type");
+            pasta_curves::fq_scale_slice(x, &k[0]);
+        } else {
+            for v in x.iter_mut() {
+                *v *= *k;
+            }
+        }
+    }
+}
+
 /// This represents an element of a group with basic operations that can be
 /// performed. This allows an FFT implementation (for example) to operate
 /// generically over either a field or elliptic curve group.

crates/halo2_proofs/src/plonk/evaluation.rs

@@ -223,6 +223,9 @@ fn convert_point<F: Field, T: Field>(point: EvaluationPoint<F>) -> EvaluationPoi
 }
 
 fn deferred_inner_product<F: DeferredField>(polynomial: &[F], powers: &[F]) -> F {
+    if let Some(value) = crate::arithmetic::batch::inner_product(polynomial, powers) {
+        return value;
+    }
     F::inner_product(polynomial, powers)
 }