diff --git a/crates/halo2_gadgets/src/ecc/chip/mul.rs b/crates/halo2_gadgets/src/ecc/chip/mul.rs index d9dfaef9..397c3734 100644 --- a/crates/halo2_gadgets/src/ecc/chip/mul.rs +++ b/crates/halo2_gadgets/src/ecc/chip/mul.rs @@ -45,13 +45,14 @@ const INCOMPLETE_LO_LEN: usize = INCOMPLETE_LEN - INCOMPLETE_HI_LEN; const COMPLETE_RANGE: Range = INCOMPLETE_LEN..(INCOMPLETE_LEN + NUM_COMPLETE_BITS); #[derive(Clone, Copy, Debug)] -struct JacobianPoint { +struct ProjectivePoint { x: pallas::Base, y: pallas::Base, z: pallas::Base, + z_sq: pallas::Base, } -impl JacobianPoint { +impl ProjectivePoint { fn from_affine(point: pallas::Affine) -> Self { let coordinates = point .coordinates() @@ -60,67 +61,89 @@ impl JacobianPoint { x: *coordinates.x(), y: *coordinates.y(), z: pallas::Base::ONE, + z_sq: pallas::Base::ONE, } } /// Doubles a nonidentity Pallas point without exceptional-case handling. fn double_unchecked(self) -> Self { // dbl-2009-l for a short-Weierstrass curve with a = 0. - let a = self.x.square(); - let b = self.y.square(); - let c = b.square(); - let d = ((self.x + b).square() - a - c).double(); + let a = square_with_runtime_backend(&self.x); + let b = square_with_runtime_backend(&self.y); + let c = square_with_runtime_backend(&b); + let d = (square_with_runtime_backend(&(self.x + b)) - a - c).double(); let e = a.double() + a; - let f = e.square(); + let f = square_with_runtime_backend(&e); let z = (self.z * self.y).double(); + let z_sq = square_with_runtime_backend(&z); let x = f - d.double(); let y = e * (d - x) - c.double().double().double(); - Self { x, y, z } + Self { x, y, z, z_sq } } - /// Adds a nonidentity affine point that is neither equal to nor the - /// negation of `self`. - fn add_mixed_unchecked(self, x: pallas::Base, y: pallas::Base) -> Self { - let z_squared = self.z.square(); - let u = x * z_squared; - let s = y * z_squared * self.z; - let h = u - self.x; - let hh = h.square(); - let i = hh.double().double(); - let j = h * i; - let r = (s - self.y).double(); - let v = self.x * i; - let result_x = r.square() - j - v.double(); - let result_y = r * (v - result_x) - (self.y * j).double(); - let result_z = (self.z + h).square() - z_squared - hh; + /// Computes witnesses for the incomplete addition `2A + P` while keeping + /// the accumulator in projective coordinates. + fn double_and_add(&mut self, (x_p, y_p): (pallas::Base, pallas::Base)) -> DoubleAndAddWitness { + let z_cubed = self.z_sq * self.z; + let h = x_p * self.z_sq - self.x; + let r = y_p * z_cubed - self.y; + + let h_sq = square_with_runtime_backend(&h); + let h_cubed = h_sq * h; + let x_h_sq = self.x * h_sq; + let x_r = square_with_runtime_backend(&r) - h_cubed - Field::double(&x_h_sq); + let d = x_h_sq - x_r; + + let d_sq = square_with_runtime_backend(&d); + let d_cubed = d_sq * d; + let y_h_cubed = self.y * h_cubed; + // Scale lambda_1 by d so that both slopes use z_new as their + // denominator. + let r_d = r * d; + let lambda_2_numerator = Field::double(&y_h_cubed) - r_d; + + let x_h_sq_d_sq = x_h_sq * d_sq; + let x_new = square_with_runtime_backend(&lambda_2_numerator) - Field::double(&x_h_sq_d_sq) + + d_cubed; + let y_new = lambda_2_numerator * (x_h_sq_d_sq - x_new) - y_h_cubed * d_cubed; + let z_new = self.z * h * d; + let z_new_sq = square_with_runtime_backend(&z_new); + + *self = Self { + x: x_new, + y: y_new, + z: z_new, + z_sq: z_new_sq, + }; - Self { - x: result_x, - y: result_y, - z: result_z, + DoubleAndAddWitness { + lambda_1_numerator: r_d, + lambda_2_numerator, } } } #[derive(Clone, Copy, Debug)] -struct AffinePoint { - x: pallas::Base, - y: pallas::Base, +struct DoubleAndAddWitness { + lambda_1_numerator: pallas::Base, + lambda_2_numerator: pallas::Base, } #[derive(Clone, Copy, Debug)] struct IncompleteRowWitness { z: pallas::Base, - point: AffinePoint, + point_x: pallas::Base, lambda_1: pallas::Base, lambda_2: pallas::Base, } #[derive(Clone, Debug)] struct IncompleteMulWitness { + initial_x: pallas::Base, rows: Vec, - output: AffinePoint, + hi_output_y: pallas::Base, + output_y: pallas::Base, } impl IncompleteMulWitness { @@ -138,92 +161,155 @@ impl IncompleteMulWitness { // and 3 * 2^i - 1. Thus c < 2^252 and 3c < 9 * 2^250 < q, so neither c // nor 3c can be 0 or +/-1 modulo the Pallas group order q. The final // output coefficient is also nonzero because it is less than 2^253. - let mut accumulator = JacobianPoint::from_affine(base).double_unchecked(); - let mut accumulators = Vec::with_capacity(INCOMPLETE_LEN + 1); - accumulators.push(accumulator); - for bit in &bits[..INCOMPLETE_LEN] { - accumulator = accumulator.double_unchecked(); - let addend_y = if *bit { base_y } else { -base_y }; - accumulator = accumulator.add_mixed_unchecked(base_x, addend_y); - accumulators.push(accumulator); - } - - let affine = batch_normalize_nonidentity(&accumulators); - let mut denominators = Vec::with_capacity(INCOMPLETE_LEN * 2); - for points in affine.windows(2) { - let current = points[0]; - let next = points[1]; - denominators.push(current.x - base_x); - denominators.push(current.x - next.x); - } - batch_invert_nonzero(&mut denominators); - + let mut accumulator = ProjectivePoint::from_affine(base).double_unchecked(); + let initial = accumulator; + let mut projective_rows = Vec::with_capacity(INCOMPLETE_LEN); let mut z = pallas::Base::ZERO; - let mut rows = Vec::with_capacity(INCOMPLETE_LEN); - for ((bit, points), inverses) in bits[..INCOMPLETE_LEN] - .iter() - .zip(affine.windows(2)) - .zip(denominators.chunks_exact(2)) - { - let current = points[0]; - let next = points[1]; + for bit in &bits[..INCOMPLETE_LEN] { let addend_y = if *bit { base_y } else { -base_y }; + let witness = accumulator.double_and_add((base_x, addend_y)); z = z.double() + pallas::Base::from(*bit as u64); - rows.push(IncompleteRowWitness { - z, - point: current, - lambda_1: (current.y - addend_y) * inverses[0], - lambda_2: (current.y + next.y) * inverses[1], - }); + projective_rows.push((z, accumulator, witness)); } + // Each round's two slopes have denominator z_new, while its affine + // x-coordinate has denominator z_new^2. Invert every accumulator z + // once, then materialize those three witnesses together. Only the + // boundary between halves and the final row require affine y. + let mut z_inverses = Vec::with_capacity(INCOMPLETE_LEN + 1); + z_inverses.push(initial.z); + z_inverses.extend(projective_rows.iter().map(|(_, point, _)| point.z)); + let mut scratch = vec![pallas::Base::ZERO; z_inverses.len()]; + batch_invert_nonzero(&mut z_inverses, &mut scratch); + + let initial_z_inverse_sq = square_with_runtime_backend(&z_inverses[0]); + let initial_x = initial.x * initial_z_inverse_sq; + let mut hi_output_y = None; + let mut output_y = None; + let rows = projective_rows + .into_iter() + .zip(z_inverses.into_iter().skip(1)) + .enumerate() + .map(|(index, ((z, point, witness), z_inverse))| { + let z_inverse_sq = square_with_runtime_backend(&z_inverse); + let point_x = point.x * z_inverse_sq; + if index + 1 == INCOMPLETE_HI_LEN { + hi_output_y = Some(point.y * z_inverse_sq * z_inverse); + } + if index + 1 == INCOMPLETE_LEN { + output_y = Some(point.y * z_inverse_sq * z_inverse); + } + IncompleteRowWitness { + z, + point_x, + lambda_1: witness.lambda_1_numerator * z_inverse, + lambda_2: witness.lambda_2_numerator * z_inverse, + } + }) + .collect(); + Self { + initial_x, rows, - output: *affine.last().expect("the accumulator list is nonempty"), + hi_output_y: hi_output_y.expect("the high incomplete half is nonempty"), + output_y: output_y.expect("the incomplete multiplication is nonempty"), } } - fn point(&self, index: usize) -> AffinePoint { - if index == self.rows.len() { - self.output + fn point_x(&self, index: usize) -> pallas::Base { + if index == 0 { + self.initial_x } else { - self.rows[index].point + self.rows[index - 1].point_x } } -} -fn batch_normalize_nonidentity(points: &[JacobianPoint]) -> Vec { - let mut z_inverses = points.iter().map(|point| point.z).collect::>(); - batch_invert_nonzero(&mut z_inverses); - points - .iter() - .zip(z_inverses) - .map(|(point, z_inverse)| { - let z_squared = z_inverse.square(); - AffinePoint { - x: point.x * z_squared, - y: point.y * z_squared * z_inverse, - } - }) - .collect() + fn point_y(&self, index: usize) -> pallas::Base { + if index == INCOMPLETE_HI_LEN { + self.hi_output_y + } else { + assert_eq!(index, INCOMPLETE_LEN); + self.output_y + } + } } -fn batch_invert_nonzero(values: &mut [pallas::Base]) { - let mut scratch = Vec::with_capacity(values.len()); - let mut accumulator = pallas::Base::ONE; - for value in values.iter() { - scratch.push(accumulator); - accumulator *= value; +fn batch_invert_nonzero(values: &mut [pallas::Base], scratch: &mut [pallas::Base]) { + assert_eq!(values.len(), scratch.len()); + let Some((first, values)) = values.split_first_mut() else { + return; + }; + + debug_assert!(!first.is_zero_vartime()); + if values.is_empty() { + *first = first.invert().unwrap(); + return; } - accumulator = accumulator + let (second, values) = values.split_first_mut().unwrap(); + debug_assert!(!second.is_zero_vartime()); + let scratch = &mut scratch[2..]; + + // Keep even and odd prefix products in independent dependency chains. + // Joining them before the one field inversion adds three multiplications + // per call, but exposes multiplication throughput throughout both walks. + // A trailing element has an even index and belongs to the first chain. + let mut acc_0 = *first; + let mut acc_1 = *second; + for (pair, slots) in values.chunks_exact(2).zip(scratch.chunks_exact_mut(2)) { + debug_assert!(!pair[0].is_zero_vartime()); + debug_assert!(!pair[1].is_zero_vartime()); + slots[0] = acc_0; + acc_0 *= pair[0]; + slots[1] = acc_1; + acc_1 *= pair[1]; + } + if let (Some(value), Some(slot)) = ( + values.chunks_exact(2).remainder().first(), + scratch.chunks_exact_mut(2).into_remainder().first_mut(), + ) { + debug_assert!(!value.is_zero_vartime()); + *slot = acc_0; + acc_0 *= value; + } + + let inverse = (acc_0 * acc_1) .invert() .expect("incomplete multiplication denominators are nonzero"); - for (value, prefix) in values.iter_mut().zip(scratch).rev() { - let original = *value; - *value = accumulator * prefix; - accumulator *= original; + let seed_0 = inverse * acc_1; + let seed_1 = inverse * acc_0; + let mut acc_0 = seed_0; + let mut acc_1 = seed_1; + + if let (Some(value), Some(slot)) = ( + values.chunks_exact_mut(2).into_remainder().first_mut(), + scratch.chunks_exact(2).remainder().first(), + ) { + let inverted = acc_0 * slot; + acc_0 *= *value; + *value = inverted; } + for (pair, slots) in values + .chunks_exact_mut(2) + .zip(scratch.chunks_exact(2)) + .rev() + { + let inverted_0 = acc_0 * slots[0]; + let inverted_1 = acc_1 * slots[1]; + acc_0 *= pair[0]; + acc_1 *= pair[1]; + pair[0] = inverted_0; + pair[1] = inverted_1; + } + *first = acc_0; + *second = acc_1; +} + +#[inline(always)] +fn square_with_runtime_backend(value: &pallas::Base) -> pallas::Base { + // Method syntax selects `pallas::Base`'s portable inherent square. + // Trait dispatch selects the configured runtime backend instead. + Field::square(value) } #[derive(Copy, Clone, Debug, Eq, PartialEq)] @@ -662,6 +748,29 @@ pub mod tests { utilities::{UtilitiesInstructions, lookup_range_check::PallasLookupRangeCheck}, }; + /// Exercises empty, singleton, even, and odd inversion batches. + const MAX_BATCH_INVERSION_TEST_LEN: usize = 64; + + #[test] + fn batch_invert_nonzero_matches_individual_inversion() { + for len in 0..=MAX_BATCH_INVERSION_TEST_LEN { + let mut values = (1..=len) + .map(|value| { + pallas::Base::from(u64::try_from(value).expect("test length fits into u64")) + }) + .collect::>(); + let expected = values + .iter() + .map(|value| value.invert().unwrap()) + .collect::>(); + let mut scratch = vec![pallas::Base::ZERO; len]; + + super::batch_invert_nonzero(&mut values, &mut scratch); + + assert_eq!(values, expected); + } + } + #[test] fn incomplete_witness_matches_group_arithmetic_and_gate_equations() { let mut rng = rng(); @@ -684,25 +793,31 @@ pub mod tests { for (index, bit) in bits[..super::INCOMPLETE_LEN].iter().enumerate() { let row = witness.rows[index]; + let point_x = witness.point_x(index); + let lambda_1 = row.lambda_1; + let lambda_2 = row.lambda_2; let current = accumulator.to_affine(); let current_coordinates = current.coordinates().unwrap(); - assert_eq!(row.point.x, *current_coordinates.x()); - assert_eq!(row.point.y, *current_coordinates.y()); + assert_eq!(point_x, *current_coordinates.x()); + let point_y = *current_coordinates.y(); let addend_y = if *bit { base_y } else { -base_y }; - let next = witness.point(index + 1); - let x_r = row.lambda_1.square() - row.point.x - base_x; - let reconstructed_y = mul_fp_by_inverse_power_of_two( - &((row.lambda_1 + row.lambda_2) * (row.point.x - x_r)), - 1, - ); - assert_eq!(reconstructed_y, row.point.y); - assert_eq!( - row.lambda_1 * (row.point.x - base_x), - row.point.y - addend_y, - ); - assert_eq!(row.lambda_2.square() - x_r - row.point.x, next.x,); - assert_eq!(row.lambda_2 * (row.point.x - next.x), row.point.y + next.y,); + let next_x = witness.point_x(index + 1); + let next = (accumulator.double() + + if *bit { + base.to_curve() + } else { + -base.to_curve() + }) + .to_affine(); + let next_y = *next.coordinates().unwrap().y(); + let x_r = lambda_1.square() - point_x - base_x; + let reconstructed_y = + mul_fp_by_inverse_power_of_two(&((lambda_1 + lambda_2) * (point_x - x_r)), 1); + assert_eq!(reconstructed_y, point_y); + assert_eq!(lambda_1 * (point_x - base_x), point_y - addend_y,); + assert_eq!(lambda_2.square() - x_r - point_x, next_x); + assert_eq!(lambda_2 * (point_x - next_x), point_y + next_y); accumulator = accumulator.double() + if *bit { @@ -714,8 +829,14 @@ pub mod tests { let output = accumulator.to_affine(); let output_coordinates = output.coordinates().unwrap(); - assert_eq!(witness.output.x, *output_coordinates.x()); - assert_eq!(witness.output.y, *output_coordinates.y()); + assert_eq!( + witness.point_x(super::INCOMPLETE_LEN), + *output_coordinates.x() + ); + assert_eq!( + witness.point_y(super::INCOMPLETE_LEN), + *output_coordinates.y() + ); } } diff --git a/crates/halo2_gadgets/src/ecc/chip/mul/incomplete.rs b/crates/halo2_gadgets/src/ecc/chip/mul/incomplete.rs index da0b5dab..1693bca0 100644 --- a/crates/halo2_gadgets/src/ecc/chip/mul/incomplete.rs +++ b/crates/halo2_gadgets/src/ecc/chip/mul/incomplete.rs @@ -367,7 +367,7 @@ impl Config { bits.len() - 1, |row| { witness - .map(|witness| witness.point(witness_range.start + row + 1).x) + .map(|witness| witness.point_x(witness_range.start + row + 1)) .map(Assigned::Trivial) }, )?; @@ -377,7 +377,7 @@ impl Config { offset + NUM_BITS, || { witness - .map(|witness| witness.point(witness_range.end).x) + .map(|witness| witness.point_x(witness_range.end)) .map(Assigned::Trivial) }, )?; @@ -389,7 +389,7 @@ impl Config { offset + NUM_BITS, || { witness - .map(|witness| witness.point(witness_range.end).y) + .map(|witness| witness.point_y(witness_range.end)) .map(Assigned::Trivial) }, )?; diff --git a/docs/changelog/unreleased/411.md b/docs/changelog/unreleased/411.md new file mode 100644 index 00000000..9aa81f66 --- /dev/null +++ b/docs/changelog/unreleased/411.md @@ -0,0 +1,11 @@ +## zakura-halo2-gadgets + +### Changed + +- Reduced variable-base multiplication witness-generation latency by retaining + projective accumulators across fused doubling and addition steps, and by + sharing one batch inversion across slope and affine-coordinate witnesses. In + two-Action post-NU6.3 Orchard proofs with six workers, the combined witness + and rational-advice phase improved by 2.58% on Apple M4 and 1.29% on x86-64 + Linux. Full-proof point estimates improved by 0.46% and 0.29%, respectively + ([#411](https://github.com/zakura-core/common/pull/411)).