From 40bdeca9e9e2db207db9a282d7b0eee8ee572ad8 Mon Sep 17 00:00:00 2001 From: Veljko Vranic Date: Thu, 18 Sep 2025 11:43:31 +0200 Subject: [PATCH 1/5] memory optimisations --- Cargo.toml | 3 +- provekit/common/src/utils/zk_utils.rs | 12 +++---- provekit/prover/src/whir_r1cs.rs | 48 ++++++++++++--------------- provekit/verifier/src/whir_r1cs.rs | 17 +++------- 4 files changed, 34 insertions(+), 46 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ae6eda32..210fb898 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,4 +135,5 @@ ark-serialize = "0.5" ark-std = { version = "0.5", features = ["std"] } spongefish = { git = "https://github.com/arkworks-rs/spongefish", features = ["arkworks-algebra"] } spongefish-pow = { git = "https://github.com/arkworks-rs/spongefish" } -whir = { git = "https://github.com/WizardOfMenlo/whir/", features = ["tracing"], rev = "3e7f8c299783fddf4354869bbcbc995a5018a9d4" } +#whir = { git = "https://github.com/WizardOfMenlo/whir/", features = ["tracing"], rev = "3e7f8c299783fddf4354869bbcbc995a5018a9d4" } +whir = { path = "../latest-whir/whir", features = ["tracing"]} \ No newline at end of file diff --git a/provekit/common/src/utils/zk_utils.rs b/provekit/common/src/utils/zk_utils.rs index 66b1b8a1..cefa7091 100644 --- a/provekit/common/src/utils/zk_utils.rs +++ b/provekit/common/src/utils/zk_utils.rs @@ -1,16 +1,16 @@ use { crate::FieldElement, ark_ff::UniformRand, rayon::prelude::*, - whir::poly_utils::evals::EvaluationsList, + whir::poly_utils::coeffs::CoefficientList }; pub fn create_masked_polynomial( - original: &EvaluationsList, + original: &[FieldElement], mask: &[FieldElement], -) -> EvaluationsList { - let mut combined = Vec::with_capacity(original.num_evals() * 2); - combined.extend_from_slice(original.evals()); +) -> CoefficientList { + let mut combined = Vec::with_capacity(original.len() * 2); + combined.extend_from_slice(original); combined.extend_from_slice(mask); - EvaluationsList::new(combined) + CoefficientList::new(combined) } pub fn generate_random_multilinear_polynomial(num_vars: usize) -> Vec { diff --git a/provekit/prover/src/whir_r1cs.rs b/provekit/prover/src/whir_r1cs.rs index cfe4febb..2c155dc2 100644 --- a/provekit/prover/src/whir_r1cs.rs +++ b/provekit/prover/src/whir_r1cs.rs @@ -1,8 +1,5 @@ use { - anyhow::{ensure, Result}, - ark_ff::UniformRand, - ark_std::{One, Zero}, - provekit_common::{ + anyhow::{ensure, Result}, ark_ff::UniformRand, ark_std::{log2, One, Zero}, provekit_common::{ skyscraper::{SkyscraperMerkleConfig, SkyscraperSponge}, utils::{ pad_to_power_of_two, @@ -15,14 +12,11 @@ use { HALF, }, FieldElement, IOPattern, WhirConfig, WhirR1CSProof, WhirR1CSScheme, R1CS, - }, - spongefish::{ + }, spongefish::{ codecs::arkworks_algebra::{FieldToUnitSerialize, UnitToField}, ProverState, - }, - tracing::{info, instrument, warn}, - whir::{ - poly_utils::{evals::EvaluationsList, multilinear::MultilinearPoint}, + }, std::sync::Arc, tracing::{info, instrument, warn}, whir::{ + poly_utils::{coeffs::CoefficientList, evals::EvaluationsList, multilinear::MultilinearPoint}, whir::{ committer::{CommitmentWriter, Witness}, domainsep::WhirDomainSeparator, @@ -30,7 +24,7 @@ use { statement::{Statement, Weights}, utils::HintSerialize, }, - }, + } }; pub trait WhirR1CSProver { @@ -57,26 +51,28 @@ impl WhirR1CSProver for WhirR1CSScheme { let io: IOPattern = self.create_io_pattern(); let mut merlin = io.to_prover_state(); - let z = pad_to_power_of_two(witness.clone()); - let witness_polynomial_evals = EvaluationsList::new(z.clone()); + let z = pad_to_power_of_two(witness); + // let witness_polynomial_evals = EvaluationsList::new(z); let (commitment_to_witness, masked_polynomial, random_polynomial) = batch_commit_to_polynomial( self.m, &self.whir_witness, - &witness_polynomial_evals, + &z, &mut merlin, ); - // First round of sumcheck to reduce R1CS to a batch weighted evaluation of the // witness + let witness_slice = &z[..r1cs.num_witnesses()]; let (mut merlin, alpha) = run_zk_sumcheck_prover( r1cs, - &witness, + witness_slice, merlin, self.m_0, &self.whir_for_hiding_spartan, ); + drop(z); + // Compute weights from R1CS instance let alphas = calculate_external_row_of_r1cs_matrices(&alpha, r1cs); let (statement, f_sums, g_sums) = create_combined_statement_over_two_polynomials::<3>( @@ -182,30 +178,28 @@ pub fn sum_over_hypercube(g_univariates: &[[FieldElement; 4]]) -> FieldElement { pub fn batch_commit_to_polynomial( m: usize, whir_config: &WhirConfig, - witness: &EvaluationsList, + witness: &[FieldElement], merlin: &mut ProverState, ) -> ( Witness, EvaluationsList, EvaluationsList, ) { - let mask = generate_random_multilinear_polynomial(witness.num_variables()); + let num_vars = log2(witness.len()) as usize; + let mask = generate_random_multilinear_polynomial(num_vars); let masked_polynomial = create_masked_polynomial(witness, &mask); - - let masked_polynomial_coeff = masked_polynomial.to_coeffs(); - - let random_polynomial_eval = EvaluationsList::new(generate_random_multilinear_polynomial(m)); - let random_polynomial_coeff = random_polynomial_eval.to_coeffs(); + drop(mask); + let random_polynomial_coeff = CoefficientList::new(generate_random_multilinear_polynomial(m)); let committer = CommitmentWriter::new(whir_config.clone()); let witness_new = committer .commit_batch(merlin, &[ - masked_polynomial_coeff.clone(), - random_polynomial_coeff.clone(), + &masked_polynomial, + &random_polynomial_coeff, ]) .expect("WHIR prover failed to commit"); - (witness_new, masked_polynomial, random_polynomial_eval) + (witness_new, masked_polynomial.into(), random_polynomial_coeff.into()) } fn generate_blinding_spartan_univariate_polys(m_0: usize) -> Vec<[FieldElement; 4]> { @@ -259,7 +253,7 @@ pub fn run_zk_sumcheck_prover( batch_commit_to_polynomial( blinding_polynomial_variables + 1, whir_for_blinding_of_spartan_config, - &blinding_polynomial_for_commiting, + &blinding_polynomial_for_commiting.evals(), &mut merlin, ); diff --git a/provekit/verifier/src/whir_r1cs.rs b/provekit/verifier/src/whir_r1cs.rs index e56bbb6b..eb694a56 100644 --- a/provekit/verifier/src/whir_r1cs.rs +++ b/provekit/verifier/src/whir_r1cs.rs @@ -1,17 +1,12 @@ use { - anyhow::{ensure, Context, Result}, - ark_std::{One, Zero}, - provekit_common::{ + anyhow::{ensure, Context, Result}, ark_std::{One, Zero}, provekit_common::{ skyscraper::SkyscraperSponge, utils::sumcheck::{calculate_eq, eval_cubic_poly}, FieldElement, WhirConfig, WhirR1CSProof, WhirR1CSScheme, - }, - spongefish::{ + }, spongefish::{ codecs::arkworks_algebra::{FieldToUnitDeserialize, UnitToField}, VerifierState, - }, - tracing::instrument, - whir::{ + }, tracing::instrument, whir::{ poly_utils::{evals::EvaluationsList, multilinear::MultilinearPoint}, whir::{ committer::{reader::ParsedCommitment, CommitmentReader}, @@ -19,7 +14,7 @@ use { utils::HintDeserialize, verifier::Verifier, }, - }, + } }; pub struct DataFromSumcheckVerifier { @@ -34,9 +29,8 @@ pub trait WhirR1CSVerifier { impl WhirR1CSVerifier for WhirR1CSScheme { #[instrument(skip_all)] - #[allow(unused)] // TODO: Fix implementation + #[allow(unused)] fn verify(&self, proof: &WhirR1CSProof) -> Result<()> { - // Set up transcript let io = self.create_io_pattern(); let mut arthur = io.to_verifier_state(&proof.transcript); @@ -47,7 +41,6 @@ impl WhirR1CSVerifier for WhirR1CSScheme { &mut arthur, self.m_0, &self.whir_for_hiding_spartan, - // proof.whir_spartan_blinding_values, ) .context("while verifying sumcheck")?; From c89af1349cc8352df5ca87434d25f349f4793235 Mon Sep 17 00:00:00 2001 From: Veljko Vranic Date: Thu, 18 Sep 2025 11:44:02 +0200 Subject: [PATCH 2/5] memory fix --- provekit/prover/src/whir_r1cs.rs | 5 ++--- provekit/r1cs-compiler/src/whir_r1cs.rs | 4 +++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/provekit/prover/src/whir_r1cs.rs b/provekit/prover/src/whir_r1cs.rs index 2c155dc2..168ea248 100644 --- a/provekit/prover/src/whir_r1cs.rs +++ b/provekit/prover/src/whir_r1cs.rs @@ -15,11 +15,10 @@ use { }, spongefish::{ codecs::arkworks_algebra::{FieldToUnitSerialize, UnitToField}, ProverState, - }, std::sync::Arc, tracing::{info, instrument, warn}, whir::{ + }, tracing::{info, instrument, warn}, whir::{ poly_utils::{coeffs::CoefficientList, evals::EvaluationsList, multilinear::MultilinearPoint}, whir::{ committer::{CommitmentWriter, Witness}, - domainsep::WhirDomainSeparator, prover::Prover, statement::{Statement, Weights}, utils::HintSerialize, @@ -422,7 +421,7 @@ pub fn run_zk_whir_pcs_prover( warn!("More PoW bits required than specified."); } - let prover = Prover(params.clone()); + let prover = Prover::new(params.clone()); let (randomness, deferred) = prover .prove(&mut merlin, statement, witness) .expect("WHIR prover failed to generate a proof"); diff --git a/provekit/r1cs-compiler/src/whir_r1cs.rs b/provekit/r1cs-compiler/src/whir_r1cs.rs index 50003670..99190fac 100644 --- a/provekit/r1cs-compiler/src/whir_r1cs.rs +++ b/provekit/r1cs-compiler/src/whir_r1cs.rs @@ -1,7 +1,7 @@ use { provekit_common::{utils::next_power_of_two, WhirConfig, WhirR1CSScheme, R1CS}, whir::parameters::{ - default_max_pow, FoldingFactor, MultivariateParameters, ProtocolParameters, SoundnessType, + default_max_pow, DeduplicationStrategy, FoldingFactor, MerkleProofStrategy, MultivariateParameters, ProtocolParameters, SoundnessType }, }; @@ -48,6 +48,8 @@ impl WhirR1CSSchemeBuilder for WhirR1CSScheme { _pow_parameters: Default::default(), starting_log_inv_rate: 1, batch_size, + deduplication_strategy: DeduplicationStrategy::Enabled, + merkle_proof_strategy: MerkleProofStrategy::Compressed, }; WhirConfig::new(mv_params, whir_params) } From b843130bcf0723d6f5e2c8e9892a21f694c375eb Mon Sep 17 00:00:00 2001 From: Veljko Vranic Date: Fri, 19 Sep 2025 13:23:11 +0200 Subject: [PATCH 3/5] use proper remote whir --- Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 210fb898..6110a1f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,5 +135,4 @@ ark-serialize = "0.5" ark-std = { version = "0.5", features = ["std"] } spongefish = { git = "https://github.com/arkworks-rs/spongefish", features = ["arkworks-algebra"] } spongefish-pow = { git = "https://github.com/arkworks-rs/spongefish" } -#whir = { git = "https://github.com/WizardOfMenlo/whir/", features = ["tracing"], rev = "3e7f8c299783fddf4354869bbcbc995a5018a9d4" } -whir = { path = "../latest-whir/whir", features = ["tracing"]} \ No newline at end of file +whir = { git = "https://github.com/WizardOfMenlo/whir/", features = ["tracing"], rev = "2b5be1606e261bf00be9d5bf91c8546fb933f3be" } \ No newline at end of file From 7c1894700a22df986f85890f04f630863eb9c84b Mon Sep 17 00:00:00 2001 From: Veljko Vranic Date: Fri, 19 Sep 2025 13:27:35 +0200 Subject: [PATCH 4/5] fmt --- provekit/common/src/utils/zk_utils.rs | 2 +- provekit/prover/src/whir_r1cs.rs | 36 ++++++++++++++----------- provekit/r1cs-compiler/src/whir_r1cs.rs | 3 ++- provekit/verifier/src/whir_r1cs.rs | 22 ++++++++------- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/provekit/common/src/utils/zk_utils.rs b/provekit/common/src/utils/zk_utils.rs index cefa7091..2847e3c1 100644 --- a/provekit/common/src/utils/zk_utils.rs +++ b/provekit/common/src/utils/zk_utils.rs @@ -1,6 +1,6 @@ use { crate::FieldElement, ark_ff::UniformRand, rayon::prelude::*, - whir::poly_utils::coeffs::CoefficientList + whir::poly_utils::coeffs::CoefficientList, }; pub fn create_masked_polynomial( diff --git a/provekit/prover/src/whir_r1cs.rs b/provekit/prover/src/whir_r1cs.rs index 168ea248..aa8c5c3a 100644 --- a/provekit/prover/src/whir_r1cs.rs +++ b/provekit/prover/src/whir_r1cs.rs @@ -1,5 +1,8 @@ use { - anyhow::{ensure, Result}, ark_ff::UniformRand, ark_std::{log2, One, Zero}, provekit_common::{ + anyhow::{ensure, Result}, + ark_ff::UniformRand, + ark_std::{log2, One, Zero}, + provekit_common::{ skyscraper::{SkyscraperMerkleConfig, SkyscraperSponge}, utils::{ pad_to_power_of_two, @@ -12,18 +15,23 @@ use { HALF, }, FieldElement, IOPattern, WhirConfig, WhirR1CSProof, WhirR1CSScheme, R1CS, - }, spongefish::{ + }, + spongefish::{ codecs::arkworks_algebra::{FieldToUnitSerialize, UnitToField}, ProverState, - }, tracing::{info, instrument, warn}, whir::{ - poly_utils::{coeffs::CoefficientList, evals::EvaluationsList, multilinear::MultilinearPoint}, + }, + tracing::{info, instrument, warn}, + whir::{ + poly_utils::{ + coeffs::CoefficientList, evals::EvaluationsList, multilinear::MultilinearPoint, + }, whir::{ committer::{CommitmentWriter, Witness}, prover::Prover, statement::{Statement, Weights}, utils::HintSerialize, }, - } + }, }; pub trait WhirR1CSProver { @@ -54,12 +62,7 @@ impl WhirR1CSProver for WhirR1CSScheme { // let witness_polynomial_evals = EvaluationsList::new(z); let (commitment_to_witness, masked_polynomial, random_polynomial) = - batch_commit_to_polynomial( - self.m, - &self.whir_witness, - &z, - &mut merlin, - ); + batch_commit_to_polynomial(self.m, &self.whir_witness, &z, &mut merlin); // First round of sumcheck to reduce R1CS to a batch weighted evaluation of the // witness let witness_slice = &z[..r1cs.num_witnesses()]; @@ -192,13 +195,14 @@ pub fn batch_commit_to_polynomial( let committer = CommitmentWriter::new(whir_config.clone()); let witness_new = committer - .commit_batch(merlin, &[ - &masked_polynomial, - &random_polynomial_coeff, - ]) + .commit_batch(merlin, &[&masked_polynomial, &random_polynomial_coeff]) .expect("WHIR prover failed to commit"); - (witness_new, masked_polynomial.into(), random_polynomial_coeff.into()) + ( + witness_new, + masked_polynomial.into(), + random_polynomial_coeff.into(), + ) } fn generate_blinding_spartan_univariate_polys(m_0: usize) -> Vec<[FieldElement; 4]> { diff --git a/provekit/r1cs-compiler/src/whir_r1cs.rs b/provekit/r1cs-compiler/src/whir_r1cs.rs index 99190fac..5197482f 100644 --- a/provekit/r1cs-compiler/src/whir_r1cs.rs +++ b/provekit/r1cs-compiler/src/whir_r1cs.rs @@ -1,7 +1,8 @@ use { provekit_common::{utils::next_power_of_two, WhirConfig, WhirR1CSScheme, R1CS}, whir::parameters::{ - default_max_pow, DeduplicationStrategy, FoldingFactor, MerkleProofStrategy, MultivariateParameters, ProtocolParameters, SoundnessType + default_max_pow, DeduplicationStrategy, FoldingFactor, MerkleProofStrategy, + MultivariateParameters, ProtocolParameters, SoundnessType, }, }; diff --git a/provekit/verifier/src/whir_r1cs.rs b/provekit/verifier/src/whir_r1cs.rs index eb694a56..d97f912f 100644 --- a/provekit/verifier/src/whir_r1cs.rs +++ b/provekit/verifier/src/whir_r1cs.rs @@ -1,12 +1,17 @@ use { - anyhow::{ensure, Context, Result}, ark_std::{One, Zero}, provekit_common::{ + anyhow::{ensure, Context, Result}, + ark_std::{One, Zero}, + provekit_common::{ skyscraper::SkyscraperSponge, utils::sumcheck::{calculate_eq, eval_cubic_poly}, FieldElement, WhirConfig, WhirR1CSProof, WhirR1CSScheme, - }, spongefish::{ + }, + spongefish::{ codecs::arkworks_algebra::{FieldToUnitDeserialize, UnitToField}, VerifierState, - }, tracing::instrument, whir::{ + }, + tracing::instrument, + whir::{ poly_utils::{evals::EvaluationsList, multilinear::MultilinearPoint}, whir::{ committer::{reader::ParsedCommitment, CommitmentReader}, @@ -14,7 +19,7 @@ use { utils::HintDeserialize, verifier::Verifier, }, - } + }, }; pub struct DataFromSumcheckVerifier { @@ -37,12 +42,9 @@ impl WhirR1CSVerifier for WhirR1CSScheme { let commitment_reader = CommitmentReader::new(&self.whir_witness); let parsed_commitment = commitment_reader.parse_commitment(&mut arthur).unwrap(); - let data_from_sumcheck_verifier = run_sumcheck_verifier( - &mut arthur, - self.m_0, - &self.whir_for_hiding_spartan, - ) - .context("while verifying sumcheck")?; + let data_from_sumcheck_verifier = + run_sumcheck_verifier(&mut arthur, self.m_0, &self.whir_for_hiding_spartan) + .context("while verifying sumcheck")?; let whir_query_answer_sum_vectors: (Vec, Vec) = arthur.hint().unwrap(); From e8aa123abd5371038235e38ee35fb6e20326ae1e Mon Sep 17 00:00:00 2001 From: Veljko Vranic Date: Fri, 19 Sep 2025 13:44:58 +0200 Subject: [PATCH 5/5] cleanup --- provekit/prover/src/whir_r1cs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provekit/prover/src/whir_r1cs.rs b/provekit/prover/src/whir_r1cs.rs index aa8c5c3a..efe5ba22 100644 --- a/provekit/prover/src/whir_r1cs.rs +++ b/provekit/prover/src/whir_r1cs.rs @@ -59,10 +59,10 @@ impl WhirR1CSProver for WhirR1CSScheme { let mut merlin = io.to_prover_state(); let z = pad_to_power_of_two(witness); - // let witness_polynomial_evals = EvaluationsList::new(z); let (commitment_to_witness, masked_polynomial, random_polynomial) = batch_commit_to_polynomial(self.m, &self.whir_witness, &z, &mut merlin); + // First round of sumcheck to reduce R1CS to a batch weighted evaluation of the // witness let witness_slice = &z[..r1cs.num_witnesses()];