feat: log broken invariant as soon as it is found#13435
Open
CreeptoGengar wants to merge 2 commits intofoundry-rs:masterfrom
Open
feat: log broken invariant as soon as it is found#13435CreeptoGengar wants to merge 2 commits intofoundry-rs:masterfrom
CreeptoGengar wants to merge 2 commits intofoundry-rs:masterfrom
Conversation
… InvariantTest, InvariantTestRun, call_after_invariant_function, call_invariant_function, error::FailedInvariantCaseData, }; use crate::executors::{Executor, RawCallResult}; use alloy_dyn_abi::JsonAbiExt; use alloy_primitives::I256; use eyre::Result; use foundry_common::sh_println; use foundry_config::InvariantConfig; use foundry_evm_core::utils::StateChangeset; use foundry_evm_coverage::HitMaps; use foundry_evm_fuzz::{ BasicTxDetails, FuzzedCases, invariant::{FuzzRunIdentifiedContracts, InvariantContract}, }; use revm_inspectors::tracing::CallTraceArena; use std::{borrow::Cow, collections::HashMap, time::{SystemTime, UNIX_EPOCH}}; /// The outcome of an invariant fuzz test #[derive(Debug)] pub struct InvariantFuzzTestResult { pub error: Option<InvariantFuzzError>, /// Every successful fuzz test case pub cases: Vec<FuzzedCases>, /// Number of reverted fuzz calls pub reverts: usize, /// The entire inputs of the last run of the invariant campaign, used for /// replaying the run for collecting traces. pub last_run_inputs: Vec<BasicTxDetails>, /// Additional traces used for gas report construction. pub gas_report_traces: Vec<Vec<CallTraceArena>>, /// The coverage info collected during the invariant test runs. pub line_coverage: Option<HitMaps>, /// Fuzzed selectors metrics collected during the invariant test runs. pub metrics: HashMap<String, InvariantMetrics>, /// Number of failed replays from persisted corpus. pub failed_corpus_replays: usize, /// For optimization mode (int256 return): the best (maximum) value achieved. /// None means standard invariant check mode. pub optimization_best_value: Option<I256>, /// For optimization mode: the call sequence that produced the best value. pub optimization_best_sequence: Vec<BasicTxDetails>, } /// Enriched results of an invariant run check. /// /// Contains the success condition and call results of the last run pub(crate) struct RichInvariantResults { pub(crate) can_continue: bool, pub(crate) call_result: Option<RawCallResult>, } impl RichInvariantResults { pub(crate) fn new(can_continue: bool, call_result: Option<RawCallResult>) -> Self { Self { can_continue, call_result } } } /// Given the executor state, asserts that no invariant has been broken. Otherwise, it fills the /// external `invariant_failures.failed_invariant` map and returns a generic error. /// Either returns the call result if successful, or nothing if there was an error. pub(crate) fn assert_invariants( invariant_contract: &InvariantContract<'_>, invariant_config: &InvariantConfig, targeted_contracts: &FuzzRunIdentifiedContracts, executor: &Executor, calldata: &[BasicTxDetails], invariant_failures: &mut InvariantFailures, ) -> Result<Option<RawCallResult>> { let mut inner_sequence = vec![]; if let Some(fuzzer) = &executor.inspector().fuzzer && let Some(call_generator) = &fuzzer.call_generator { inner_sequence.extend(call_generator.last_sequence.read().iter().cloned()); } let (call_result, success) = call_invariant_function( executor, invariant_contract.address, invariant_contract.invariant_function.abi_encode_input(&[])?.into(), )?; if !success { // We only care about invariants which we haven't broken yet. if invariant_failures.error.is_none() { let case_data = FailedInvariantCaseData::new( invariant_contract, invariant_config, targeted_contracts, calldata, call_result, &inner_sequence, ); // Log broken invariant as soon as it is found (for benchmarking purposes) let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_secs_f64()) .unwrap_or(0.0); let invariant_name = &invariant_contract.invariant_function.name; let revert_reason = if case_data.revert_reason.is_empty() { "unknown reason".to_string() } else { case_data.revert_reason.clone() }; let _ = sh_println!( "[INVARIANT BROKEN] timestamp={:.6} invariant={} reason={} sequence_length={}", timestamp, invariant_name, revert_reason, calldata.len() ); invariant_failures.error = Some(InvariantFuzzError::BrokenInvariant(case_data)); return Ok(None); } } Ok(Some(call_result)) } /// Returns if invariant test can continue and last successful call result of the invariant test /// function (if it can continue). /// /// For optimization mode (int256 return), tracks the max value but never fails on invariant. /// For check mode, asserts the invariant and fails if broken. pub(crate) fn can_continue( invariant_contract: &InvariantContract<'_>, invariant_test: &mut InvariantTest, invariant_run: &mut InvariantTestRun, invariant_config: &InvariantConfig, call_result: RawCallResult, state_changeset: &StateChangeset, ) -> Result<RichInvariantResults> { let mut call_results = None; let is_optimization = invariant_contract.is_optimization(); let handlers_succeeded = || { invariant_test.targeted_contracts.targets.lock().keys().all(|address| { invariant_run.executor.is_success( *address, false, Cow::Borrowed(state_changeset), false, ) }) }; if !call_result.reverted && handlers_succeeded() { if let Some(traces) = call_result.traces { invariant_run.run_traces.push(traces); } if is_optimization { // Optimization mode: call invariant and track max value, never fail. let (inv_result, success) = call_invariant_function( &invariant_run.executor, invariant_contract.address, invariant_contract.invariant_function.abi_encode_input(&[])?.into(), )?; if success && inv_result.result.len() >= 32 && let Some(value) = I256::try_from_be_slice(&inv_result.result[..32]) { invariant_test.update_optimization_value(value, &invariant_run.inputs); } call_results = Some(inv_result); } else { // Check mode: assert invariants and fail if broken. call_results = assert_invariants( invariant_contract, invariant_config, &invariant_test.targeted_contracts, &invariant_run.executor, &invariant_run.inputs, &mut invariant_test.test_data.failures, )?; if call_results.is_none() { return Ok(RichInvariantResults::new(false, None)); } } } else { // Increase the amount of reverts. let invariant_data = &mut invariant_test.test_data; invariant_data.failures.reverts += 1; // If fail on revert is set, we must return immediately. if invariant_config.fail_on_revert { let case_data = FailedInvariantCaseData::new( invariant_contract, invariant_config, &invariant_test.targeted_contracts, &invariant_run.inputs, call_result, &[], ); // Log broken invariant (revert) as soon as it is found (for benchmarking purposes) let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_secs_f64()) .unwrap_or(0.0); let invariant_name = &invariant_contract.invariant_function.name; let revert_reason = if case_data.revert_reason.is_empty() { "revert".to_string() } else { case_data.revert_reason.clone() }; let _ = sh_println!( "[INVARIANT BROKEN] timestamp={:.6} invariant={} reason={} sequence_length={}", timestamp, invariant_name, revert_reason, invariant_run.inputs.len() ); invariant_data.failures.revert_reason = Some(case_data.revert_reason.clone()); invariant_data.failures.error = Some(InvariantFuzzError::Revert(case_data)); return Ok(RichInvariantResults::new(false, None)); } else if call_result.reverted && !is_optimization { // If we don't fail test on revert then remove last reverted call from inputs. // In optimization mode, we keep reverted calls to preserve warp/roll values // for correct replay during shrinking. invariant_run.inputs.pop(); } } Ok(RichInvariantResults::new(true, call_results)) } /// Given the executor state, asserts conditions within `afterInvariant` function. /// If call fails then the invariant test is considered failed. pub(crate) fn assert_after_invariant( invariant_contract: &InvariantContract<'_>, invariant_test: &mut InvariantTest, invariant_run: &InvariantTestRun, invariant_config: &InvariantConfig, ) -> Result<bool> { let (call_result, success) = call_after_invariant_function(&invariant_run.executor, invariant_contract.address)?; // Fail the test case if `afterInvariant` doesn't succeed. if !success { let case_data = FailedInvariantCaseData::new( invariant_contract, invariant_config, &invariant_test.targeted_contracts, &invariant_run.inputs, call_result, &[], ); // Log broken invariant (afterInvariant failure) as soon as it is found (for benchmarking purposes) let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_secs_f64()) .unwrap_or(0.0); let invariant_name = &invariant_contract.invariant_function.name; let revert_reason = if case_data.revert_reason.is_empty() { "afterInvariant failure".to_string() } else { case_data.revert_reason.clone() }; let _ = sh_println!( "[INVARIANT BROKEN] timestamp={:.6} invariant={} reason={} sequence_length={}", timestamp, invariant_name, revert_reason, invariant_run.inputs.len() ); invariant_test.set_error(InvariantFuzzError::BrokenInvariant(case_data)); } Ok(success) }
Add logging for broken invariants with timestamps and reasons
CreeptoGengar
requested review from
0xrusowsky,
DaniPopes,
grandizzy,
mattsse,
onbjerg and
zerosnacks
as code owners
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
1 participant
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Adds real-time logging when invariants break during fuzzing with timestamp, invariant name, reason, and sequence length for benchmarking purposes.
Closes #13285