From 2fa1926ddc35b1ca37c386a4cafc5b34f539ba46 Mon Sep 17 00:00:00 2001 From: Xinhao Yuan Date: Thu, 22 Jan 2026 14:51:11 -0800 Subject: [PATCH] Retry mutation failure once per batch if in persistent mode. This is to avoid breaking the workflow due to previous asynchronous failures. PiperOrigin-RevId: 859784885 --- centipede/BUILD | 1 + centipede/centipede_default_callbacks.cc | 19 +++++++-- centipede/centipede_test.cc | 25 +++++++++++ centipede/testing/BUILD | 21 +++++++++ centipede/testing/async_failing_target.cc | 52 +++++++++++++++++++++++ 5 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 centipede/testing/async_failing_target.cc diff --git a/centipede/BUILD b/centipede/BUILD index 371abdacb..db3134503 100644 --- a/centipede/BUILD +++ b/centipede/BUILD @@ -1896,6 +1896,7 @@ cc_test( srcs = ["centipede_test.cc"], data = [ "@com_google_fuzztest//centipede/testing:abort_fuzz_target", + "@com_google_fuzztest//centipede/testing:async_failing_target", "@com_google_fuzztest//centipede/testing:expensive_startup_fuzz_target", "@com_google_fuzztest//centipede/testing:fuzz_target_with_config", "@com_google_fuzztest//centipede/testing:fuzz_target_with_custom_mutator", diff --git a/centipede/centipede_default_callbacks.cc b/centipede/centipede_default_callbacks.cc index bd030bc2f..d6ed24fb6 100644 --- a/centipede/centipede_default_callbacks.cc +++ b/centipede/centipede_default_callbacks.cc @@ -75,8 +75,12 @@ CentipedeDefaultCallbacks::GetSerializedTargetConfig() { std::vector CentipedeDefaultCallbacks::Mutate( const std::vector &inputs, size_t num_mutants) { if (num_mutants == 0) return {}; - // Try to use the custom mutator if it hasn't been disabled. - if (custom_mutator_is_usable_.value_or(true)) { + // In persistent mode, mutation could fail due to previous asynchronous + // failure, thus give it one more chance to mutate in a clean state. + for (int num_attempts = env_.persistent_mode ? 2 : 1; num_attempts > 0; + --num_attempts) { + // Do not use the custom mutator if it has been disabled. + if (!custom_mutator_is_usable_.value_or(true)) break; MutationResult result = MutateViaExternalBinary(env_.binary, inputs, num_mutants); if (result.exit_code() == EXIT_SUCCESS) { @@ -100,6 +104,7 @@ std::vector CentipedeDefaultCallbacks::Mutate( << "Custom mutator returned no mutants; will " "generate some using the built-in mutator."; } + break; } else if (ShouldStop()) { FUZZTEST_LOG(WARNING) << "Custom mutator failed, but ignored since the stop " @@ -107,10 +112,16 @@ std::vector CentipedeDefaultCallbacks::Mutate( "condition also interrupted the mutator."; // Returning whatever mutants we got before the failure. return std::move(result).mutants(); + } else if (num_attempts > 1) { + // Failed to mutate but still has more attempts + CleanUpPersistentMode(); + FUZZTEST_LOG(ERROR) + << "Test binary failed to mutate inputs - clean up and try again."; } else { + // Still failing at the final attempt. PrintExecutionLog(); - FUZZTEST_LOG(ERROR) - << "Test binary failed when asked to mutate inputs - exiting."; + FUZZTEST_LOG(ERROR) << "Test binary failed to mutate inputs at the final " + "attempt - exiting."; RequestEarlyStop(EXIT_FAILURE); return {}; } diff --git a/centipede/centipede_test.cc b/centipede/centipede_test.cc index dac430249..fe7224415 100644 --- a/centipede/centipede_test.cc +++ b/centipede/centipede_test.cc @@ -1234,5 +1234,30 @@ TEST_F(CentipedeWithTemporaryLocalDir, ExecuteEndsAfterCustomFailure) { Not(HasSubstr("custom failure 2")))); } +TEST_F(CentipedeWithTemporaryLocalDir, TolerateAsyncFailureInMutation) { + Environment env; + env.binary = + GetDataDependencyFilepath("centipede/testing/async_failing_target"); + CentipedeDefaultCallbacks callbacks(env); + BatchResult result; + std::vector inputs = { + {'s', 'o', 'm', 'e'}, + }; + ClearEarlyStopRequestAndSetStopTime(absl::InfiniteFuture()); + EXPECT_TRUE(callbacks.Execute(env.binary, inputs, result)); + // Match the error log to check for retrying mutation. + EXPECT_DEATH( + [&] { + callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), + inputs.size()); + FUZZTEST_LOG(INFO) << "Mutate() succeeded"; + std::_Exit(EXIT_FAILURE); + }(), + AllOf( + HasSubstr( + "Test binary failed to mutate inputs - clean up and try again."), + HasSubstr("Mutate() succeeded"))); +} + } // namespace } // namespace fuzztest::internal diff --git a/centipede/testing/BUILD b/centipede/testing/BUILD index 12a8e2c04..e125aa589 100644 --- a/centipede/testing/BUILD +++ b/centipede/testing/BUILD @@ -154,6 +154,27 @@ centipede_fuzz_target( fuzz_target = "_external_target_server", ) +# Binary for :async_failing_target. +cc_binary( + name = "_async_failing_target", + srcs = ["async_failing_target.cc"], + # Cannot be built directly - build :async_failing_target instead. + tags = [ + "local", + "manual", + "notap", + ], + deps = [ + "@com_google_fuzztest//centipede:centipede_runner_no_main", + "@com_google_fuzztest//common:defs", + ], +) + +centipede_fuzz_target( + name = "async_failing_target", + fuzz_target = "_async_failing_target", +) + cc_binary( name = "_external_target", srcs = ["external_target.cc"], diff --git a/centipede/testing/async_failing_target.cc b/centipede/testing/async_failing_target.cc new file mode 100644 index 000000000..e0cc362a8 --- /dev/null +++ b/centipede/testing/async_failing_target.cc @@ -0,0 +1,52 @@ +// Copyright 2026 The Centipede Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "./centipede/runner_interface.h" +#include "./common/defs.h" + +namespace { + +class AsyncFailingTargetRunnerCallbacks + : public fuzztest::internal::RunnerCallbacks { + public: + bool Execute(fuzztest::internal::ByteSpan input) override { + to_fail_in_mutation = true; + return true; + } + + bool Mutate(const std::vector& inputs, + size_t num_mutants, + std::function + new_mutant_callback) override { + if (to_fail_in_mutation) { + fprintf(stderr, "Fail in mutation\n"); + std::abort(); + } + return true; + } + + bool HasCustomMutator() const override { return true; } + + private: + bool to_fail_in_mutation = false; +}; + +} // namespace + +int main(int argc, char** absl_nonnull argv) { + AsyncFailingTargetRunnerCallbacks runner_callbacks; + return fuzztest::internal::RunnerMain(argc, argv, runner_callbacks); +}