Define targets in codspeed.yaml to execute with codspeed run

.gitignore

@@ -1,2 +1,3 @@
 /target
 .DS_Store
+.codspeed

crates/exec-harness/src/analysis.rs

@@ -1,24 +1,29 @@
 use crate::prelude::*;
 
-use crate::uri::NameAndUri;
+use crate::BenchmarkCommand;
+use crate::uri;
 use codspeed::instrument_hooks::InstrumentHooks;
 use std::process::Command;
 
-pub fn perform(name_and_uri: NameAndUri, command: Vec<String>) -> Result<()> {
+pub fn perform(commands: Vec<BenchmarkCommand>) -> Result<()> {
     let hooks = InstrumentHooks::instance();
 
-    let mut cmd = Command::new(&command[0]);
-    cmd.args(&command[1..]);
-    hooks.start_benchmark().unwrap();
-    let status = cmd.status();
-    hooks.stop_benchmark().unwrap();
-    let status = status.context("Failed to execute command")?;
+    for benchmark_cmd in commands {
+        let name_and_uri = uri::generate_name_and_uri(&benchmark_cmd.name, &benchmark_cmd.command);
 
-    if !status.success() {
-        bail!("Command exited with non-zero status: {status}");
-    }
+        let mut cmd = Command::new(&benchmark_cmd.command[0]);
+        cmd.args(&benchmark_cmd.command[1..]);
+        hooks.start_benchmark().unwrap();
+        let status = cmd.status();
+        hooks.stop_benchmark().unwrap();
+        let status = status.context("Failed to execute command")?;
+
+        if !status.success() {
+            bail!("Command exited with non-zero status: {status}");
+        }
 
-    hooks.set_executed_benchmark(&name_and_uri.uri).unwrap();
+        hooks.set_executed_benchmark(&name_and_uri.uri).unwrap();
+    }
 
     Ok(())
 }

crates/exec-harness/src/lib.rs

@@ -1,15 +1,83 @@
 use clap::ValueEnum;
+use prelude::*;
 use serde::{Deserialize, Serialize};
+use std::io::{self, BufRead};
 
 pub mod analysis;
 pub mod prelude;
-pub mod uri;
+mod uri;
 pub mod walltime;
 
-#[derive(ValueEnum, Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[derive(ValueEnum, Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
 #[serde(rename_all = "lowercase")]
 pub enum MeasurementMode {
     Walltime,
     Memory,
     Simulation,
 }
+
+/// A single benchmark command for stdin mode input.
+///
+/// This struct defines the JSON format for passing benchmark commands to exec-harness
+/// via stdin (when invoked with `-`). The runner uses this same struct to serialize
+/// targets from codspeed.yaml.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BenchmarkCommand {
+    /// The command and arguments to execute
+    pub command: Vec<String>,
+
+    /// Optional benchmark name
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub name: Option<String>,
+
+    /// Walltime execution options (flattened into the JSON object)
+    #[serde(default)]
+    pub walltime_args: walltime::WalltimeExecutionArgs,
+}
+
+/// Read and parse benchmark commands from stdin as JSON
+pub fn read_commands_from_stdin() -> Result<Vec<BenchmarkCommand>> {
+    let stdin = io::stdin();
+    let mut input = String::new();
+
+    for line in stdin.lock().lines() {
+        let line = line.context("Failed to read line from stdin")?;
+        input.push_str(&line);
+        input.push('\n');
+    }
+
+    let commands: Vec<BenchmarkCommand> =
+        serde_json::from_str(&input).context("Failed to parse JSON from stdin")?;
+
+    if commands.is_empty() {
+        bail!("No commands provided in stdin input");
+    }
+
+    for cmd in &commands {
+        if cmd.command.is_empty() {
+            bail!("Empty command in stdin input");
+        }
+    }
+
+    Ok(commands)
+}
+
+/// Execute benchmark commands
+pub fn execute_benchmarks(
+    commands: Vec<BenchmarkCommand>,
+    measurement_mode: Option<MeasurementMode>,
+) -> Result<()> {
+    match measurement_mode {
+        Some(MeasurementMode::Walltime) | None => {
+            walltime::perform(commands)?;
+        }
+        Some(MeasurementMode::Memory) => {
+            analysis::perform(commands)?;
+        }
+        Some(MeasurementMode::Simulation) => {
+            bail!("Simulation measurement mode is not yet supported by exec-harness");
+        }
+    }
+
+    Ok(())
+}

crates/exec-harness/src/main.rs

@@ -1,9 +1,9 @@
 use clap::Parser;
-use exec_harness::MeasurementMode;
-use exec_harness::analysis;
 use exec_harness::prelude::*;
-use exec_harness::uri;
-use exec_harness::walltime;
+use exec_harness::walltime::WalltimeExecutionArgs;
+use exec_harness::{
+    BenchmarkCommand, MeasurementMode, execute_benchmarks, read_commands_from_stdin,
+};
 
 #[derive(Parser, Debug)]
 #[command(name = "exec-harness")]
@@ -21,9 +21,10 @@ struct Args {
     measurement_mode: Option<MeasurementMode>,
 
     #[command(flatten)]
-    execution_args: walltime::WalltimeExecutionArgs,
+    walltime_args: WalltimeExecutionArgs,
 
-    /// The command and arguments to execute
+    /// The command and arguments to execute.
+    /// Use "-" as the only argument to read a JSON payload from stdin.
     #[arg(trailing_var_arg = true, allow_hyphen_values = true)]
     command: Vec<String>,
 }
@@ -37,26 +38,20 @@ fn main() -> Result<()> {
     debug!("Starting exec-harness with pid {}", std::process::id());
 
     let args = Args::parse();
-
-    if args.command.is_empty() {
-        bail!("Error: No command provided");
-    }
-
-    let bench_name_and_uri = uri::generate_name_and_uri(&args.name, &args.command);
-
-    match args.measurement_mode {
-        Some(MeasurementMode::Walltime) | None => {
-            let execution_options: walltime::ExecutionOptions = args.execution_args.try_into()?;
-
-            walltime::perform(bench_name_and_uri, args.command, &execution_options)?;
-        }
-        Some(MeasurementMode::Memory) => {
-            analysis::perform(bench_name_and_uri, args.command)?;
-        }
-        Some(MeasurementMode::Simulation) => {
-            bail!("Simulation measurement mode is not yet supported by exec-harness");
-        }
-    }
+    let measurement_mode = args.measurement_mode;
+
+    // Determine if we're in stdin mode or CLI mode
+    let commands = match args.command.as_slice() {
+        [single] if single == "-" => read_commands_from_stdin()?,
+        [] => bail!("No command provided"),
+        _ => vec![BenchmarkCommand {
+            command: args.command,
+            name: args.name,
+            walltime_args: args.walltime_args,
+        }],
+    };
+
+    execute_benchmarks(commands, measurement_mode)?;
 
     Ok(())
 }

crates/exec-harness/src/walltime/config.rs

@@ -1,4 +1,5 @@
 use crate::prelude::*;
+use serde::{Deserialize, Serialize};
 use std::time::Duration;
 
 const DEFAULT_WARMUP_TIME_NS: u64 = 1_000_000_000; // 1 second
@@ -27,7 +28,7 @@ fn parse_duration_to_ns(s: &str) -> Result<u64> {
 ///
 /// ⚠️ Make sure to update WalltimeExecutionArgs::to_cli_args() when fields change, else the runner
 /// will not properly forward arguments
-#[derive(Debug, Clone, Default, clap::Args)]
+#[derive(Debug, Clone, Default, clap::Args, Serialize, Deserialize)]
 pub struct WalltimeExecutionArgs {
     /// Duration of the warmup phase before measurement starts.
     /// During warmup, the benchmark runs to stabilize performance (e.g., JIT compilation, cache warming).

crates/exec-harness/src/walltime/mod.rs

@@ -6,35 +6,42 @@ pub use config::WalltimeExecutionArgs;
 use runner_shared::walltime_results::WalltimeBenchmark;
 pub use runner_shared::walltime_results::WalltimeResults;
 
+use crate::BenchmarkCommand;
 use crate::prelude::*;
 use crate::uri::NameAndUri;
+use crate::uri::generate_name_and_uri;
 use codspeed::instrument_hooks::InstrumentHooks;
 use std::process::Command;
 
-pub fn perform(
-    name_and_uri: NameAndUri,
-    command: Vec<String>,
-    execution_options: &ExecutionOptions,
-) -> Result<()> {
-    let NameAndUri {
-        name: bench_name,
-        uri: bench_uri,
-    } = name_and_uri;
-
-    let times_per_round_ns = run_rounds(bench_uri.clone(), command, execution_options)?;
-
-    // Collect walltime results
-    let max_time_ns = times_per_round_ns.iter().copied().max();
-
-    let walltime_benchmark = WalltimeBenchmark::from_runtime_data(
-        bench_name.clone(),
-        bench_uri.clone(),
-        vec![1; times_per_round_ns.len()],
-        times_per_round_ns,
-        max_time_ns,
-    );
-
-    let walltime_results = WalltimeResults::from_benchmarks(vec![walltime_benchmark])
+pub fn perform(commands: Vec<BenchmarkCommand>) -> Result<()> {
+    let mut walltime_benchmarks = Vec::with_capacity(commands.len());
+
+    for cmd in commands {
+        let name_and_uri = generate_name_and_uri(&cmd.name, &cmd.command);
+        let execution_options: ExecutionOptions = cmd.walltime_args.try_into()?;
+
+        let NameAndUri {
+            name: bench_name,
+            uri: bench_uri,
+        } = name_and_uri;
+
+        let times_per_round_ns = run_rounds(bench_uri.clone(), cmd.command, &execution_options)?;
+
+        // Collect walltime results
+        let max_time_ns = times_per_round_ns.iter().copied().max();
+
+        let walltime_benchmark = WalltimeBenchmark::from_runtime_data(
+            bench_name.clone(),
+            bench_uri.clone(),
+            vec![1; times_per_round_ns.len()],
+            times_per_round_ns,
+            max_time_ns,
+        );
+
+        walltime_benchmarks.push(walltime_benchmark);
+    }
+
+    let walltime_results = WalltimeResults::from_benchmarks(walltime_benchmarks)
         .expect("Failed to create walltime results");
 
     walltime_results
@@ -56,7 +63,7 @@ fn run_rounds(
     let warmup_time_ns = config.warmup_time_ns;
     let hooks = InstrumentHooks::instance();
 
-    let do_one_round = |times_per_round_ns: &mut Vec<u128>| {
+    let do_one_round = |times_per_round_ns: &mut Vec<u128>, add_markers: bool| {
         let mut child = Command::new(&command[0])
             .args(&command[1..])
             .spawn()
@@ -67,27 +74,28 @@ fn run_rounds(
             .context("Failed to wait for command to finish")?;
 
         let bench_round_end_ts_ns = InstrumentHooks::current_timestamp();
-        hooks.add_benchmark_timestamps(bench_round_start_ts_ns, bench_round_end_ts_ns);
+
+        if add_markers {
+            hooks.add_benchmark_timestamps(bench_round_start_ts_ns, bench_round_end_ts_ns);
+        }
+        times_per_round_ns.push((bench_round_end_ts_ns - bench_round_start_ts_ns) as u128);
 
         if !status.success() {
             bail!("Command exited with non-zero status: {status}");
         }
 
-        times_per_round_ns.push((bench_round_end_ts_ns - bench_round_start_ts_ns) as u128);
-
         Ok(())
     };
 
     // Compute the number of rounds to perform, either from warmup or directly from config
+    hooks.start_benchmark().unwrap();
     let rounds_to_perform = if warmup_time_ns > 0 {
         let mut warmup_times_ns = Vec::new();
         let warmup_start_ts_ns = InstrumentHooks::current_timestamp();
 
-        hooks.start_benchmark().unwrap();
         while InstrumentHooks::current_timestamp() < warmup_start_ts_ns + warmup_time_ns {
-            do_one_round(&mut warmup_times_ns)?;
+            do_one_round(&mut warmup_times_ns, false)?;
         }
-        hooks.stop_benchmark().unwrap();
         let warmup_end_ts_ns = InstrumentHooks::current_timestamp();
 
         if let [single_warmup_round_duration_ns] = warmup_times_ns.as_slice() {
@@ -98,6 +106,8 @@ fn run_rounds(
                             "Warmup duration ({single_warmup_round_duration_ns} ns) exceeded or met max_time ({time_ns} ns). No more rounds will be performed."
                         );
                         // Mark benchmark as executed for the runner to register
+                        hooks.add_benchmark_timestamps(warmup_start_ts_ns, warmup_end_ts_ns);
+                        hooks.stop_benchmark().unwrap();
                         hooks.set_executed_benchmark(&bench_uri).unwrap();
                         return Ok(warmup_times_ns);
                     }
@@ -170,9 +180,8 @@ fn run_rounds(
     let round_start_ts_ns = InstrumentHooks::current_timestamp();
     let mut times_per_round_ns = Vec::with_capacity(rounds_to_perform.try_into().unwrap());
 
-    hooks.start_benchmark().unwrap();
     for round in 0..rounds_to_perform {
-        do_one_round(&mut times_per_round_ns)?;
+        do_one_round(&mut times_per_round_ns, true)?;
 
         // Check if we've exceeded max time
         let max_time_ns = match &config.max {

src/exec/mod.rs

@@ -9,6 +9,7 @@ use crate::run::uploader::UploadResult;
 use clap::Args;
 use std::path::Path;
 
+pub mod multi_targets;
 mod poll_results;
 
 /// We temporarily force this name for all exec runs
@@ -78,8 +79,8 @@ pub async fn run(
     setup_cache_dir: Option<&Path>,
 ) -> Result<()> {
     let merged_args = args.merge_with_project_config(project_config);
-
     let config = crate::executor::Config::try_from(merged_args)?;
+
     let mut execution_context = executor::ExecutionContext::try_from((config, codspeed_config))?;
     debug!("config: {:#?}", execution_context.config);
     let executor = executor::get_executor_from_mode(