diff --git a/Makefile b/Makefile index f8f007a86..7a902c349 100644 --- a/Makefile +++ b/Makefile @@ -232,14 +232,21 @@ clean: clean-all: clean clean-sysroot -test-prefix/lib64/libkrun.pc: $(LIBRARY_RELEASE_$(OS)) +test-prefix/$(LIBDIR_$(OS))/libkrun.pc: $(LIBRARY_RELEASE_$(OS)) mkdir -p test-prefix PREFIX="$$(realpath test-prefix)" make install -test-prefix: test-prefix/lib64/libkrun.pc +test-prefix: test-prefix/$(LIBDIR_$(OS))/libkrun.pc TEST ?= all TEST_FLAGS ?= +# Library path variable differs by OS +LIBPATH_VAR_Linux = LD_LIBRARY_PATH +LIBPATH_VAR_Darwin = DYLD_LIBRARY_PATH +# Extra library paths needed for tests (libkrunfw, llvm) +EXTRA_LIBPATH_Linux = +EXTRA_LIBPATH_Darwin = /opt/homebrew/opt/libkrunfw/lib:/opt/homebrew/opt/llvm/lib + test: test-prefix - cd tests; RUST_LOG=trace LD_LIBRARY_PATH="$$(realpath ../test-prefix/lib64/)" PKG_CONFIG_PATH="$$(realpath ../test-prefix/lib64/pkgconfig/)" ./run.sh test --test-case "$(TEST)" $(TEST_FLAGS) + cd tests; RUST_LOG=trace $(LIBPATH_VAR_$(OS))="$$(realpath ../test-prefix/$(LIBDIR_$(OS))/):$(EXTRA_LIBPATH_$(OS)):$${$(LIBPATH_VAR_$(OS))}" PKG_CONFIG_PATH="$$(realpath ../test-prefix/$(LIBDIR_$(OS))/pkgconfig/)" ./run.sh test --test-case "$(TEST)" $(TEST_FLAGS) diff --git a/tests/README.md b/tests/README.md index bc61b9da2..4eb6bac43 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,9 +1,46 @@ # End-to-end tests -The testing framework here allows you to write code to configure libkrun (using the public API) and run some specific code in the guest. +The testing framework here allows you to write code to configure libkrun (using the public API) and run some specific code in the guest. ## Running the tests: The tests can be ran using `make test` (from the main libkrun directory). -You can also run `./run.sh` inside the `test` directory. When using the `./run.sh` script you probably want specify the `PKG_CONFIG_PATH` enviroment variable, otherwise you will be testing the system wide installation of libkrun. +You can also run `./run.sh` inside the `test` directory. When using the `./run.sh` script you probably want specify the `PKG_CONFIG_PATH` enviroment variable, otherwise you will be testing the system wide installation of libkrun. + +## Running on macOS + +### Prerequisites + +1. Install required build tools: + ```bash + brew install lld xz + rustup target add aarch64-unknown-linux-musl + ``` + +2. Install libkrunfw (required for non-EFI builds). Either via homebrew: + ```bash + brew install libkrunfw + ``` + + Or build from source: + ```bash + curl -LO https://github.com/containers/libkrunfw/releases/download/v5.2.0/libkrunfw-prebuilt-aarch64.tgz + tar -xzf libkrunfw-prebuilt-aarch64.tgz + cd libkrunfw + make + sudo make install + ``` + + If installed from source, add `/usr/local/lib` to your library path: + ```bash + export DYLD_LIBRARY_PATH="/usr/local/lib:${DYLD_LIBRARY_PATH}" + ``` + + The test harness automatically handles the library path for homebrew installations. + +### Running tests + +```bash +make test +``` ## Adding tests To add a test you need to add a new rust module in the `test_cases` directory, implement the required host and guest side methods (see existing tests) and register the test in the `test_cases/src/lib.rs` to be ran. \ No newline at end of file diff --git a/tests/guest-agent/src/main.rs b/tests/guest-agent/src/main.rs index 1f9b7965c..668eb9688 100644 --- a/tests/guest-agent/src/main.rs +++ b/tests/guest-agent/src/main.rs @@ -8,7 +8,7 @@ fn run_guest_agent(test_name: &str) -> anyhow::Result<()> { .into_iter() .find(|t| t.name() == test_name) .context("No such test!")?; - let TestCase { test, name: _ } = test_case; + let TestCase { test, .. } = test_case; test.in_guest(); Ok(()) } diff --git a/tests/run.sh b/tests/run.sh index 128b3e546..b8136fd5f 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -6,15 +6,53 @@ set -e +OS=$(uname -s) +ARCH=$(uname -m) + # Run the unit tests first (this tests the testing framework itself not libkrun) cargo test -p test_cases --features guest -GUEST_TARGET_ARCH="$(uname -m)-unknown-linux-musl" +# Determine guest target architecture +# macOS uses arm64 but Rust uses aarch64 +if [ "$ARCH" = "arm64" ]; then + RUST_ARCH="aarch64" +else + RUST_ARCH="$ARCH" +fi +GUEST_TARGET="${RUST_ARCH}-unknown-linux-musl" + +# On macOS, we need to cross-compile for Linux musl +if [ "$OS" = "Darwin" ]; then + SYSROOT="../linux-sysroot" + if [ ! -d "$SYSROOT" ]; then + echo "ERROR: Linux sysroot not found at $SYSROOT" + echo "Run 'make' in the libkrun root directory first to create it." + exit 1 + fi -cargo build --target=$GUEST_TARGET_ARCH -p guest-agent + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER="clang" + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-C link-arg=-target -C link-arg=aarch64-linux-gnu -C link-arg=-fuse-ld=lld -C link-arg=--sysroot=$SYSROOT -C link-arg=-static" + echo "Cross-compiling guest-agent for $GUEST_TARGET..." +fi + +cargo build --target=$GUEST_TARGET -p guest-agent cargo build -p runner -export KRUN_TEST_GUEST_AGENT_PATH="target/$GUEST_TARGET_ARCH/debug/guest-agent" +# On macOS, the runner needs entitlements to use Hypervisor.framework +if [ "$OS" = "Darwin" ]; then + codesign --entitlements /dev/stdin --force -s - target/debug/runner <<'EOF' + + + + + com.apple.security.hypervisor + + + +EOF +fi + +export KRUN_TEST_GUEST_AGENT_PATH="target/$GUEST_TARGET/debug/guest-agent" # Build runner args: pass through all arguments RUNNER_ARGS="$*" @@ -24,7 +62,7 @@ if [ -n "${KRUN_TEST_BASE_DIR}" ]; then RUNNER_ARGS="${RUNNER_ARGS} --base-dir ${KRUN_TEST_BASE_DIR}" fi -if [ -z "${KRUN_NO_UNSHARE}" ] && which unshare 2>&1 >/dev/null; then +if [ "$OS" != "Darwin" ] && [ -z "${KRUN_NO_UNSHARE}" ] && which unshare 2>&1 >/dev/null; then unshare --user --map-root-user --net -- /bin/sh -c "ifconfig lo 127.0.0.1 && exec target/debug/runner ${RUNNER_ARGS}" else echo "WARNING: Running tests without a network namespace." diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index d3d3a702a..8d1d73487 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -8,12 +8,19 @@ use std::panic::catch_unwind; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use tempdir::TempDir; -use test_cases::{test_cases, Test, TestCase, TestSetup}; +use test_cases::{test_cases, ShouldRun, Test, TestCase, TestSetup}; + +#[derive(Clone)] +enum TestOutcome { + Pass, + Fail, + Skip(&'static str), +} struct TestResult { name: String, - passed: bool, - log_path: PathBuf, + outcome: TestOutcome, + log_path: Option, } fn get_test(name: &str) -> anyhow::Result> { @@ -39,28 +46,39 @@ fn start_vm(test_setup: TestSetup) -> anyhow::Result<()> { } fn run_single_test( - test_case: &str, + test_case: &TestCase, base_dir: &Path, keep_all: bool, max_name_len: usize, ) -> anyhow::Result { + eprint!( + "[{}] {:. anyhow::Result<()> { let summary_path = env::var("GITHUB_STEP_SUMMARY") .context("GITHUB_STEP_SUMMARY environment variable not set")?; @@ -106,33 +126,50 @@ fn write_github_summary( .open(&summary_path) .context("Failed to open GITHUB_STEP_SUMMARY")?; - let all_passed = num_ok == num_tests; - let status = if all_passed { "✅" } else { "❌" }; + let num_ran = num_pass + num_fail; + let status = if num_fail == 0 { "✅" } else { "❌" }; + let skip_msg = if num_skip > 0 { + format!(" ({num_skip} skipped)") + } else { + String::new() + }; writeln!( file, - "## {status} Integration Tests ({num_ok}/{num_tests} passed)\n" + "## {status} Integration Tests - {num_pass}/{num_ran} passed{skip_msg}\n" )?; for result in results { - let icon = if result.passed { "✅" } else { "❌" }; - let log_content = fs::read_to_string(&result.log_path).unwrap_or_default(); + let (icon, status_text) = match &result.outcome { + TestOutcome::Pass => ("✅", String::new()), + TestOutcome::Fail => ("❌", String::new()), + TestOutcome::Skip(reason) => ("⏭️", format!(" - {}", reason)), + }; writeln!(file, "
")?; - writeln!(file, "{icon} {}\n", result.name)?; - writeln!(file, "```")?; - // Limit log size to avoid huge summaries (2 MiB limit) - const MAX_LOG_SIZE: usize = 2 * 1024 * 1024; - let truncated = if log_content.len() > MAX_LOG_SIZE { - format!( - "... (truncated, showing last 1 MiB) ...\n{}", - &log_content[log_content.len() - MAX_LOG_SIZE..] - ) - } else { - log_content - }; - writeln!(file, "{truncated}")?; - writeln!(file, "```")?; + writeln!( + file, + "{icon} {}{}\n", + result.name, status_text + )?; + + if let Some(log_path) = &result.log_path { + let log_content = fs::read_to_string(log_path).unwrap_or_default(); + writeln!(file, "```")?; + // Limit log size to avoid huge summaries (2 MiB limit) + const MAX_LOG_SIZE: usize = 2 * 1024 * 1024; + let truncated = if log_content.len() > MAX_LOG_SIZE { + format!( + "... (truncated, showing last 1 MiB) ...\n{}", + &log_content[log_content.len() - MAX_LOG_SIZE..] + ) + } else { + log_content + }; + writeln!(file, "{truncated}")?; + writeln!(file, "```")?; + } + writeln!(file, "
\n")?; } @@ -157,40 +194,61 @@ fn run_tests( }; let mut results: Vec = Vec::new(); + let all_tests = test_cases(); - if test_case == "all" { - let all_tests = test_cases(); - let max_name_len = all_tests.iter().map(|t| t.name.len()).max().unwrap_or(0); - - for TestCase { name, test: _ } in all_tests { - results.push(run_single_test(name, &base_dir, keep_all, max_name_len).context(name)?); - } + let tests_to_run: Vec<_> = if test_case == "all" { + all_tests } else { - let max_name_len = test_case.len(); - results.push( - run_single_test(test_case, &base_dir, keep_all, max_name_len) - .context(test_case.to_string())?, - ); + all_tests + .into_iter() + .filter(|t| t.name == test_case) + .collect() + }; + + if tests_to_run.is_empty() { + anyhow::bail!("No such test: {test_case}"); + } + + let max_name_len = tests_to_run.iter().map(|t| t.name.len()).max().unwrap_or(0); + + for tc in &tests_to_run { + results.push(run_single_test(tc, &base_dir, keep_all, max_name_len).context(tc.name)?); } - let num_tests = results.len(); - let num_ok = results.iter().filter(|r| r.passed).count(); + let num_pass = results + .iter() + .filter(|r| matches!(r.outcome, TestOutcome::Pass)) + .count(); + let num_fail = results + .iter() + .filter(|r| matches!(r.outcome, TestOutcome::Fail)) + .count(); + let num_skip = results + .iter() + .filter(|r| matches!(r.outcome, TestOutcome::Skip(_))) + .count(); + let num_ran = num_pass + num_fail; // Write GitHub Actions summary if requested if github_summary { - write_github_summary(&results, num_ok, num_tests)?; + write_github_summary(&results, num_pass, num_fail, num_skip)?; } - let num_failures = num_tests - num_ok; - if num_failures > 0 { + let skip_msg = if num_skip > 0 { + format!(" ({num_skip} skipped)") + } else { + String::new() + }; + + if num_fail > 0 { eprintln!("(See test artifacts at: {})", base_dir.display()); - println!("\nFAIL (PASSED {num_ok}/{num_tests})"); + println!("\nFAIL - {num_pass}/{num_ran} passed{skip_msg}"); anyhow::bail!("") } else { if keep_all { eprintln!("(See test artifacts at: {})", base_dir.display()); } - eprintln!("\nOK ({num_ok}/{num_tests} passed)"); + eprintln!("\nOK - {num_pass}/{num_ran} passed{skip_msg}"); } Ok(()) diff --git a/tests/test_cases/src/lib.rs b/tests/test_cases/src/lib.rs index dfe5211a0..f164ed87e 100644 --- a/tests/test_cases/src/lib.rs +++ b/tests/test_cases/src/lib.rs @@ -13,6 +13,22 @@ use test_tsi_tcp_guest_listen::TestTsiTcpGuestListen; mod test_multiport_console; use test_multiport_console::TestMultiportConsole; +pub enum ShouldRun { + Yes, + No(&'static str), +} + +impl ShouldRun { + /// Returns Yes unless on macOS, in which case returns No with the given reason. + pub fn yes_unless_macos(reason: &'static str) -> Self { + if cfg!(target_os = "macos") { + ShouldRun::No(reason) + } else { + ShouldRun::Yes + } + } +} + pub fn test_cases() -> Vec { // Register your test here: vec![ @@ -80,6 +96,11 @@ pub trait Test { let output = child.wait_with_output().unwrap(); assert_eq!(String::from_utf8(output.stdout).unwrap(), "OK\n"); } + + /// Check if this test should run on this platform. + fn should_run(&self) -> ShouldRun { + ShouldRun::Yes + } } #[guest] @@ -100,6 +121,12 @@ impl TestCase { Self { name, test } } + /// Check if this test should run on this platform. + #[host] + pub fn should_run(&self) -> ShouldRun { + self.test.should_run() + } + #[allow(dead_code)] pub fn name(&self) -> &'static str { self.name diff --git a/tests/test_cases/src/test_tsi_tcp_guest_connect.rs b/tests/test_cases/src/test_tsi_tcp_guest_connect.rs index 038501b37..f16a84db2 100644 --- a/tests/test_cases/src/test_tsi_tcp_guest_connect.rs +++ b/tests/test_cases/src/test_tsi_tcp_guest_connect.rs @@ -21,11 +21,15 @@ mod host { use crate::common::setup_fs_and_enter; use crate::{krun_call, krun_call_u32}; - use crate::{Test, TestSetup}; + use crate::{ShouldRun, Test, TestSetup}; use krun_sys::*; use std::thread; impl Test for TestTsiTcpGuestConnect { + fn should_run(&self) -> ShouldRun { + ShouldRun::yes_unless_macos("broken on macOS") + } + fn start_vm(self: Box, test_setup: TestSetup) -> anyhow::Result<()> { let listener = self.tcp_tester.create_server_socket(); thread::spawn(move || self.tcp_tester.run_server(listener)); diff --git a/tests/test_cases/src/test_tsi_tcp_guest_listen.rs b/tests/test_cases/src/test_tsi_tcp_guest_listen.rs index 9838ed893..b3efd24ed 100644 --- a/tests/test_cases/src/test_tsi_tcp_guest_listen.rs +++ b/tests/test_cases/src/test_tsi_tcp_guest_listen.rs @@ -19,7 +19,7 @@ impl TestTsiTcpGuestListen { mod host { use super::*; use crate::common::setup_fs_and_enter; - use crate::{krun_call, krun_call_u32, Test, TestSetup}; + use crate::{krun_call, krun_call_u32, ShouldRun, Test, TestSetup}; use krun_sys::*; use std::ffi::CString; use std::ptr::null; @@ -27,6 +27,10 @@ mod host { use std::time::Duration; impl Test for TestTsiTcpGuestListen { + fn should_run(&self) -> ShouldRun { + ShouldRun::yes_unless_macos("broken on macOS") + } + fn start_vm(self: Box, test_setup: TestSetup) -> anyhow::Result<()> { unsafe { thread::spawn(move || { diff --git a/tests/test_cases/src/test_vsock_guest_connect.rs b/tests/test_cases/src/test_vsock_guest_connect.rs index bb0482f29..8e66a086d 100644 --- a/tests/test_cases/src/test_vsock_guest_connect.rs +++ b/tests/test_cases/src/test_vsock_guest_connect.rs @@ -35,7 +35,7 @@ mod host { use crate::common::setup_fs_and_enter; use crate::{krun_call, krun_call_u32}; - use crate::{Test, TestSetup}; + use crate::{ShouldRun, Test, TestSetup}; use krun_sys::*; use std::ffi::CString; use std::io::Write; @@ -55,6 +55,10 @@ mod host { } impl Test for TestVsockGuestConnect { + fn should_run(&self) -> ShouldRun { + ShouldRun::yes_unless_macos("broken on macOS") + } + fn start_vm(self: Box, test_setup: TestSetup) -> anyhow::Result<()> { let sock_path = test_setup.tmp_dir.join("test.sock"); let sock_path_cstr = CString::new(sock_path.as_os_str().as_bytes())?;