diff --git a/.gitignore b/.gitignore index fcdfba44..732385d7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ fort.* RUN/PICS *.sublime-* RUN/ +/golden_record/ *.pdf \#* *.so diff --git a/cmake/Util.cmake b/cmake/Util.cmake index c6d2e330..5d36e1cc 100644 --- a/cmake/Util.cmake +++ b/cmake/Util.cmake @@ -1,39 +1,107 @@ include(FetchContent) function(find_or_fetch DEPENDENCY) - if(DEFINED ENV{CODE}) - set(${DEPENDENCY}_SOURCE_DIR $ENV{CODE}/${DEPENDENCY}) - message(STATUS "Using ${DEPENDENCY} in $ENV{CODE}/${DEPENDENCY}") - else() - set(REPO_URL https://github.com/itpplasma/${DEPENDENCY}.git) - set(REMOTE_BRANCH "") - - if(${DEPENDENCY} STREQUAL "libneo") - if(DEFINED LIBNEO_BRANCH AND NOT "${LIBNEO_BRANCH}" STREQUAL "") - string(STRIP "${LIBNEO_BRANCH}" REMOTE_BRANCH) - elseif(DEFINED ENV{LIBNEO_BRANCH} AND NOT "$ENV{LIBNEO_BRANCH}" STREQUAL "") - string(STRIP "$ENV{LIBNEO_BRANCH}" REMOTE_BRANCH) + string(TOUPPER "${DEPENDENCY}" _dep_upper) + string(TOLOWER "${DEPENDENCY}" _dep_lower) + + set(_source_dir "") + set(_use_local FALSE) + + # Allow explicit override via _SOURCE_DIR cache variable + set(_override_var "${_dep_upper}_SOURCE_DIR") + if(DEFINED ${_override_var}) + get_filename_component(_candidate "${${_override_var}}" ABSOLUTE "${CMAKE_SOURCE_DIR}") + if(IS_DIRECTORY "${_candidate}") + set(_source_dir "${_candidate}") + set(_use_local TRUE) + else() + message(FATAL_ERROR "${_override_var}='${_candidate}' is not a directory") + endif() + endif() + + if(NOT _use_local AND DEFINED ENV{CODE}) + get_filename_component(_candidate "$ENV{CODE}/${_dep_lower}" ABSOLUTE) + if(IS_DIRECTORY "${_candidate}") + set(_source_dir "${_candidate}") + set(_use_local TRUE) + endif() + endif() + + if(_use_local) + message(STATUS "Using ${_dep_lower} in ${_source_dir}") + set(${_dep_upper}_SOURCE_DIR "${_source_dir}" CACHE PATH "" FORCE) + if("${_dep_lower}" STREQUAL "libneo") + set(LIBNEO_ENABLE_TESTS OFF CACHE BOOL "" FORCE) + endif() + add_subdirectory("${_source_dir}" "${CMAKE_CURRENT_BINARY_DIR}/${_dep_lower}" EXCLUDE_FROM_ALL) + return() + endif() + + set(_repo_url "https://github.com/itpplasma/${_dep_lower}.git") + set(_repo_var "${_dep_upper}_REPO_URL") + if(DEFINED ${_repo_var} AND NOT "${${_repo_var}}" STREQUAL "") + set(_repo_url "${${_repo_var}}") + elseif(DEFINED ENV{${_repo_var}} AND NOT "$ENV{${_repo_var}}" STREQUAL "") + set(_repo_url "$ENV{${_repo_var}}") + endif() + if("${_dep_lower}" STREQUAL "libneo") + set(_branch "") + set(_branch_source "") + if(DEFINED LIBNEO_BRANCH AND NOT "${LIBNEO_BRANCH}" STREQUAL "") + string(STRIP "${LIBNEO_BRANCH}" _branch) + set(_branch_source "cache") + elseif(DEFINED ENV{LIBNEO_BRANCH} AND NOT "$ENV{LIBNEO_BRANCH}" STREQUAL "") + string(STRIP "$ENV{LIBNEO_BRANCH}" _branch) + set(_branch_source "env") + endif() + + if(NOT _branch STREQUAL "") + execute_process( + COMMAND git ls-remote --heads ${_repo_url} ${_branch} + OUTPUT_VARIABLE _branch_exists + RESULT_VARIABLE _branch_check_status + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + set(_branch_valid FALSE) + if(_branch_check_status EQUAL 0 AND NOT _branch_exists STREQUAL "") + set(_branch_valid TRUE) + endif() + if(NOT _branch_valid) + message(WARNING "LIBNEO branch '${_branch}' (source: ${_branch_source}) not found; falling back to auto-detected branch") + set(_branch "") endif() endif() - if(REMOTE_BRANCH STREQUAL "") - get_branch_or_main(${REPO_URL} REMOTE_BRANCH) + if(_branch STREQUAL "") + get_branch_or_main(${_repo_url} _branch) endif() - message(STATUS "Using ${DEPENDENCY} branch ${REMOTE_BRANCH} from ${REPO_URL}") - - FetchContent_Declare( - ${DEPENDENCY} - DOWNLOAD_EXTRACT_TIMESTAMP TRUE - GIT_REPOSITORY ${REPO_URL} - GIT_TAG ${REMOTE_BRANCH} - ) - FetchContent_Populate(${DEPENDENCY}) + + message(STATUS "Using ${_dep_lower} branch ${_branch} from ${_repo_url}") + + set(LIBNEO_ENABLE_TESTS OFF CACHE BOOL "" FORCE) + else() + set(_branch "") + set(_branch_var "${_dep_upper}_BRANCH") + if(DEFINED ${_branch_var} AND NOT "${${_branch_var}}" STREQUAL "") + string(STRIP "${${_branch_var}}" _branch) + elseif(DEFINED ENV{${_branch_var}} AND NOT "$ENV{${_branch_var}}" STREQUAL "") + string(STRIP "$ENV{${_branch_var}}" _branch) + endif() + + if(_branch STREQUAL "") + get_branch_or_main(${_repo_url} _branch) + endif() + + message(STATUS "Using ${_dep_lower} branch ${_branch} from ${_repo_url}") endif() - add_subdirectory(${${DEPENDENCY}_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/${DEPENDENCY} - EXCLUDE_FROM_ALL + FetchContent_Declare(${_dep_lower} + GIT_REPOSITORY ${_repo_url} + GIT_TAG ${_branch} + GIT_PROGRESS TRUE + DOWNLOAD_EXTRACT_TIMESTAMP TRUE ) + FetchContent_MakeAvailable(${_dep_lower}) endfunction() diff --git a/test/golden_record/compare_golden_results.sh b/test/golden_record/compare_golden_results.sh index 25039370..74e7b727 100755 --- a/test/golden_record/compare_golden_results.sh +++ b/test/golden_record/compare_golden_results.sh @@ -57,8 +57,17 @@ compare_cases() { for CASE in $TEST_CASES; do total_cases=$((total_cases + 1)) - echo "Comparing $CASE case..." + + if [ ! -d "$REFERENCE_DIR/$CASE" ]; then + echo " ⚠ Skipping (missing in reference run)" + continue + fi + if [ ! -d "$CURRENT_DIR/$CASE" ]; then + echo " ✗ Current run missing directory: $CURRENT_DIR/$CASE" + missing_cases=$((missing_cases + 1)) + continue + fi # Check if this is the classifier_fast case with multiple files if [ "$CASE" = "classifier_fast" ]; then @@ -134,4 +143,4 @@ else echo "Some tests failed or had missing files." fi -exit $exit_code \ No newline at end of file +exit $exit_code diff --git a/test/golden_record/golden_record.sh b/test/golden_record/golden_record.sh index 8fc9e3dc..654e763e 100755 --- a/test/golden_record/golden_record.sh +++ b/test/golden_record/golden_record.sh @@ -1,194 +1,170 @@ #!/bin/bash -CLONE_URL="https://github.com/itpplasma/SIMPLE.git" +set -euo pipefail -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -CUR_VER=$(git -C "$SCRIPT_DIR" describe --tags --always --dirty) -REF_VER=${1:-"main"} +# Golden record comparison against a single reference (default: main) -# Allow override of directories via environment variables -GOLDEN_RECORD_BASE_DIR=${GOLDEN_RECORD_BASE_DIR:-"$(pwd)/golden_record"} -GOLDEN_RECORD_REF_DIR=${GOLDEN_RECORD_REF_DIR:-""} -GOLDEN_RECORD_CUR_DIR=${GOLDEN_RECORD_CUR_DIR:-""} +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROJECT_ROOT_CUR_DEFAULT=$(cd "$SCRIPT_DIR/../.." && pwd) +REF_SPEC=${1:-main} -# Set up deterministic directories -if [ -z "$GOLDEN_RECORD_REF_DIR" ]; then - PROJECT_ROOT_REF="$GOLDEN_RECORD_BASE_DIR/simple_$REF_VER" +REMOTE_URL=$(git -C "$PROJECT_ROOT_CUR_DEFAULT" remote get-url origin 2>/dev/null || echo "git@github.com:itpplasma/SIMPLE.git") +if [[ "$REMOTE_URL" == https://github.com/* ]]; then + REPO_PREFIX="https://github.com/itpplasma" else - PROJECT_ROOT_REF="$GOLDEN_RECORD_REF_DIR" + REPO_PREFIX="git@github.com:itpplasma" fi +LIBNEO_URL="${REPO_PREFIX%/}/libneo.git" -if [ -z "$GOLDEN_RECORD_CUR_DIR" ]; then - PROJECT_ROOT_CUR=$(cd "$SCRIPT_DIR/../.." && pwd) -else - PROJECT_ROOT_CUR="$GOLDEN_RECORD_CUR_DIR" -fi +GOLDEN_RECORD_BASE_DIR=${GOLDEN_RECORD_BASE_DIR:-"$(pwd)/golden_record"} +PROJECT_ROOT_CUR=${GOLDEN_RECORD_CUR_DIR:-$PROJECT_ROOT_CUR_DEFAULT} +PROJECT_ROOT_REF=${GOLDEN_RECORD_REF_DIR:-"$GOLDEN_RECORD_BASE_DIR/reference_$REF_SPEC"} -RUN_DIR_REF="$GOLDEN_RECORD_BASE_DIR/runs/run_$REF_VER" -RUN_DIR_CUR="$GOLDEN_RECORD_BASE_DIR/runs/run_$CUR_VER" +RUN_DIR_CUR="$GOLDEN_RECORD_BASE_DIR/runs/run_current" +RUN_DIR_REF="$GOLDEN_RECORD_BASE_DIR/runs/run_reference" TEST_DATA_DIR="$GOLDEN_RECORD_BASE_DIR/test_data" -# Find test cases - they should be copied by CMake to the golden_record directory -if [ -d "$GOLDEN_RECORD_BASE_DIR/test_cases" ]; then - TEST_CASES="$(cd "$GOLDEN_RECORD_BASE_DIR/test_cases" && find . -name simple.in -exec dirname {} \; | sed 's|^\./||' | sort)" -else - # Fallback to original location if CMake hasn't copied them yet - TEST_CASES="$(cd "$SCRIPT_DIR" && find . -name simple.in -exec dirname {} \; | sed 's|^\./||' | sort)" -fi - -echo "Golden record base directory: $GOLDEN_RECORD_BASE_DIR" -echo "Reference version: $REF_VER" -echo "Current version: $CUR_VER" -echo "Project root (reference): $PROJECT_ROOT_REF" -echo "Project root (current): $PROJECT_ROOT_CUR" -echo "Test run directories:" -echo " Current: $RUN_DIR_CUR" -echo " Reference: $RUN_DIR_REF" - -handle_success() { - echo "Tests completed successfully." - echo "Results preserved in: $GOLDEN_RECORD_BASE_DIR" -} - -handle_failure() { - local exit_code=$1 - echo "Tests failed with exit code: $exit_code" - echo "Results preserved for debugging in: $GOLDEN_RECORD_BASE_DIR" - exit $exit_code +log() { + echo "[golden] $*" } -# Determine and check out appropriate libneo version for historical builds setup_libneo() { - local PROJECT_ROOT="$1" - local BASE_DIR="$(dirname "$PROJECT_ROOT")" - local LIBNEO_URL="https://github.com/itpplasma/libneo.git" - - # Timestamp of SIMPLE commit - local SIMPLE_TS - SIMPLE_TS=$(git -C "$PROJECT_ROOT" log -1 --format=%ct) - - # Cache repository with all tags for lookup - local LIBNEO_TAGS_DIR="$BASE_DIR/libneo_tags" - if [ ! -d "$LIBNEO_TAGS_DIR/.git" ]; then - git clone --quiet --no-checkout "$LIBNEO_URL" "$LIBNEO_TAGS_DIR" + local project_root="$1" + local base_dir + base_dir="$(dirname "$project_root")" + + local simple_ts + simple_ts=$(git -C "$project_root" log -1 --format=%ct) + + local tags_dir="$base_dir/libneo_tags" + if [ ! -d "$tags_dir/.git" ]; then + git clone --quiet --no-checkout "$LIBNEO_URL" "$tags_dir" else - git -C "$LIBNEO_TAGS_DIR" fetch --tags --quiet + git -C "$tags_dir" fetch --tags --quiet fi - # Find newest tag not newer than SIMPLE commit - # Use creatordate which works for both annotated and lightweight tags - local LIBNEO_TAG - LIBNEO_TAG=$(git -C "$LIBNEO_TAGS_DIR" for-each-ref --sort=-creatordate \ - --format '%(refname:short) %(creatordate:unix)' refs/tags | while read tag ts; do - # Check that timestamp is not empty and is valid - if [ -n "$ts" ] && [ "$ts" -le "$SIMPLE_TS" ] 2>/dev/null; then + local libneo_tag + libneo_tag=$(git -C "$tags_dir" for-each-ref --sort=-creatordate \ + --format '%(refname:short) %(creatordate:unix)' refs/tags | while read -r tag ts; do + if [ -n "$ts" ] && [ "$ts" -le "$simple_ts" ] 2>/dev/null; then echo "$tag" break fi done) - if [ -z "$LIBNEO_TAG" ]; then - echo "No suitable libneo tag found; using oldest available for compatibility" - # Use oldest tag for better compatibility with historical commits - LIBNEO_TAG=$(git -C "$LIBNEO_TAGS_DIR" for-each-ref --sort=creatordate \ + if [ -z "$libneo_tag" ]; then + log "No libneo tag older than commit timestamp; falling back to oldest" + libneo_tag=$(git -C "$tags_dir" for-each-ref --sort=creatordate \ --format '%(refname:short)' refs/tags | head -n 1) fi - echo "Using libneo tag: $LIBNEO_TAG" - local LIBNEO_TAG_DIR="$BASE_DIR/libneo_$LIBNEO_TAG" - if [ ! -d "$LIBNEO_TAG_DIR/.git" ]; then - git clone --quiet --branch "$LIBNEO_TAG" --depth 1 "$LIBNEO_URL" "$LIBNEO_TAG_DIR" + log "Using libneo tag: $libneo_tag" + local libneo_dir="$base_dir/libneo_$libneo_tag" + if [ ! -d "$libneo_dir/.git" ]; then + git clone --quiet --branch "$libneo_tag" --depth 1 "$LIBNEO_URL" "$libneo_dir" fi - ln -sfn "$LIBNEO_TAG_DIR" "$BASE_DIR/libneo" + ln -sfn "$libneo_dir" "$base_dir/libneo" } -main() { - set -e # Exit on any error - - # Create base directories - mkdir -p "$GOLDEN_RECORD_BASE_DIR" - mkdir -p "$TEST_DATA_DIR" +resolve_reference() { + git -C "$PROJECT_ROOT_CUR_DEFAULT" fetch --tags --quiet origin || true - # Check if we need to build reference version - if [ ! -f "$PROJECT_ROOT_REF/build/simple.x" ]; then - echo "Reference build not found, cloning and building..." - clone "$REF_VER" "$PROJECT_ROOT_REF" || handle_failure $? - build "$PROJECT_ROOT_REF" || handle_failure $? - else - echo "Using existing reference build at: $PROJECT_ROOT_REF/build/simple.x" + if git -C "$PROJECT_ROOT_CUR_DEFAULT" rev-parse --verify --quiet "$REF_SPEC^{commit}" >/dev/null; then + git -C "$PROJECT_ROOT_CUR_DEFAULT" rev-parse "$REF_SPEC" + return fi - # Check if current version needs building (it should already be built) - if [ ! -f "$PROJECT_ROOT_CUR/build/simple.x" ]; then - echo "Current build not found at: $PROJECT_ROOT_CUR/build/simple.x" - handle_failure 1 - else - echo "Using current build at: $PROJECT_ROOT_CUR/build/simple.x" + if git -C "$PROJECT_ROOT_CUR_DEFAULT" rev-parse --verify --quiet "origin/$REF_SPEC^{commit}" >/dev/null; then + git -C "$PROJECT_ROOT_CUR_DEFAULT" rev-parse "origin/$REF_SPEC" + return fi - # Use the new scripts to run tests and compare - "$SCRIPT_DIR/run_golden_tests.sh" "$PROJECT_ROOT_REF" "$RUN_DIR_REF" "$TEST_DATA_DIR" || handle_failure $? - "$SCRIPT_DIR/run_golden_tests.sh" "$PROJECT_ROOT_CUR" "$RUN_DIR_CUR" "$TEST_DATA_DIR" || handle_failure $? + log "Reference '$REF_SPEC' not found locally or on origin" + exit 1 +} + +prepare_reference() { + mkdir -p "$GOLDEN_RECORD_BASE_DIR" - # Compare results - "$SCRIPT_DIR/compare_golden_results.sh" "$RUN_DIR_REF" "$RUN_DIR_CUR" - comparison_result=$? + local resolved_sha + resolved_sha=$(resolve_reference) - if [ $comparison_result -eq 0 ]; then - handle_success + if [ ! -e "$PROJECT_ROOT_REF/.git" ]; then + rm -rf "$PROJECT_ROOT_REF" + log "Creating reference worktree for $REF_SPEC" + git -C "$PROJECT_ROOT_CUR_DEFAULT" worktree add --force --detach "$PROJECT_ROOT_REF" "$resolved_sha" >/dev/null else - handle_failure $comparison_result + log "Updating reference worktree to $resolved_sha" + git -C "$PROJECT_ROOT_REF" reset --hard "$resolved_sha" --quiet fi -} + local build_dir="$PROJECT_ROOT_REF/build" + local marker_file="$build_dir/.golden_reference_sha" + + if [ -f "$marker_file" ] && [ -f "$build_dir/simple.x" ] && \ + [ "$(cat "$marker_file")" = "$resolved_sha" ]; then + log "Reference build @$resolved_sha already prepared" + return + fi + + log "Building reference SIMPLE at $resolved_sha" + rm -rf "$build_dir" -clone() { - local VERSION="$1" - local PROJECT_ROOT="$2" - if [ ! -d "$PROJECT_ROOT" ]; then - echo "Cloning SIMPLE version $VERSION" - git clone --filter=blob:none --no-checkout "$CLONE_URL" "$PROJECT_ROOT" + if [ -f "$PROJECT_ROOT_REF/SRC/CMakeLists.txt" ] && \ + grep -q "../libneo" "$PROJECT_ROOT_REF/SRC/CMakeLists.txt" 2>/dev/null; then + setup_libneo "$PROJECT_ROOT_REF" fi - cd "$PROJECT_ROOT" - git fetch --all --quiet - git checkout "$VERSION" --quiet + ( + export LIBNEO_BRANCH="main" + + if ! cmake -S "$PROJECT_ROOT_REF" -B "$build_dir" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON_INTERFACE=OFF -DENABLE_GVEC=OFF \ + > "$PROJECT_ROOT_REF/configure.log" 2>&1; then + log "CMake configuration failed; see $PROJECT_ROOT_REF/configure.log" + tail -n 50 "$PROJECT_ROOT_REF/configure.log" + exit 1 + fi + + if ! cmake --build "$build_dir" --config Release > "$PROJECT_ROOT_REF/build.log" 2>&1; then + log "Build failed; see $PROJECT_ROOT_REF/build.log" + tail -n 50 "$PROJECT_ROOT_REF/build.log" + exit 1 + fi + ) || exit 1 + + echo "$resolved_sha" > "$marker_file" } -build() { - local PROJECT_ROOT="$1" - echo "Building SIMPLE in $PROJECT_ROOT" - cd $PROJECT_ROOT +run_suite() { + local project_root="$1" + local run_dir="$2" - # For older versions, check if libneo is needed as a sibling directory - if [ -f "SRC/CMakeLists.txt" ] && grep -q "../libneo" "SRC/CMakeLists.txt" 2>/dev/null; then - echo "Old project structure detected, setting up libneo..." - setup_libneo "$PROJECT_ROOT" - fi + "$SCRIPT_DIR/run_golden_tests.sh" "$project_root" "$run_dir" "$TEST_DATA_DIR" +} - # Check if CMakeLists.txt has fortplot dependency and adjust accordingly - local CMAKE_OPTS="-DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON_INTERFACE=OFF" - - if grep -q "fortplot" CMakeLists.txt 2>/dev/null; then - echo "Detected fortplot dependency in CMakeLists.txt" - # Don't disable GVEC if fortplot is present, as they might be related - else - # For older versions without fortplot, disable GVEC - CMAKE_OPTS="$CMAKE_OPTS -DENABLE_GVEC=OFF" - fi +main() { + mkdir -p "$GOLDEN_RECORD_BASE_DIR" "$TEST_DATA_DIR" - cmake -S . -Bbuild -GNinja $CMAKE_OPTS > $PROJECT_ROOT/configure.log 2>&1 - if [ $? -ne 0 ]; then - echo "CMake configuration failed. Check $PROJECT_ROOT/configure.log" - return 1 + if [ ! -f "$PROJECT_ROOT_CUR/build/simple.x" ]; then + log "Current build missing at $PROJECT_ROOT_CUR/build/simple.x" + exit 1 fi - cmake --build build --config Release > $PROJECT_ROOT/build.log 2>&1 - if [ $? -ne 0 ]; then - echo "Build failed. Check $PROJECT_ROOT/build.log" - return 1 - fi -} + prepare_reference + local ref_sha + ref_sha=$(git -C "$PROJECT_ROOT_REF" rev-parse HEAD) + local cur_sha + cur_sha=$(git -C "$PROJECT_ROOT_CUR" rev-parse HEAD 2>/dev/null || echo "working-tree") + + log "Reference: $REF_SPEC @ $ref_sha" + log "Current: $cur_sha" + + run_suite "$PROJECT_ROOT_REF" "$RUN_DIR_REF" + run_suite "$PROJECT_ROOT_CUR" "$RUN_DIR_CUR" + + "$SCRIPT_DIR/compare_golden_results.sh" "$RUN_DIR_REF" "$RUN_DIR_CUR" +} main "$@" diff --git a/test/golden_record/golden_record_multi.sh b/test/golden_record/golden_record_multi.sh deleted file mode 100755 index 692daf26..00000000 --- a/test/golden_record/golden_record_multi.sh +++ /dev/null @@ -1,240 +0,0 @@ -#!/bin/bash -# Golden record test script that runs working copy tests once -# and compares against multiple reference versions - -CLONE_URL="https://github.com/itpplasma/SIMPLE.git" - -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -CUR_VER=$(git -C "$SCRIPT_DIR" describe --tags --always --dirty) - -# Parse arguments: reference versions and optional --expect-fail flags -REFS=() -EXPECT_FAIL=() -while [[ $# -gt 0 ]]; do - case $1 in - --expect-fail) - EXPECT_FAIL+=("true") - shift - ;; - *) - REFS+=("$1") - EXPECT_FAIL+=("false") - shift - ;; - esac -done - -if [ ${#REFS[@]} -eq 0 ]; then - echo "Usage: $0 [--expect-fail] [--expect-fail] ..." - echo "Example: $0 46f1f53 --expect-fail e25ca7e main" - exit 1 -fi - -# Allow override of directories via environment variables -GOLDEN_RECORD_BASE_DIR=${GOLDEN_RECORD_BASE_DIR:-"$(pwd)/golden_record"} -PROJECT_ROOT_CUR=$(cd "$SCRIPT_DIR/../.." && pwd) - -RUN_DIR_CUR="$GOLDEN_RECORD_BASE_DIR/runs/run_$CUR_VER" -TEST_DATA_DIR="$GOLDEN_RECORD_BASE_DIR/test_data" - -# Find test cases -if [ -d "$GOLDEN_RECORD_BASE_DIR/test_cases" ]; then - TEST_CASES="$(cd "$GOLDEN_RECORD_BASE_DIR/test_cases" && find . -name simple.in -exec dirname {} \; | sed 's|^\./||' | sort)" -else - TEST_CASES="$(cd "$SCRIPT_DIR" && find . -name simple.in -exec dirname {} \; | sed 's|^\./||' | sort)" -fi - -echo "Golden record multi-comparison test" -echo "===================================" -echo "Current version: $CUR_VER" -echo "Reference versions to compare: ${REFS[@]}" -echo "Expected failures: ${EXPECT_FAIL[@]}" -echo "" - -# Determine and check out appropriate libneo version for historical builds -setup_libneo() { - local PROJECT_ROOT="$1" - local BASE_DIR="$(dirname "$PROJECT_ROOT")" - local LIBNEO_URL="https://github.com/itpplasma/libneo.git" - - local SIMPLE_TS - SIMPLE_TS=$(git -C "$PROJECT_ROOT" log -1 --format=%ct) - - local LIBNEO_TAGS_DIR="$BASE_DIR/libneo_tags" - if [ ! -d "$LIBNEO_TAGS_DIR/.git" ]; then - git clone --quiet --no-checkout "$LIBNEO_URL" "$LIBNEO_TAGS_DIR" - else - git -C "$LIBNEO_TAGS_DIR" fetch --tags --quiet - fi - - # Use creatordate which works for both annotated and lightweight tags - local LIBNEO_TAG - LIBNEO_TAG=$(git -C "$LIBNEO_TAGS_DIR" for-each-ref --sort=-creatordate \ - --format '%(refname:short) %(creatordate:unix)' refs/tags | while read tag ts; do - # Check that timestamp is not empty and is valid - if [ -n "$ts" ] && [ "$ts" -le "$SIMPLE_TS" ] 2>/dev/null; then - echo "$tag" - break - fi - done) - - if [ -z "$LIBNEO_TAG" ]; then - echo " No suitable libneo tag found; using oldest available for compatibility" - # Use oldest tag for better compatibility with historical commits - LIBNEO_TAG=$(git -C "$LIBNEO_TAGS_DIR" for-each-ref --sort=creatordate \ - --format '%(refname:short)' refs/tags | head -n 1) - fi - - echo " Using libneo tag: $LIBNEO_TAG" - local LIBNEO_TAG_DIR="$BASE_DIR/libneo_$LIBNEO_TAG" - if [ ! -d "$LIBNEO_TAG_DIR/.git" ]; then - git clone --quiet --branch "$LIBNEO_TAG" --depth 1 "$LIBNEO_URL" "$LIBNEO_TAG_DIR" - fi - - ln -sfn "$LIBNEO_TAG_DIR" "$BASE_DIR/libneo" -} - -# Function to clone and build a reference version -prepare_reference() { - local ref_ver="$1" - local project_root="$GOLDEN_RECORD_BASE_DIR/simple_$ref_ver" - - if [ ! -f "$project_root/build/simple.x" ]; then - echo "Preparing reference version $ref_ver..." - - # Clone if needed - if [ ! -d "$project_root" ]; then - echo " Cloning..." - git clone --filter=blob:none --no-checkout "$CLONE_URL" "$project_root" - fi - - cd "$project_root" - git fetch --all --quiet - git checkout "$ref_ver" --quiet - - # Build - echo " Building..." - - # Handle old project structure with libneo - if [ -f "SRC/CMakeLists.txt" ] && grep -q "../libneo" "SRC/CMakeLists.txt" 2>/dev/null; then - setup_libneo "$project_root" - fi - - cmake -S . -Bbuild -GNinja -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON_INTERFACE=OFF -DENABLE_GVEC=OFF > configure.log 2>&1 - if [ $? -ne 0 ]; then - echo " CMake configuration failed for $ref_ver. Check $project_root/configure.log" - return 1 - fi - - cmake --build build --config Release > build.log 2>&1 - if [ $? -ne 0 ]; then - echo " Build failed for $ref_ver. Check $project_root/build.log" - return 1 - fi - - echo " Build complete for $ref_ver" - else - echo "Using existing build for $ref_ver" - fi - - return 0 -} - -# Main execution -main() { - local overall_status=0 - - # Create base directories - mkdir -p "$GOLDEN_RECORD_BASE_DIR" - mkdir -p "$TEST_DATA_DIR" - - # Check current build - if [ ! -f "$PROJECT_ROOT_CUR/build/simple.x" ]; then - echo "ERROR: Current build not found at: $PROJECT_ROOT_CUR/build/simple.x" - exit 1 - fi - - # Prepare all reference versions first - echo "Step 1: Preparing reference versions" - echo "------------------------------------" - for ref in "${REFS[@]}"; do - prepare_reference "$ref" - if [ $? -ne 0 ]; then - echo "ERROR: Failed to prepare reference version $ref" - exit 1 - fi - done - - # Run working copy tests ONCE - echo "" - echo "Step 2: Running tests on working copy" - echo "-------------------------------------" - "$SCRIPT_DIR/run_golden_tests.sh" "$PROJECT_ROOT_CUR" "$RUN_DIR_CUR" "$TEST_DATA_DIR" - if [ $? -ne 0 ]; then - echo "ERROR: Failed to run tests on working copy" - exit 1 - fi - - # Compare against each reference version - echo "" - echo "Step 3: Comparing against reference versions" - echo "--------------------------------------------" - - for i in "${!REFS[@]}"; do - ref="${REFS[$i]}" - expect_fail="${EXPECT_FAIL[$i]}" - run_dir_ref="$GOLDEN_RECORD_BASE_DIR/runs/run_$ref" - project_root_ref="$GOLDEN_RECORD_BASE_DIR/simple_$ref" - - echo "" - echo "Comparing against $ref (expect failure: $expect_fail)..." - - # Run tests for reference version - "$SCRIPT_DIR/run_golden_tests.sh" "$project_root_ref" "$run_dir_ref" "$TEST_DATA_DIR" - if [ $? -ne 0 ]; then - echo "ERROR: Failed to run tests for reference $ref" - if [ "$expect_fail" = "false" ]; then - overall_status=1 - fi - continue - fi - - # Compare results - "$SCRIPT_DIR/compare_golden_results.sh" "$run_dir_ref" "$RUN_DIR_CUR" - comparison_result=$? - - if [ $comparison_result -eq 0 ]; then - echo " ✓ Results match with $ref" - if [ "$expect_fail" = "true" ]; then - echo " WARNING: Expected this comparison to fail, but it passed!" - overall_status=1 - fi - else - echo " ✗ Results differ from $ref" - if [ "$expect_fail" = "false" ]; then - echo " ERROR: Unexpected difference!" - overall_status=1 - else - echo " (This was expected to fail)" - fi - fi - done - - # Summary - echo "" - echo "Summary" - echo "=======" - echo "Working copy version: $CUR_VER" - echo "Reference versions tested: ${REFS[@]}" - echo "Results preserved in: $GOLDEN_RECORD_BASE_DIR" - - if [ $overall_status -eq 0 ]; then - echo "Overall result: PASSED" - else - echo "Overall result: FAILED" - fi - - exit $overall_status -} - -main "$@" diff --git a/test/golden_record/meiss/simple.in b/test/golden_record/meiss/simple.in new file mode 100644 index 00000000..0bdb2d6b --- /dev/null +++ b/test/golden_record/meiss/simple.in @@ -0,0 +1,9 @@ +&config +multharm = 3 ! reduced spline order for speed +trace_time = 1d-4 ! short integration time for fast golden test +sbeg = 0.3d0 ! starting normalized toroidal flux +ntestpart = 32 ! number of test particles +netcdffile = 'wout.nc' ! VMEC equilibrium used for golden record +isw_field_type = 3 ! Meiss canonical field evaluation +deterministic = .true. ! ensure reproducible random seeds +/ diff --git a/test/tests/CMakeLists.txt b/test/tests/CMakeLists.txt index c383fa08..d10e2f86 100644 --- a/test/tests/CMakeLists.txt +++ b/test/tests/CMakeLists.txt @@ -235,15 +235,14 @@ file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/../golden_record/boozer file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/../golden_record/canonical DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/golden_record/test_cases/) -# Efficient golden record test: compare against main branch -# This runs the working copy tests only once and compares against main +# Golden record test: compare working copy against main branch (default) add_test( - NAME golden_record_multi - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/../golden_record/golden_record_multi.sh - main # Always compare with main branch + NAME golden_record + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/../golden_record/golden_record.sh + main WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) -set_tests_properties(golden_record_multi PROPERTIES +set_tests_properties(golden_record PROPERTIES TIMEOUT 600 # 10 minutes for comparison LABELS "golden_record;regression" ENVIRONMENT "GOLDEN_RECORD_BASE_DIR=${CMAKE_CURRENT_BINARY_DIR}/golden_record" @@ -259,4 +258,3 @@ add_test( set_tests_properties(golden_record_sanity PROPERTIES TIMEOUT 60 # 1 minute for sanity checks ) -