diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 400f615..2dbf68d 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -10,6 +10,12 @@ on: - cron: "0 4 * * *" # after gap-docker(-master) is rebuilt workflow_dispatch: +# cancel runs for pull requests on force push +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt }} + cancel-in-progress: true + jobs: test: strategy: @@ -19,43 +25,66 @@ jobs: runs-on: ubuntu-latest container: image: ${{ matrix.image }} + defaults: + run: + working-directory: /home/gap/.gap/pkg/ + env: + # HOME is already set in the docker container, but GitHub Actions overwrites it + HOME: /home/gap steps: - # keep workflow active even if repository has no activity for 60 days (do not execute for pull requests) - - run: '[ "$GITHUB_EVENT_NAME" = "pull_request" ] || curl --fail -X PUT -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/$GITHUB_REPOSITORY/actions/workflows/Tests.yml/enable' - - uses: actions/checkout@v1 - - run: mkdir -p /home/gap/.gap/pkg/ - - run: sudo cp -a $GITHUB_WORKSPACE /home/gap/.gap/pkg/ - - run: sudo chown -R gap:gap /home/gap/.gap/pkg/ - - run: | - export HOME="/home/gap" - cd /home/gap/.gap/pkg/ - cp ./LessGenerators/ci_gaprc /home/gap/.gap/gaprc - git clone --depth 1 https://github.com/gap-packages/AutoDoc.git - git clone --depth 1 https://github.com/homalg-project/homalg_project.git - # set SOURCE_DATE_EPOCH for reproducible PDFs - export SOURCE_DATE_EPOCH=0 - # build documentation of packages which we might want to reference, keep this in sync with `release-gap-package` - [ -d "CAP_project/CAP" ] && make -C "CAP_project/CAP" doc - [ -d "CAP_project/CompilerForCAP" ] && make -C "CAP_project/CompilerForCAP" doc - [ -d "CAP_project/MonoidalCategories" ] && make -C "CAP_project/MonoidalCategories" doc - [ -d "CAP_project/CartesianCategories" ] && make -C "CAP_project/CartesianCategories" doc - [ -d "CAP_project/FreydCategoriesForCAP" ] && make -C "CAP_project/FreydCategoriesForCAP" doc - [ -d "HigherHomologicalAlgebra/ToolsForHigherHomologicalAlgebra" ] && make -C "HigherHomologicalAlgebra/ToolsForHigherHomologicalAlgebra" doc - [ -d "homalg_project/homalg" ] && make -C "homalg_project/homalg" doc - [ -d "homalg_project/Modules" ] && make -C "homalg_project/Modules" doc - [ -d "Toposes" ] && make -C "Toposes" doc - TERM=dumb make -C LessGenerators -j $(nproc) --output-sync ci-test - cp ./LessGenerators/.codecov.yml ./ - (cd LessGenerators && LANG=C.UTF-8 python3 process_coverage_ignored_lines.py) - [ "$GITHUB_EVENT_NAME" != "schedule" ] && [ "${{ matrix.image }}" = "ghcr.io/homalg-project/gap-docker-master:latest" ] && ./LessGenerators/upload_codecov.sh + - name: Keep workflow active even if repository has no activity for 60 days + if: github.event_name != 'pull_request' + run: | + curl --fail -X PUT -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/$GITHUB_REPOSITORY/actions/workflows/Tests.yml/enable" + - name: Check out repo + uses: actions/checkout@v3 + with: + # the persisted token interferes with the subsplit token used below + persist-credentials: false + fetch-depth: 0 + - name: Move checked out repo to GAP user root dir + run: | + cp -a $GITHUB_WORKSPACE /home/gap/.gap/pkg/ + - name: Prepare environment + run: | + cp ./LessGenerators/dev/ci_gaprc /home/gap/.gap/gaprc + git clone --depth 1 -vv https://github.com/homalg-project/homalg_project.git + - name: Build documentation of packages which we might want to reference + run: | + # keep this in sync with `dev/.release` + if [ -d "CAP_project/CAP" ]; then make -C "CAP_project/CAP" doc; fi + if [ -d "CAP_project/CompilerForCAP" ]; then make -C "CAP_project/CompilerForCAP" doc; fi + if [ -d "CAP_project/MonoidalCategories" ]; then make -C "CAP_project/MonoidalCategories" doc; fi + if [ -d "CAP_project/CartesianCategories" ]; then make -C "CAP_project/CartesianCategories" doc; fi + if [ -d "CAP_project/AdditiveClosuresForCAP" ]; then make -C "CAP_project/AdditiveClosuresForCAP" doc; fi + if [ -d "CAP_project/FreydCategoriesForCAP" ]; then make -C "CAP_project/FreydCategoriesForCAP" doc; fi + if [ -d "HigherHomologicalAlgebra/ToolsForHigherHomologicalAlgebra" ]; then make -C "HigherHomologicalAlgebra/ToolsForHigherHomologicalAlgebra" doc; fi + if [ -d "homalg_project/homalg" ]; then make -C "homalg_project/homalg" doc; fi + if [ -d "homalg_project/Modules" ]; then make -C "homalg_project/Modules" doc; fi + if [ -d "CategoricalTowers/ToolsForCategoricalTowers" ]; then make -C "CategoricalTowers/ToolsForCategoricalTowers" doc; fi + if [ -d "CategoricalTowers/Toposes" ]; then make -C "CategoricalTowers/Toposes" doc; fi + - name: Test LessGenerators + run: | + make -C LessGenerators --trace -j $(nproc) --output-sync ci-test + - name: Release package or simulate release + run: | + cd LessGenerators + python3 dev/process_coverage.py git config --global user.name "Bot" git config --global user.email "empty" - cd LessGenerators CUR_SHA=$(git rev-parse --verify HEAD) + git fetch origin gh-pages + git worktree add gh-pages/ gh-pages || (echo "There was an error. Make sure there is a branch named 'gh-pages'. See https://github.com/homalg-project/PackageJanitor#error-there-was-an-error-make-sure-there-is-a-branch-named-gh-pages"; exit 1) if [ "${{ matrix.image }}" = "ghcr.io/homalg-project/gap-docker:latest" ] && [ "$CUR_SHA" = "$(git rev-parse origin/master)" ] && [ $(dirname "$GITHUB_REPOSITORY") = "homalg-project" ]; then \ - git worktree add gh-pages/ gh-pages || (echo "There was an error. Make sure there is a branch named 'gh-pages'. See https://github.com/homalg-project/PackageJanitor#error-there-was-an-error-make-sure-there-is-a-branch-named-gh-pages"; exit 1); \ git checkout master; \ - LANG=C.UTF-8 ./make_dist.sh --token "${{ secrets.GITHUB_TOKEN }}"; \ + TOKEN="${{ secrets.GITHUB_TOKEN }}" SUBSPLIT_PUSH_SECRET="${{ secrets.SUBSPLIT_PUSH_SECRET }}" ./dev/make_dist.sh; \ else \ - echo "Not making a release."; \ + TOKEN="${{ secrets.GITHUB_TOKEN }}" ./dev/simulate_dist.sh; \ fi + - name: Upload code coverage + if: github.event_name != 'schedule' && matrix.image == 'ghcr.io/homalg-project/gap-docker:latest' + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + run: | + cd LessGenerators + ./dev/upload_codecov.sh diff --git a/PackageInfo.g b/PackageInfo.g index 1c85b72..eb7cd2f 100644 --- a/PackageInfo.g +++ b/PackageInfo.g @@ -10,10 +10,10 @@ SetPackageInfo( rec( PackageName := "LessGenerators", Subtitle := "Find smaller generating sets for modules", -Version := "2022.07-01", +Version := "2025.12-01", Date := ~.Version{[ 1 .. 10 ]}, -Date := Concatenation( "01/", ~.Version{[ 6, 7 ]}, "/", ~.Version{[ 1 .. 4 ]} ), +Date := (function ( ) if IsBound( GAPInfo.SystemEnvironment.GAP_PKG_RELEASE_DATE ) then return GAPInfo.SystemEnvironment.GAP_PKG_RELEASE_DATE; else return Concatenation( ~.Version{[ 1 .. 4 ]}, "-", ~.Version{[ 6, 7 ]}, "-01" ); fi; end)( ), License := "GPL-2.0-or-later", Persons := [ @@ -100,7 +100,7 @@ PackageDoc := rec( ), Dependencies := rec( - GAP := ">= 4.11.1", + GAP := ">= 4.13.0", NeededOtherPackages := [ [ "AutoDoc", ">= 2013.08.07" ], [ "Modules", ">= 2017.06.19" ], diff --git a/README.md b/README.md index d6762d6..c362c7e 100644 --- a/README.md +++ b/README.md @@ -31,11 +31,11 @@ To obtain current versions of all dependencies, `git clone` (or `git pull` to up [date-img]: https://img.shields.io/endpoint?url=https://homalg-project.github.io/LessGenerators/badge_date.json&label=🔗%20released%20on&color=yellow [date-url]: https://homalg-project.github.io/LessGenerators/view_release.html -[tests-img]: https://github.com/homalg-project/LessGenerators/workflows/Tests/badge.svg?branch=master -[tests-url]: https://github.com/homalg-project/LessGenerators/actions?query=workflow%3ATests+branch%3Amaster +[tests-img]: https://github.com/homalg-project/LessGenerators/actions/workflows/Tests.yml/badge.svg?branch=master +[tests-url]: https://github.com/homalg-project/LessGenerators/actions/workflows/Tests.yml?query=branch%3Amaster [codecov-img]: https://codecov.io/gh/homalg-project/LessGenerators/branch/master/graph/badge.svg -[codecov-url]: https://codecov.io/gh/homalg-project/LessGenerators +[codecov-url]: https://app.codecov.io/gh/homalg-project/LessGenerators [code-img]: https://img.shields.io/badge/-View%20code-blue?logo=github [code-url]: https://github.com/homalg-project/LessGenerators#top diff --git a/ci_gaprc b/ci_gaprc deleted file mode 100644 index 88fd324..0000000 --- a/ci_gaprc +++ /dev/null @@ -1,12 +0,0 @@ -# Prefer GAPInfo.UserGapRoot over other package directories to prevent accidentally testing package -# versions distributed with GAP. -(function() - local name, package_info, pos; - for name in RecNames( GAPInfo.PackagesInfo ) do - package_info := GAPInfo.PackagesInfo.(name); - pos := PositionProperty( package_info, info -> StartsWith( info.InstallationPath, GAPInfo.UserGapRoot ) ); - if pos <> fail then - SetPackagePath( name, package_info[pos].InstallationPath ); - fi; - od; -end)(); diff --git a/dev/.release b/dev/.release new file mode 100644 index 0000000..6dca44c --- /dev/null +++ b/dev/.release @@ -0,0 +1,35 @@ +#!/bin/bash +# This script is intended to be run by the release-gap-package script which is +# part of the the ReleaseTools for GAP: +# +# https://github.com/gap-system/ReleaseTools + +set -e + +# adjust date +# Note that we cannot use sed's `-i` option for in-place editing, as +# that is a non-portable extension of POSIX, which works differently in +# BSD and GNU make. +sed "s;Date := .*;Date := \"$(date -I)\",;" PackageInfo.g > PackageInfo.g.bak +mv PackageInfo.g.bak PackageInfo.g + +# replace links to packages which are possibly referenced in the documentation, keep this in sync with `Tests.yml.j2` +for package in CAP_project/CAP CAP_project/CompilerForCAP CAP_project/MonoidalCategories CAP_project/CartesianCategories CAP_project/AdditiveClosuresForCAP CAP_project/FreydCategoriesForCAP HigherHomologicalAlgebra/ToolsForHigherHomologicalAlgebra homalg_project/homalg homalg_project/Modules CategoricalTowers/ToolsForCategoricalTowers CategoricalTowers/Toposes; do + + # adjust links to other manuals + # Note that we cannot use sed's `-i` option for in-place editing, as + # that is a non-portable extension of POSIX, which works differently in + # BSD and GNU make. + for f in ./*/*.htm* ; do + sed "s;href=\"/home/gap/.gap/pkg/$package/doc/;href=\"https://homalg-project.github.io/$package/doc/;g" "$f" > "$f.bak" + mv "$f.bak" "$f" + done + +done + +if [ "$PDFFiles" != "doc/manual.pdf" ] ; then + error "PackageDoc set to $PDFFiles but expected doc/manual.pdf" +fi + +# remove dev directory +rm -rf dev diff --git a/dev/ci_gaprc b/dev/ci_gaprc new file mode 100644 index 0000000..3b423e7 --- /dev/null +++ b/dev/ci_gaprc @@ -0,0 +1,29 @@ +SetUserPreference( "ShortBanners", true ); + +# Prefer GAPInfo.UserGapRoot over other package directories to prevent accidentally testing package +# versions distributed with GAP. + +FORCE_LOADING_FROM_USER_GAP_ROOT := function ( ) + local name, package_info, pos; + for name in RecNames( GAPInfo.PackagesInfo ) do + package_info := GAPInfo.PackagesInfo.(name); + pos := PositionProperty( package_info, info -> StartsWith( info.InstallationPath, GAPInfo.UserGapRoot ) ); + if pos <> fail then + SetPackagePath( name, package_info[pos].InstallationPath ); + fi; + od; +end; + +FORCE_LOADING_FROM_USER_GAP_ROOT( ); + +MakeReadWriteGlobal( "InitializePackagesInfoRecords" ); +FORCE_LOADING_FROM_USER_GAP_ROOT_InitializePackagesInfoRecords_orig := InitializePackagesInfoRecords; +InitializePackagesInfoRecords := function ( args... ) + local old_PackagesInfo; + old_PackagesInfo := GAPInfo.PackagesInfo; + CallFuncList( FORCE_LOADING_FROM_USER_GAP_ROOT_InitializePackagesInfoRecords_orig, args ); + if not IsIdenticalObj( old_PackagesInfo, GAPInfo.PackagesInfo ) then + FORCE_LOADING_FROM_USER_GAP_ROOT( ); + fi; +end; +MakeReadOnlyGlobal( "InitializePackagesInfoRecords" ); diff --git a/.codecov.yml b/dev/codecov.yml similarity index 50% rename from .codecov.yml rename to dev/codecov.yml index 00729d5..658e7f0 100644 --- a/.codecov.yml +++ b/dev/codecov.yml @@ -1,7 +1,3 @@ codecov: disable_default_path_fixes: true require_ci_to_pass: false -fixes: - - "/home/gap/.gap/pkg/LessGenerators/::" -ignore: - - "home/" diff --git a/dev/make_dist.sh b/dev/make_dist.sh new file mode 100755 index 0000000..6c2bf9d --- /dev/null +++ b/dev/make_dist.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +GAP_PKG_RELEASE_DATE=$(date -I) ./dev/release-gap-package --skip-existing-release --release-script dev/.release diff --git a/process_coverage_ignored_lines.py b/dev/process_coverage.py similarity index 80% rename from process_coverage_ignored_lines.py rename to dev/process_coverage.py index aaf7598..7635a75 100755 --- a/process_coverage_ignored_lines.py +++ b/dev/process_coverage.py @@ -7,6 +7,12 @@ # c) calls of `TryNextMethod` leading to external code. ####################################################### +# This script applies three rules to all coverage*.json files below the current working directory: +# a) Remove all coverage information of files not below the current working directory +# (this technically includes "tst/testall.g" because it is not referenced by an absolute path). +# b) Strip the current working directory from the paths of files below the current working directory. +# c) Ignore lines with one of the "COVERAGE_IGNORE_..." annotations (see below). + import json import re import os @@ -17,14 +23,18 @@ regex_next_line = re.compile("^\s*# COVERAGE_IGNORE_NEXT_LINE$") regex_return_void = re.compile("^\s*return;$") -for coverage_filename in Path(".").glob("**/coverage*.json"): +for coverage_filename in Path("../").glob("**/coverage*.json"): print("processing coverage file " + str(coverage_filename)) + new_data = {} + new_data["coverage"] = {} with open(coverage_filename) as json_file: data = json.load(json_file) files = data["coverage"] for filename, lines_covered in files.items(): - if filename.startswith(os.getcwd()): + if filename.startswith(os.getcwd() + "/") and not filename.endswith("manual.six"): print(" processing code file " + filename) + # ignored lines will be delete from lines_covered in-place + new_data["coverage"][filename.replace(os.getcwd() + "/", "", 1)] = lines_covered line_number = 0 ignoring = False ignored_lines = [] @@ -77,4 +87,4 @@ del lines_covered[str(line_number)] with open(coverage_filename, "w") as outfile: - json.dump(data, outfile, indent=0) + json.dump(new_data, outfile, indent=0) diff --git a/release-gap-package b/dev/release-gap-package similarity index 73% rename from release-gap-package rename to dev/release-gap-package index 1f318bd..1350c4a 100755 --- a/release-gap-package +++ b/dev/release-gap-package @@ -50,6 +50,7 @@ Paths --tmpdir path of temporary directory [Default: tmp subdirectory of srcdir] --webdir path of web directory [Default: gh-pages subdirectory of srcdir] --update-script path of the update script [Default: update.g in webdir] + --release-script path of the release script [Default: .release in srcdir (must be checked into git)] Custom settings --token GitHub access token @@ -71,20 +72,37 @@ EOF # # Various little helper functions +if test -t 1; then + + # see if it supports colors... + ncolors=$(tput colors) + + if test -n "$ncolors" && test $ncolors -ge 8; then + bold="$(tput bold)" + underline="$(tput smul)" + standout="$(tput smso)" + normal="$(tput sgr0)" + black="$(tput setaf 0)" + red="$(tput setaf 1)" + green="$(tput setaf 2)" + yellow="$(tput setaf 3)" + blue="$(tput setaf 4)" + magenta="$(tput setaf 5)" + cyan="$(tput setaf 6)" + white="$(tput setaf 7)" + fi +fi -# print notices in green notice() { - printf '\033[32m%s\033[0m\n' "$*" + printf "${green}%s${normal}\n" "$*" } -# print warnings in yellow warning() { - printf '\033[33mWARNING: %s\033[0m\n' "$*" + printf "${yellow}WARNING: %s${normal}\n" "$*" } -# print error in red and exit error() { - printf '\033[31mERROR: %s\033[0m\n' "$*" + printf "${red}ERROR: %s${normal}\n" "$*" 1>&2 exit 1 } @@ -101,13 +119,14 @@ run_gap() { # for people whose gap script is broken (by not honoring -A, thus loading # Browse, which inserts that extra character). I am looking at you, # Sebastian! + mkdir -p "$TMP_DIR" gap_output=$( \ (echo 'OnBreak:=function() Print("FATAL ERROR"); FORCE_QUIT_GAP(1); end;;' ; cat - ; echo ; echo "FORCE_QUIT_GAP(0);") \ | $GAP -A -q -b 2>&1 \ - | tr -d '\r' ) - mkdir -p "$TMP_DIR" - echo "$gap_output" > "$TMP_DIR/gap-error.log" - if echo "$gap_output" | grep -q '\(Error\|FATAL ERROR\|Syntax \)' ; then + | tr -d '\r' \ + | tee "$TMP_DIR/gap-error.log" ) + if grep -q '\(Error\|FATAL ERROR\|Syntax \)' "$TMP_DIR/gap-error.log" ; then + cat "$TMP_DIR/gap-error.log" error "there was an error running GAP, see $TMP_DIR/gap-error.log" fi } @@ -117,13 +136,16 @@ run_gap() { # we may end up parsing the same JSON data two times, but that # doesn't really matter as it is tiny. json_get_key() { - echo "$response" | python -c 'import json,sys;obj=json.load(sys.stdin);print(obj.get("'"$1"'",""))' + echo "$response" | $PYTHON -c 'import json,sys;obj=json.load(sys.stdin);print(obj.get("'"$1"'",""))' } # On Mac OS X, tar stores extended attributes in ._FOO files inside archives. # Setting COPYFILE_DISABLE prevents that. See export COPYFILE_DISABLE=1 +# set SOURCE_DATE_EPOCH for reproducible PDFs +export SOURCE_DATE_EPOCH=0 + ###################################################################### # @@ -147,11 +169,13 @@ while [ x"$1" != x ]; do --webdir ) WEB_DIR="$1"; shift ;; --tmpdir ) TMP_DIR="$1"; shift ;; --update-script ) UPDATE_SCRIPT="$1"; shift ;; + --release-script ) RELEASE_SCRIPT="$1"; shift ;; --srcdir=*) SRC_DIR=${option#--srcdir=}; shift ;; --webdir=*) WEB_DIR=${option#--webdir=}; shift ;; --tmpdir=*) TMP_DIR=${option#--tmpdir=}; shift ;; --update-script=*) UPDATE_SCRIPT=${option#--update-script=}; shift ;; + --release-script=*) RELEASE_SCRIPT=${option#--release-script=}; shift ;; --token ) TOKEN="$1"; shift ;; @@ -187,7 +211,7 @@ if [ "x$WEB_DIR" = x ] ; then WEB_DIR="$SRC_DIR/gh-pages" fi if [ ! -d "$WEB_DIR" ] ; then - error "could not find 'webdir' with clone of your gh-pages branch" + error "could not find webdir \"${WEB_DIR}\" with clone of your gh-pages branch" fi # Check for presence of the update script @@ -198,6 +222,11 @@ if [ ! -f "$UPDATE_SCRIPT" ] ; then error "could not find update script \"${UPDATE_SCRIPT}\"" fi +# Check for presence of the release script (if given) +if [ "x$RELEASE_SCRIPT" != x ] && [ ! -f "$RELEASE_SCRIPT" ] ; then + error "could not find release script \"${RELEASE_SCRIPT}\"" +fi + # Check whether GAP is usable GAP=${GAP:-gap} command -v "$GAP" >/dev/null 2>&1 || @@ -209,8 +238,20 @@ command -v curl >/dev/null 2>&1 || command -v git >/dev/null 2>&1 || error "the 'git' command was not found, please install it" -command -v python >/dev/null 2>&1 || - error "the 'python' command was not found, please install it" +if [ "x$PYTHON" != x ] ; then + if ! command -v "$PYTHON" >/dev/null 2>&1; then + error "Could not execute python from PYTHON environment variable" + fi +elif command -v python >/dev/null 2>&1; then + PYTHON=python +elif command -v python3 >/dev/null 2>&1; then + PYTHON=python3 +elif command -v python2 >/dev/null 2>&1; then + PYTHON=python2 +else + error "Python not found, please install it and/or set the PYTHON environment variable to the path of a python executable" +fi +notice "Using PYTHON = '$PYTHON'" verify_git_clean @@ -245,7 +286,10 @@ else tmp := GAPInfo.PackageInfoCurrent.ArchiveURL; Print("GAP_ERROR=\"The ArchiveURL has unexpected value '",tmp,"'\"\n"); fi; -Print("PDFFile=\"",GAPInfo.PackageInfoCurrent.PackageDoc.PDFFile,"\"\n"); +docs:=GAPInfo.PackageInfoCurrent.PackageDoc;; +if not IsList(docs) then docs:=[docs];fi; +pdffiles:=JoinStringsWithSeparator(List(docs,x->x.PDFFile)," ");; +Print("PDFFiles=\"",pdffiles,"\"\n"); EOF # evaluate the output of GAP, which should be valid shell script code @@ -268,11 +312,14 @@ esac if [ -f makedoc.g ] ; then notice "Building GAP package documentation (using makedoc.g)" run_gap </dev/null 2>&1 || (cat makedoc.log; error "non resolved reference(s) found in the manual") + rm -f makedoc.log elif [ -f doc/make_doc ] ; then notice "Building GAP package documentation (using doc/make_doc)" cd doc && ./make_doc && cd .. @@ -316,14 +363,24 @@ fi # # Determine GitHub repository and username, and the current branch # +if [ $ONLY_TARBALL = no ] ; then notice "Using GitHub repository $REPO" -GITHUB_USER=$(dirname "$REPO") +if [ "x$GITHUB_USER" = x ] ; then + GITHUB_USER=$(git config --get github.user || echo) +fi +if [ "x$GITHUB_USER" = x ] ; then + GITHUB_USER=$(dirname "$REPO") +fi notice "Using GitHub username $GITHUB_USER" -BRANCH=$(git symbolic-ref -q --short HEAD) +BRANCH=$(git symbolic-ref -q --short HEAD || echo) +if [ "x$BRANCH" = x ] ; then + notice "no branch! Cannot proceed, exiting." + exit 1 +fi notice "Using branch $BRANCH" - +fi ###################################################################### # @@ -340,7 +397,7 @@ UPLOAD_URL=https://uploads.github.com/repos/$REPO/releases verify_git_clean if git show-ref -q "$TAG" ; then - notice "Using git tag $TAG (derived from ArchiveURL in in PackageInfo.g)" + notice "Using git tag $TAG (derived from ArchiveURL in PackageInfo.g)" else notice "Creating git tag $TAG" git tag "$TAG" @@ -358,7 +415,9 @@ if [ "$MESSAGE" = "Not Found" ] ; then MESSAGE= # release does not yet exist -> that's how we like it elif [ x"$RELEASE_ID" != x ] ; then # release already exists -> skip, error out or delete it - if [ "x$SKIP_EXISTING_RELEASE" = xyes ] ; then + if [ "x$ONLY_TARBALL" = xyes ] ; then + warning "release $TAG already exists on GitHub, but only tarballs were requested, so continue anyway" + elif [ "x$SKIP_EXISTING_RELEASE" = xyes ] ; then notice "release $TAG already exists on GitHub, skipping release" exit 0 elif [ "x$FORCE" = xyes ] ; then @@ -378,17 +437,23 @@ fi # # Validate the tag # -HEAD_REF=$(git rev-parse --verify HEAD) -TAG_REF=$(git rev-parse --verify "$TAG^{}") +if [ "x$ONLY_TARBALL" = xno ] ; then + HEAD_REF=$(git rev-parse --verify HEAD) + TAG_REF=$(git rev-parse --verify "$TAG^{}") + + if [ "x$TAG_REF" != "x$HEAD_REF" ] ; then + if [ "x$FORCE" = xyes ] ; then + notice "Force update of git tag $TAG" + git tag -f "$TAG" + else + error "tag $TAG is not the HEAD commit -- did you tag the right commit?" + fi + fi -if [ "x$TAG_REF" != "x$HEAD_REF" ] ; then - error "tag $TAG is not the HEAD commit -- did you tag the right commit?" + echo "" fi -echo "" - - ###################################################################### # # Get fresh (unmodified) copies of the files, and generate some stuff @@ -403,79 +468,69 @@ rm -rf "${TMP_DIR:?}/$BASENAME"* umask 0022 notice "Exporting repository content for tag '$TAG'" -git archive --prefix="$BASENAME/" "$TAG" . | tar xf - -C "$TMP_DIR" +git archive --prefix="$BASENAME/" HEAD . | tar xf - -C "$TMP_DIR" # Build the package documentation, run autoconf, etc. cd "$TMP_DIR/$BASENAME" -# adjust date -# Note that we cannot use sed's `-i` option for in-place editing, as -# that is a non-portable extension of POSIX, which works differently in -# BSD and GNU make. -sed "s;Date := .*;Date := \"$(date +%d/%m/%Y)\",;" PackageInfo.g > PackageInfo.g.bak -mv PackageInfo.g.bak PackageInfo.g - -notice "Removing unnecessary files" -# Remove recursively in case there is a .github directory -rm -rf .git* .hg* .cvs* .circleci -rm -f .appveyor.yml .codecov.yml .travis.yml - -# execute .release script, if present -if [ -f .release ] ; then - # the .release script can perform additional preparation, e.g.: - # * add files for distribution which are not part of the repository; - # * remove further files not intended for distribution (e.g. scripts/ directory); - # * build the package manual in a custom way; - # * perform additional sanity checks; - # * ... - . ./.release - rm -f .release -fi - -if [ -x autogen.sh ] ; then - notice "Generating build system files" - sh autogen.sh - rm -rf autom4te.cache -fi - if [ -f makedoc.g ] ; then notice "Building GAP package documentation for archives (using makedoc.g)" run_gap </dev/null 2>&1 || (cat makedoc.log; error "non resolved reference(s) found in the manual") + rm -f makedoc.log rm -f doc/*.tex - rm -f doc/*.aux doc/*.bbl doc/*.blg doc/*.brf doc/*.idx doc/*.ilg doc/*.ind doc/*.log doc/*.out doc/*.pnr doc/*.tst elif [ -f doc/make_doc ] ; then notice "Copying GAP package documentation for archives (using doc/make_doc)" cp -r "$SRC_DIR/doc" . - cp -r "$SRC_DIR/htm" . - rm -f doc/*.aux doc/*.bbl doc/*.blg doc/*.brf doc/*.idx doc/*.ilg doc/*.ind doc/*.log doc/*.out doc/*.pnr doc/*.tst + test -d "$SRC_DIR/htm" && cp -r "$SRC_DIR/htm" . # for packages with a single manual + test -d "$SRC_DIR/doc/htm" && cp -r "$SRC_DIR/doc/htm" doc/ # for packages with multiple manual +fi + +notice "Removing unnecessary files" +# Remove recursively in case there is a .github directory +rm -rf .git* .hg* .cvs* .circleci +rm -f .mailmap .clang-format +rm -f .appveyor.yml .codecov.yml .travis.yml azure-pipelines.yml +# remove TeX leftovers +find doc -name '*.aux' -o -name '*.bbl' -o -name '*.blg' -o -name '*.brf' -o -name '*.idx' -o -name '*.ilg' -o -name '*.ind' -o -name '*.log' -o -name '*.out' -o -name '*.pnr' -o -name '*.toc' -o -name '*.tst' | xargs rm -f +# remove macOS auxiliary files +find . -name .DS_Store -exec rm -f {} + + +# execute release script, if present +# the release script can perform additional preparation, e.g.: +# * add files for distribution which are not part of the repository; +# * remove further files not intended for distribution (e.g. scripts/ directory); +# * build the package manual in a custom way; +# * perform additional sanity checks; +# * ... +if [ "x$RELEASE_SCRIPT" != x ] ; then + . "$RELEASE_SCRIPT" + # do not delete the custom release script since it might be reused, for example by additional packages of a monorepo + # if it should be deleted, it can always simply delete itself +elif [ -f .release ] ; then + . ./.release + rm -f .release +fi + +if [ -x autogen.sh ] ; then + notice "Generating build system files" + sh autogen.sh + rm -rf autom4te.cache fi # make sure every file is readable chmod -R a+r . -# replace links to packages which are possibly referenced in the documentation, keep this in sync with `Tests.yml.j2` -for package in CAP_project/CAP CAP_project/CompilerForCAP CAP_project/MonoidalCategories CAP_project/CartesianCategories CAP_project/FreydCategoriesForCAP HigherHomologicalAlgebra/ToolsForHigherHomologicalAlgebra homalg_project/homalg homalg_project/Modules Toposes; do - - # adjust links to other manuals - # Note that we cannot use sed's `-i` option for in-place editing, as - # that is a non-portable extension of POSIX, which works differently in - # BSD and GNU make. - for f in ./*/*.htm* ; do - sed "s;href=\"/home/gap/.gap/pkg/$package/doc/;href=\"https://homalg-project.github.io/$package/doc/;g" "$f" > "$f.bak" - mv "$f.bak" "$f" - done - -done - # basic sanity check -fgrep -q -r ' "$f.bak" + sed \ + -e 's;href="../../../doc/;href="https://docs.gap-system.org/doc/;g' \ + -e 's;href="../../../pkg/GAPDoc[^\/]*/doc/;href="http://www.math.rwth-aachen.de/~Frank.Luebeck/GAPDoc/doc/;g' \ + -e 's;href="../../../pkg/gapdoc[^\/]*/doc/;href="http://www.math.rwth-aachen.de/~Frank.Luebeck/GAPDoc/doc/;g' \ + "$f" > "$f.bak" mv "$f.bak" "$f" done diff --git a/dev/simulate_dist.sh b/dev/simulate_dist.sh new file mode 100755 index 0000000..95ef51f --- /dev/null +++ b/dev/simulate_dist.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +GAP_PKG_RELEASE_DATE=$(date -I) ./dev/release-gap-package --release-script dev/.release --only-tarball diff --git a/upload_codecov.sh b/dev/upload_codecov.sh similarity index 70% rename from upload_codecov.sh rename to dev/upload_codecov.sh index 77b485c..ff3812c 100755 --- a/upload_codecov.sh +++ b/dev/upload_codecov.sh @@ -21,6 +21,16 @@ curl -O https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig gpgv codecov.SHA256SUM.sig codecov.SHA256SUM shasum -a 256 -c codecov.SHA256SUM +# read the token +if [ -z "$CODECOV_TOKEN" ]; then + echo -e "\033[0;33mCODECOV_TOKEN is not set. Proceeding without token.\033[0m" +else + echo -e "\033[0;32mUsing CODECOV_TOKEN from environment variable.\033[0m" +fi + # execute chmod +x codecov -./codecov -Z || ./codecov -Z || ./codecov -Z +while ! ./codecov -Z -v -s ../ -t $CODECOV_TOKEN; do + echo "Codecov upload failed, retrying in 60s" + sleep 60 +done diff --git a/make_dist.sh b/make_dist.sh deleted file mode 100755 index 00431c7..0000000 --- a/make_dist.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -e - -./release-gap-package --skip-existing-release $@ diff --git a/makedoc.g b/makedoc.g index 3c3a383..3551fe9 100644 --- a/makedoc.g +++ b/makedoc.g @@ -3,9 +3,9 @@ # # This file is a script which compiles the package manual. # -if fail = LoadPackage( "AutoDoc", "2019.05.20" ) then +if fail = LoadPackage( "AutoDoc", "2025.12.19" ) then - Error( "AutoDoc version 2019.05.20 or newer is required." ); + Error( "AutoDoc version 2025.12.19 or newer is required." ); fi; @@ -25,7 +25,7 @@ AutoDoc( rec( ), ), scaffold := rec( - entities := [ "homalg", "CAP" ], + entities := rec( homalg := "homalg", CAP := "CAP" ), ), ) ); diff --git a/makedoc_with_overfull_hbox_warnings.g b/makedoc_with_overfull_hbox_warnings.g index 68d4a43..f923e90 100644 --- a/makedoc_with_overfull_hbox_warnings.g +++ b/makedoc_with_overfull_hbox_warnings.g @@ -3,9 +3,9 @@ # # This file is a script which compiles the package manual and prints overfull hbox warnings. # -if fail = LoadPackage( "AutoDoc", "2019.05.20" ) then +if fail = LoadPackage( "AutoDoc", "2025.12.19" ) then - Error( "AutoDoc version 2019.05.20 or newer is required." ); + Error( "AutoDoc version 2025.12.19 or newer is required." ); fi; @@ -36,7 +36,7 @@ AutoDoc( rec( ), ), scaffold := rec( - entities := [ "homalg", "CAP" ], + entities := rec( homalg := "homalg", CAP := "CAP" ), ), ) ); diff --git a/makefile b/makefile index 8dc8108..20e45a8 100644 --- a/makefile +++ b/makefile @@ -8,7 +8,7 @@ doc: doc/manual.six doc/manual.six: makedoc.g \ PackageInfo.g \ $(wildcard doc/*.autodoc gap/*.gd gap/*.gi examples/*.g examples/*/*.g) - gap makedoc.g + gap --quitonbreak makedoc.g clean: (cd doc ; ./clean) @@ -17,10 +17,10 @@ test: doc gap tst/testall.g test-basic-spacing: - grep -RPl "\t" examples/ gap/ && echo "Tabs found" && exit 1 || exit 0 - grep -RPl "\r" examples/ gap/ && echo "Windows line-endings found" && exit 1 || exit 0 - # the second grep is a hack to fix the exit code with -L for grep <= 3.1 - grep -RPzL "\n\z" examples/ gap/ | grep "" && echo "File with no newline at end of file found" && exit 1 || exit 0 + # exit code 1 means no match, which is what we want here (exit code 2 signals an error) + grep -RPl "\t" examples/ gap/; test $$? -eq 1 || (echo "Tabs found" && exit 1) + grep -RPl "\r" examples/ gap/; test $$? -eq 1 || (echo "Windows line-endings found" && exit 1) + grep -RPzL "\n\z" examples/ gap/ | grep ""; test $$? -eq 1 || (echo "File with no newline at end of file found" && exit 1) test-doc: doc cp -aT doc/ doc_tmp/ @@ -29,15 +29,15 @@ test-doc: doc test-with-coverage: doc gap --quitonbreak --cover stats tst/testall.g - echo 'LoadPackage("profiling"); OutputJsonCoverage("stats", "coverage.json");' | gap --quitonbreak + gap --quitonbreak --norepl -c 'LoadPackage("profiling"); OutputJsonCoverage("stats", "coverage.json");' test-spacing: - grep -R "[^ [\"] " gap/*.gi && echo "Duplicate spaces found" && exit 1 || exit 0 - grep -RE '[^ ] +$$' gap/* && echo "Trailing whitespace found" && exit 1 || exit 0 + # exit code 1 means no match, which is what we want here (exit code 2 signals an error) + grep -R "[^ [\"] " gap/*.gi; test $$? -eq 1 || (echo "Duplicate spaces found" && exit 1) + grep -RE '[^ ] +$$' gap/*; test $$? -eq 1 || (echo "Trailing whitespace found" && exit 1) for filename in gap/*; do \ echo $$filename; \ - echo "LoadPackage(\"LessGenerators\"); SizeScreen([4096]); func := ReadAsFunction(\"$$filename\"); FileString(\"gap_spacing\", DisplayString(func));" | gap --quitonbreak --banner; \ - echo -e "\033[0m"; \ + gap --quitonbreak --norepl --banner -c "LoadPackage(\"LessGenerators\"); SizeScreen([4096]); func := ReadAsFunction(\"$$filename\"); FileString(\"gap_spacing\", DisplayString(func));"; \ # In a perfect world, the DisplayString of a function would exactly match our code. However, our line breaks and indentation might differ from the GAP ones, \ # so we remove all indentation, line breaks, and empty lines, and afterwards insert line breaks at semicolons again for better readability. \ cat "gap_spacing" | tail -n +2 | head -n -2 | sed 's/\[ \]/[ ]/g' | sed 's/( )/( )/g' | sed 's/( :/( :/g' | sed 's/ *$$//' | sed 's/^ *//' | grep -v "^$$" | tr "\n" " " | sed 's/;/;\n/g' > modified_gap_spacing; \ diff --git a/tst/testall.g b/tst/testall.g index e27c10d..7d94ae7 100644 --- a/tst/testall.g +++ b/tst/testall.g @@ -11,6 +11,25 @@ options := rec( ), ); +# reverse RecNames 50% of the time to detect code relying on the order of RecNames +if Random( RandomSource( IsMersenneTwister, NanosecondsSinceEpoch( ) ), [ false, true ] ) then + + Display( "Executing with reversed RecNames" ); + + MakeReadWriteGlobal( "RecNames" ); + + old_RecNames := RecNames; + + RecNames := record -> Reversed( old_RecNames( record ) ); + + MakeReadOnlyGlobal( "RecNames" ); + +else + + Display( "Executing with non-reversed RecNames" ); + +fi; + TestDirectory( DirectoriesPackageLibrary( "LessGenerators", "tst" ), options ); FORCE_QUIT_GAP( 1 ); # if we ever get here, there was an error