From 78619cac13e1afc79af6edb4abb7d0d6c7140b2a Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 15 Dec 2025 19:18:58 -0500 Subject: [PATCH 1/4] build: default to `--lfs` and remove `--xrootd` - make `--lfs` the default - remove `--xrootd` since no longer available - add `--clasweb` to support `wget`/`curl` for field maps - refactored a bit, replacing `if $useLfs` etc. with `case $dataRetrieval in` --- build-coatjava.sh | 239 ++++++++++++++++++++++++---------------------- 1 file changed, 126 insertions(+), 113 deletions(-) diff --git a/build-coatjava.sh b/build-coatjava.sh index 954f462ef3..a622128ef7 100755 --- a/build-coatjava.sh +++ b/build-coatjava.sh @@ -5,6 +5,24 @@ set -e set -u set -o pipefail +################################################################################ +# default options +################################################################################ + +cleanBuild=false +anaDepends=false +runSpotBugs=false +downloadMaps=true +downloadNets=true +runUnitTests=false +dataRetrieval=lfs +installClara=false +downloadData=false + +################################################################################ +# usage +################################################################################ + usage='''build-coatjava.sh [OPTIONS]... GENERAL OPTIONS @@ -15,11 +33,11 @@ GENERAL OPTIONS --help show this message DATA RETRIEVAL OPTIONS - How to retrieve magnetic field maps, neural network models, etc.; - choose only one, e.g., if the automated default choice fails: + How to retrieve magnetic field maps, neural network models, etc. + Choose only one; default is `--'$dataRetrieval'` --lfs use Git Large File Storage (requires `git-lfs`) --cvmfs use CernVM-FS (requires `/cvfms`) - --xrootd use XRootD (requires `xrootd`) + --clasweb use clasweb (only works for field maps) Options to disable data retrieval: --nomaps do not download/overwrite field maps --nonets do not download/overwrite neural networks @@ -40,17 +58,6 @@ MAVEN OPTIONS # parse arguments ################################################################################ -cleanBuild=false -anaDepends=false -runSpotBugs=false -downloadMaps=true -downloadNets=true -runUnitTests=false -useXrootd=false -useCvmfs=false -useLfs=false -installClara=false -downloadData=false mvnArgs=() wgetArgs=() for xx in $@ @@ -71,11 +78,11 @@ do mvnArgs+=(--no-transfer-progress) wgetArgs+=(--no-verbose) ;; - --xrootd) useXrootd=true ;; - --cvmfs) useCvmfs=true ;; - --lfs) useLfs=true ;; - --clara) installClara=true ;; - --data) downloadData=true ;; + --cvmfs) dataRetrieval=cvmfs ;; + --lfs) dataRetrieval=lfs ;; + --clasweb) dataRetrieval=clasweb ;; + --clara) installClara=true ;; + --data) downloadData=true ;; -h|--help) echo "$usage" exit 2 @@ -84,41 +91,6 @@ do esac done -# check if a command exists -command_exists () { - type "$1" &> /dev/null -} - -# count how many data-retrieval options are set -count_download_opts() { - local n=0 - for o in $useLfs $useCvmfs $useXrootd; do - $o && ((n++)) - done - echo $n -} - -# if the user did not choose a data retrieval method, choose a reasonable one -if [[ $(count_download_opts) -eq 0 ]]; then - echo 'INFO: no data-retrieval option set; choosing a default...' - if ! [[ $(hostname) == *.jlab.org ]] && command_exists git-lfs ; then - echo 'INFO: ... using `--lfs` since you are likely offsite and have git-lfs installed' - useLfs=true - elif [ -d /cvmfs/oasis.opensciencegrid.org/jlab ]; then - echo 'INFO: ... using `--cvmfs` since you appear to have /cvmfs/oasis.opensciencegrid.org' - useCvmfs=true - else - echo 'WARNING: default data-retrieval option cannot be determined; use `--help` for guidance' >&2 - sleep 1 - fi -fi - -# if they chose too many, fail -if [[ $(count_download_opts) -gt 1 ]]; then - echo 'ERROR: more than one data-retrieval option is set' >&2 - exit 1 -fi - ################################################################################ # setup @@ -141,15 +113,6 @@ wget="wget ${wgetArgs[@]:-}" # environment source libexec/env.sh --no-classpath -# install LFS -if $useLfs; then - if ! command_exists git-lfs ; then - echo 'ERROR: `git-lfs` not found; please install it, or use a different option other than `--lfs`' >&2 - exit 1 - fi - git lfs install -fi - ################################################################################ # cleaning, dependency analysis, etc. ################################################################################ @@ -189,6 +152,11 @@ fi # download field maps, NN models, etc. ################################################################################ +# check if a command exists +command_exists () { + type "$1" &> /dev/null +} + # print retrieval notice notify_retrieval() { echo "Retrieving $1 from $2 ..." @@ -196,10 +164,6 @@ notify_retrieval() { # update an LFS submodule download_lfs() { - if ! $useLfs; then - echo 'ERROR: attempted to use LFS, but option `--lfs` not set' >&2 - exit 1 - fi cd $src_dir > /dev/null git submodule update --init $1 cd - > /dev/null @@ -207,76 +171,125 @@ download_lfs() { # download a magnetic field map download_map () { - ret=0 - if $useXrootd; then - notify_retrieval 'field map' 'xrootd' - xrdcp $1 ./ - ret=$? - elif $useCvmfs; then - notify_retrieval 'field map' 'cvmfs' - cp $1 ./ - ret=$? - elif command_exists wget ; then + ret=0 + case $dataRetrieval in + cvmfs) + notify_retrieval 'field map' 'cvmfs' + cp $1 ./ + ret=$? + ;; + clasweb) + if command_exists wget ; then notify_retrieval 'field map' 'clasweb via wget' $wget $1 ret=$? - elif command_exists curl ; then + elif command_exists curl ; then notify_retrieval 'field map' 'clasweb via curl' if ! [ -e ${1##*/} ]; then curl $1 -o ${1##*/} ret=$? fi - else + else ret=1 echo "ERROR::::::::::: Could not find wget nor curl." >&2 - fi - return $ret + fi + ;; + *) + ret=1 + echo "ERROR::::::::::: called 'download_map' with bad 'dataRetrieval'." >&2 + ;; + esac + return $ret } +# check data-retrieval options, and prepare accordingly +case $dataRetrieval in + lfs) + if ! command_exists git-lfs ; then + echo 'ERROR: `git-lfs` not found; please install it, or use a different data-retrieval option other than `--lfs`' >&2 + exit 1 + fi + git lfs install + ;; + cvmfs) + path=/cvmfs/oasis.opensciencegrid.org/jlab + if [ ! -d $path ]; then + echo "ERROR: cannot find CVMFS path '$path'; data retrieval option \`--cvmfs\` failed" >&2 + exit 1 + fi + ;; + clasweb) + ;; + *) + echo "ERROR: data retrieval option '$dataRetrieval' is not supported" >&2 + exit 1 + ;; +esac + # download the default field maps, as defined in libexec/env.sh: # (and duplicated in etc/services/reconstruction.yaml): if $downloadMaps; then - if $useLfs; then - notify_retrieval 'field maps' 'lfs' - download_lfs etc/data/magfield - else - webDir=https://clasweb.jlab.org/clas12offline/magfield - if $useXrootd; then webDir=xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/coatjava/magfield; fi - if $useCvmfs; then webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield; fi - mkdir -p $magfield_dir - cd $magfield_dir - for map in $COAT_MAGFIELD_SOLENOIDMAP $COAT_MAGFIELD_TORUSMAP $COAT_MAGFIELD_TORUSSECONDARYMAP - do - download_map $webDir/$map - if [ $? -ne 0 ]; then - echo "ERROR::::::::::: Could not download field map:" >&2 - echo "$webDir/$map" >&2 - echo "One option is to download manually into etc/data/magfield and then run this build script with --nomaps" >&2 - exit 1 + case $dataRetrieval + lfs) + notify_retrieval 'field maps' 'lfs' + download_lfs etc/data/magfield + ;; + cvmfs|clasweb) + webDir=https://clasweb.jlab.org/clas12offline/magfield + if [ "$dataRetrieval" = "cvmfs" ]; then + webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield fi - done - cd - - fi + mkdir -p $magfield_dir + cd $magfield_dir + for map in $COAT_MAGFIELD_SOLENOIDMAP $COAT_MAGFIELD_TORUSMAP $COAT_MAGFIELD_TORUSSECONDARYMAP + do + download_map $webDir/$map + if [ $? -ne 0 ]; then + echo "ERROR::::::::::: Could not download field map:" >&2 + echo "$webDir/$map" >&2 + echo "One option is to download manually into etc/data/magfield and then run this build script with --nomaps" >&2 + exit 1 + fi + done + cd - + ;; + *) + echo "ERROR: data retrieval option '$dataRetrieval' not supported for field maps" >&2 + exit 1 + ;; + esac fi # download neural networks if $downloadNets; then - if $useLfs; then - notify_retrieval 'neural networks' 'lfs' - download_lfs etc/data/nnet - elif $useCvmfs; then - notify_retrieval 'neural networks' 'cvmfs' - cp -r /cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/networks/* etc/data/nnet/ - else - echo 'WARNING: neural networks not downloaded; run with `--help` for guidance' >&2 - sleep 1 - fi + case $dataRetrieval in + lfs) + notify_retrieval 'neural networks' 'lfs' + download_lfs etc/data/nnet + ;; + cvmfs) + notify_retrieval 'neural networks' 'cvmfs' + cp -r /cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/networks/* etc/data/nnet/ + ;; + *) + echo 'WARNING: neural networks not downloaded; run with `--help` for guidance' >&2 + sleep 1 + ;; + esac fi # download validation data if $downloadData; then - notify_retrieval 'validation data' 'lfs' - download_lfs validation/advanced-tests/data + case $dataRetrieval in + lfs) + notify_retrieval 'validation data' 'lfs' + download_lfs validation/advanced-tests/data + ;; + *) + echo 'ERROR: option `--lfs` must be used when using `--data`' >&2 + exit 1 + ;; + esac fi From 34c471628d489bfdd03215ee7d45d84476f0df16 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 15 Dec 2025 19:26:03 -0500 Subject: [PATCH 2/4] fix: typo and make `--xrootd` usage fail --- build-coatjava.sh | 51 ++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/build-coatjava.sh b/build-coatjava.sh index a622128ef7..eb9b1dcfe7 100755 --- a/build-coatjava.sh +++ b/build-coatjava.sh @@ -81,6 +81,7 @@ do --cvmfs) dataRetrieval=cvmfs ;; --lfs) dataRetrieval=lfs ;; --clasweb) dataRetrieval=clasweb ;; + --xrootd) dataRetrieval=xrootd ;; --clara) installClara=true ;; --data) downloadData=true ;; -h|--help) @@ -157,6 +158,30 @@ command_exists () { type "$1" &> /dev/null } +# check data-retrieval options, and prepare accordingly +case $dataRetrieval in + lfs) + if ! command_exists git-lfs ; then + echo 'ERROR: `git-lfs` not found; please install it, or use a different data-retrieval option other than `--lfs`' >&2 + exit 1 + fi + git lfs install + ;; + cvmfs) + path=/cvmfs/oasis.opensciencegrid.org/jlab + if [ ! -d $path ]; then + echo "ERROR: cannot find CVMFS path '$path'; data retrieval option \`--cvmfs\` failed" >&2 + exit 1 + fi + ;; + clasweb) + ;; + *) + echo "ERROR: data retrieval option '$dataRetrieval' is not supported" >&2 + exit 1 + ;; +esac + # print retrieval notice notify_retrieval() { echo "Retrieving $1 from $2 ..." @@ -202,34 +227,10 @@ download_map () { return $ret } -# check data-retrieval options, and prepare accordingly -case $dataRetrieval in - lfs) - if ! command_exists git-lfs ; then - echo 'ERROR: `git-lfs` not found; please install it, or use a different data-retrieval option other than `--lfs`' >&2 - exit 1 - fi - git lfs install - ;; - cvmfs) - path=/cvmfs/oasis.opensciencegrid.org/jlab - if [ ! -d $path ]; then - echo "ERROR: cannot find CVMFS path '$path'; data retrieval option \`--cvmfs\` failed" >&2 - exit 1 - fi - ;; - clasweb) - ;; - *) - echo "ERROR: data retrieval option '$dataRetrieval' is not supported" >&2 - exit 1 - ;; -esac - # download the default field maps, as defined in libexec/env.sh: # (and duplicated in etc/services/reconstruction.yaml): if $downloadMaps; then - case $dataRetrieval + case $dataRetrieval in lfs) notify_retrieval 'field maps' 'lfs' download_lfs etc/data/magfield From 85f8afbeafb0973d8b97a08813e449d970ddb9bf Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 16 Dec 2025 10:13:59 -0500 Subject: [PATCH 3/4] feat: `--wipe` --- build-coatjava.sh | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/build-coatjava.sh b/build-coatjava.sh index eb9b1dcfe7..fefa2e5540 100755 --- a/build-coatjava.sh +++ b/build-coatjava.sh @@ -38,9 +38,10 @@ DATA RETRIEVAL OPTIONS --lfs use Git Large File Storage (requires `git-lfs`) --cvmfs use CernVM-FS (requires `/cvfms`) --clasweb use clasweb (only works for field maps) - Options to disable data retrieval: + Additional options --nomaps do not download/overwrite field maps --nonets do not download/overwrite neural networks + --wipe remove retrieved data TESTING OPTIONS --spotbugs also run spotbugs plugin @@ -81,9 +82,13 @@ do --cvmfs) dataRetrieval=cvmfs ;; --lfs) dataRetrieval=lfs ;; --clasweb) dataRetrieval=clasweb ;; - --xrootd) dataRetrieval=xrootd ;; + --wipe) dataRetrieval=wipe ;; --clara) installClara=true ;; --data) downloadData=true ;; + --xrootd) + echo "ERROR: option \`$xx\` is deprecated; use \`--help\` for guidance" >&2 + exit 1 + ;; -h|--help) echo "$usage" exit 2 @@ -130,14 +135,17 @@ if $cleanBuild; then for target_dir in $(find $src_dir -type d -name target); do echo "WARNING: target directory '$target_dir' was not removed! JAR files within may be accidentally installed!" >&2 done - echo """DONE CLEANING. - NOTE: - - to remove local magnetic field maps: - rm $magfield_dir/*.dat - - to clear all LFS git submodules: - git submodule deinit --all - - Now re-run without \`--clean\` to build.""" +fi + +# wipe retrieved data (field maps, NN models, etc.) +if [ "$dataRetrieval" = "wipe" ]; then + git submodule deinit --all --force +fi + +# print cleanup note and exit +if $cleanBuild || [ "$dataRetrieval" = "wipe" ]; then + [ "$dataRetrieval" = "wipe" ] && echo "[+] REMOVED RETRIEVED DATA" || echo "[+] NOTE: retrieved data not removed; use \`--wipe\` if you need to remove them" + $cleanBuild && echo "[+] DONE CLEANING; rerun without \`--clean\` to build" exit fi From 33bc3774b683f09f305b879c7e4784e9dbc3472f Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 16 Dec 2025 10:19:34 -0500 Subject: [PATCH 4/4] refactor!: `--clasweb` -> `--https` --- build-coatjava.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/build-coatjava.sh b/build-coatjava.sh index fefa2e5540..914a1fd58f 100755 --- a/build-coatjava.sh +++ b/build-coatjava.sh @@ -37,7 +37,7 @@ DATA RETRIEVAL OPTIONS Choose only one; default is `--'$dataRetrieval'` --lfs use Git Large File Storage (requires `git-lfs`) --cvmfs use CernVM-FS (requires `/cvfms`) - --clasweb use clasweb (only works for field maps) + --https use clasweb HTTPS (field maps only) Additional options --nomaps do not download/overwrite field maps --nonets do not download/overwrite neural networks @@ -79,12 +79,12 @@ do mvnArgs+=(--no-transfer-progress) wgetArgs+=(--no-verbose) ;; - --cvmfs) dataRetrieval=cvmfs ;; - --lfs) dataRetrieval=lfs ;; - --clasweb) dataRetrieval=clasweb ;; - --wipe) dataRetrieval=wipe ;; - --clara) installClara=true ;; - --data) downloadData=true ;; + --cvmfs) dataRetrieval=cvmfs ;; + --lfs) dataRetrieval=lfs ;; + --https) dataRetrieval=https ;; + --wipe) dataRetrieval=wipe ;; + --clara) installClara=true ;; + --data) downloadData=true ;; --xrootd) echo "ERROR: option \`$xx\` is deprecated; use \`--help\` for guidance" >&2 exit 1 @@ -182,7 +182,7 @@ case $dataRetrieval in exit 1 fi ;; - clasweb) + https) ;; *) echo "ERROR: data retrieval option '$dataRetrieval' is not supported" >&2 @@ -211,7 +211,7 @@ download_map () { cp $1 ./ ret=$? ;; - clasweb) + https) if command_exists wget ; then notify_retrieval 'field map' 'clasweb via wget' $wget $1 @@ -243,7 +243,7 @@ if $downloadMaps; then notify_retrieval 'field maps' 'lfs' download_lfs etc/data/magfield ;; - cvmfs|clasweb) + cvmfs|https) webDir=https://clasweb.jlab.org/clas12offline/magfield if [ "$dataRetrieval" = "cvmfs" ]; then webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield