From c97c9a117bea6bacbbeb1069e6a0b8da0ca19897 Mon Sep 17 00:00:00 2001
From: Onno Zweers <onno.zweers@surfsara.nl>
Date: Mon, 20 Oct 2025 22:06:33 +0200
Subject: [PATCH 1/9] Implement --upload * Implement --upload, including
 --verify-checksum * Improve debug messages in get-webdav-door * New function
 exists() tells whether a path exists in dCache (file/dir)

---
 ada/ada | 181 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 178 insertions(+), 3 deletions(-)

diff --git a/ada/ada b/ada/ada
index 4b7c13a..3d7fd3a 100755
--- a/ada/ada
+++ b/ada/ada
@@ -122,6 +122,25 @@ usage() {
 	      Please note, that dCache storage systems usually
 	      don't have an undelete option.
 
+	  --upload /localpath/file /remotepath/[file] [--verify-checksum]
+	      Upload a file from the local system to dCache.
+	      With --verify-checksum, Ada computes the local file's
+	      MD5 checksum and sends it along with the upload.
+	      If the uploaded file's checksum does not match,
+	      dCache aborts the transfer and removes the corrupt file.
+	      This ensures data integrity during transfer.
+
+	      Ada will try several ways to discover a WebDAV door
+	      to use for the transfer.
+	      If that fails, you can specify the door explicitly by
+	      prepending it to the remote path, for example:
+	      https://webdav.example.org/remotepath/remotefile
+
+	  --download /remotepath/file /localpath/[file] [--verify-checksum]
+	      Download a file from dCache.
+	      With --verify-checksum, Ada will verify the file's checksum
+	      after download, and show an error message if it doesn't match.
+
 	  --setlabel <file> <label>
 	      Attach a label to a file.
 	  --rmlabel <file> <label>
@@ -316,6 +335,7 @@ set_defaults() {
   path=
   request_id=
   recursive=false
+  verify_checksum=false
   force=false
   resume=false
 
@@ -495,6 +515,12 @@ get_args() {
         path="$2"
         shift ; shift
         ;;
+      --upload )
+        command='upload'
+        source="$2"
+        target="$3"
+        shift ; shift ; shift
+        ;;
       --setlabel )
         command='setlabel'
         path="$2"
@@ -647,6 +673,10 @@ get_args() {
         recursive=true
         shift
         ;;
+      --verify-checksum )
+        verify_checksum=true
+        shift
+        ;;
       --force )
         force=true
         shift
@@ -1016,16 +1046,20 @@ get_webdav_door () {
   local webdav_door
   local command
   # Method 1: if there's an Rclone config file, read the WebDAV door from that.
+  $debug && echo "Getting WebDAV door." 1>&2
+  $debug && echo "Method 1: get WebDAV door from tokenfile." 1>&2
   if [ -f "$tokenfile" ] ; then
     $debug && echo "Getting WebDAV door from '$tokenfile' ..." 1>&2
     webdav_door=$(awk -F' *= *' '/^url *=/ {print $2}' "$tokenfile")
     if [ -n "$webdav_door" ] ; then
+      $debug && echo 1>&2 "Selected WebDAV door: '$webdav_door'"
       echo "$webdav_door"
       return 0
     fi
   fi
-  $debug && echo "Method failed. Trying next method." 1>&2
+  $debug && echo "Method 1 failed. Trying next method." 1>&2
   # Method 2: read the /scripts/config.js file from the API server.
+  $debug && echo "Method 2: get WebDAV door from API configuration." 1>&2
   command='$debug && set -x
            curl "${curl_authorization[@]}" \
                 "${curl_options_common[@]}" \
@@ -1041,10 +1075,11 @@ get_webdav_door () {
   fi
   # If we got a WebDAV door, we're done! Print value and return.
   if [ -n "$webdav_door" ] ; then
+    $debug && echo 1>&2 "Selected WebDAV door: '$webdav_door'"
     echo "$webdav_door"
     return 0
   fi
-  $debug && echo "Method failed. Trying next method." 1>&2
+  $debug && echo "Method 2 failed. Trying next method." 1>&2
   # Method 3.
   # This uses the /doors API call, which lists all doors.
   # The list of doors can be quite long.
@@ -1055,6 +1090,7 @@ get_webdav_door () {
   # - a lot of tags (which means it's publically advertised)
   # - port 443 is preferred (reduces firewall issues)
   # - a low load (we don't want a busy door)
+  $debug && echo "Method 3: ask API for door information" 1>&2
   command='$debug && set -x
            curl "${curl_authorization[@]}" \
                 "${curl_options_common[@]}" \
@@ -1084,10 +1120,14 @@ get_webdav_door () {
   fi
   # If we got a WebDAV door, we're done! Print value and return.
   if [ -n "$webdav_door" ] ; then
+    $debug && echo 1>&2 "Selected WebDAV door: '$webdav_door'"
     echo "$webdav_door"
     return 0
   else
-    echo "Unable to get WebDAV door. Please try again with --debug to get more information." 1>&2
+    $debug && echo "Method 3 failed. No other methods available." 1>&2
+    echo 1>&2 "Unable to get WebDAV door. Please try again with --debug to get more information." \
+              "If you want to upload or download, you could prepend the remote file with the webdav door." \
+              "See the help text for more information."
     return 1
   fi
 }
@@ -1133,6 +1173,29 @@ pathtype () {
 }
 
 
+exists () {
+  # Check whether a path exists. If the path doesn't exist, just continue.
+  # Similar to pathtype(), but that one quits with error if the path doesn't exist.
+  encoded_path=$(urlencode "$1")
+  # Warn the developers not to call this function before the auth has been set up.
+  if [ -z "$curl_authorization" ] ; then
+    echo 1>&2 "ERROR: calling function exists() before authentication has been set up." \
+              "Please submit an issue for this at https://github.com/sara-nl/SpiderScripts/ ."
+    exit 1
+  fi
+  result=$(
+            $debug && set -x
+            curl "${curl_authorization[@]}" \
+                 -H "accept: application/json" \
+                 --silent \
+                 -X GET "$api/namespace/$encoded_path" 2>/dev/null
+          )
+  # jq will check if key .fileType exists.
+  # The exit code of jq will be passed on to the caller of the function.
+  echo "$result" | jq 'has("fileType")' | grep --silent 'true'
+}
+
+
 is_dir () {
   # Check whether a path is a directory. If the path doesn't exist, just continue.
   # Similar to pathtype(), but that one quits with error if the path doesn't exist.
@@ -2247,6 +2310,9 @@ validate_input() {
       ;;
   esac
 
+  # Basic input validation, such as checking if necessary arguments are provided.
+  # Validation that uses the API is done later, because authentication has
+  # not been set up yet.
   case $command in
     list | stat | mkdir | mv | delete | \
     setlabel | lslabel | findlabel | rmlabel | \
@@ -2322,6 +2388,16 @@ validate_input() {
         exit 1
       fi
       ;;
+    upload )
+      if [[ -z $source || $source == --* ]] ; then
+        echo 1>&2 "ERROR: please specify a source file for your upload."
+        exit 1
+      fi
+      if [[ -z $target || $target == --* ]] ; then
+        echo 1>&2 "ERROR: please specify a target for your upload."
+        exit 1
+      fi
+      ;;
     stat-request )
       if [[ -z $request_id || $request_id =~ ^-- ]] ; then
         echo 1>&2 "ERROR: command $command requires a request-id."
@@ -2598,6 +2674,105 @@ api_call () {
           ;;
       esac
       ;;
+    upload )
+      # Check source (local) file
+      if [[ ! -f $source ]] ; then
+        echo 1>&2 "ERROR: local file '$source' does not exist or is not a file."
+        exit 1
+      fi
+      # Check target (remote) path.
+      # Has the user prepended a WebDAV door to the target? Split it.
+      if [[ "$target" == *"://"* ]]; then
+        webdav_door="${target%%//*}://${target#*//}"
+        webdav_door="${webdav_door%%/*}"
+        path="/${target#*://*/}"
+      else
+        webdav_door=""
+        path="$target"
+      fi
+      # If the target exists, it may be OK, if it's a dir.
+      if exists "$path" ; then
+        $debug && echo 1>&2 "Path $path exists."
+        # Let's check the target path. If it exists, it should be a dir, not a file,
+        # because dCache usually does not allow overwrites, and even if it does,
+        # overwrites can be dangerous.
+        case $(pathtype "$path") in
+          REGULAR )
+            echo 1>&2 "ERROR: target '$path' is an existing file. Ada doesn't support overwriting a file."
+            exit 1
+            ;;
+          LINK )
+            echo 1>&2 "ERROR: target '$path' is a symlink. A symlink can point to a file or a directory." \
+                      "Ada does not support uploading to a symlink."
+            exit 1
+            ;;
+          DIR )
+            real_target="$target$(basename "$source")"
+            if exists "$real_target" ; then
+              echo 1>&2 "Target '$real_target' already exists. Ada does not support overwriting."
+              exit 1
+            fi
+            ;;
+        esac
+      else
+        # target does not exist. That is OK, if the parent dir exists.
+        $debug && echo 1>&2 "Path $path does not exists."
+        parent=$(dirname "$path")
+        if ! is_dir "$parent" ; then
+          echo 1>&2 "ERROR: parent of '$path', '$parent', is not a directory." \
+                    "Can't write there. Please create that directory first."
+          exit 1
+        fi
+      fi
+      #
+      # Does the user want to automatically verify checksum?
+      # This has a few advantages:
+      # - An MD5 checksum will be added to the file in dCache
+      # - Upload will fail when the checksum doesn't match, and
+      #   the corrupt file will be removed automatically.
+      curl_args=()
+      if $verify_checksum ; then
+        $debug && set -x   # In debug mode, switch on tracing
+        source_checksum=$(md5sum "$source" | cut -d' ' -f1 | xxd -r -p | base64)
+        { set +x ; } 2>/dev/null   # Silently switch off tracing
+        if [[ -z $source_checksum ]] ; then
+          echo 1>&2 "ERROR: you want to upload with checksum verification," \
+                    "but the checksum of '$source' could not be found."
+          exit 1
+        fi
+        curl_args=(
+          -H "Content-MD5: $source_checksum"
+        )
+      fi
+      # We can't upload through the API; we need a WebDAV door for that.
+      # If the user hasn't prepended the WebDAV door to the target,
+      # we need to discover it.
+      if [[ -z $webdav_door ]] ; then
+        webdav_door=$(get_webdav_door "$api")
+      else
+        $debug && echo 1>&2 "User specified WebDAV door: $webdav_door"
+      fi
+      # We should be ready to upload now!
+      # If --debug was specified, run curl with --verbose.
+      if $debug ; then
+        curl_args+=(--verbose)
+      fi
+      $debug && set -x   # If --debug is specified, show curl command (tracing on)
+      curl "${curl_authorization[@]}" \
+           --location --post302 \
+           "${curl_args[@]}" \
+           -T "$source" \
+           "$webdav_door$target"
+      { set +x ; } 2>/dev/null   # Silently switch off tracing
+      ;;
+    download )
+      # Check source (remote)
+      #  - Is file? Then OK
+      # Check target (local)
+      #  - Is file? Overwrite or not?
+      #  - Is dir? Use remote file name
+      #  - Does not exist? Check if parent is an existing dir, then OK, use remote file name
+      ;;
     setlabel | lslabel | rmlabel )
       case $(pathtype "$path") in
         DIR )

From b17ebdf553e22f4dd4855d6ee4a4b78efc3459c9 Mon Sep 17 00:00:00 2001
From: Onno Zweers <onno.zweers@surfsara.nl>
Date: Wed, 22 Oct 2025 16:16:11 +0200
Subject: [PATCH 2/9] Implement --download * Implemented ada --download
 function, including checksum verification * Created get_checksum function, to
 get a single checksum * Created get_local_checksum function, to get a
 checksum of a local file * Fixed rogue header from a curl command * Removed
 --verbose from curl when debugging; output is too overwhelming

---
 ada/ada | 183 ++++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 170 insertions(+), 13 deletions(-)

diff --git a/ada/ada b/ada/ada
index 3d7fd3a..6e0d08c 100755
--- a/ada/ada
+++ b/ada/ada
@@ -521,6 +521,12 @@ get_args() {
         target="$3"
         shift ; shift ; shift
         ;;
+      --download )
+        command='download'
+        source="$2"
+        target="$3"
+        shift ; shift ; shift
+        ;;
       --setlabel )
         command='setlabel'
         path="$2"
@@ -1063,7 +1069,6 @@ get_webdav_door () {
   command='$debug && set -x
            curl "${curl_authorization[@]}" \
                 "${curl_options_common[@]}" \
-                -H "Authorization: bearer $BEARER_TOKEN" \
                 -H "Accept: application/json" \
                 "$api_server/scripts/config.js" \
            | jq -r ".\"dcache-view.endpoints.webdav\"" '
@@ -1737,6 +1742,55 @@ get_checksums () {
 }
 
 
+get_checksum () {
+  # Prints the requested checksum of a dCache file; nothing else.
+  # If the checksum of that type is not in the dCache database, print nothing.
+  local type="$1"
+  local path="$2"
+  encoded_path=$(urlencode "$path")
+  # Make sure the checksum type is something that dCache understands
+  case $type in
+    adler32 | ADLER32 )
+      type=ADLER32
+      ;;
+    md5* | MD5* )
+      type=MD5_TYPE
+      ;;
+    * )
+      echo 1>&2 "ERROR: checksum type '$type' is unsupported."
+      exit 1
+      ;;
+  esac
+  # We output only the requested checksum, nothing else.
+  # If the requested checksum for this file is not in the dCache database,
+  # the result will be empty.
+  curl "${curl_authorization[@]}" \
+       "${curl_options_no_errors[@]}" \
+       -X GET "$api/namespace/$encoded_path?checksum=true" \
+  | jq -r ".checksums[] | select(.type == \"$type\") | .value"
+}
+
+
+get_local_checksum () {
+  # prints the checksum of a local file (not in dCache).
+  local type="$1"
+  local file="$2"
+  case $type in
+    adler32 | ADLER32 )
+      # This might be the most portable way to calculate an Adler32 checksum.
+      python3 -c "import zlib,sys;print(format(zlib.adler32(open('$file','rb').read()) & 0xffffffff,'08x'))"
+      ;;
+    md5* | MD5* )
+      md5sum --quiet "$file"
+      ;;
+    * )
+      echo 1>&2 "ERROR: checksum type '$type' is unsupported."
+      exit 1
+      ;;
+  esac
+}
+
+
 get_channel_by_name () {
   local channel_name="$1"
   # Many other API calls depend on this one.
@@ -2388,13 +2442,13 @@ validate_input() {
         exit 1
       fi
       ;;
-    upload )
+    upload | download )
       if [[ -z $source || $source == --* ]] ; then
-        echo 1>&2 "ERROR: please specify a source file for your upload."
+        echo 1>&2 "ERROR: please specify a source file for your $command."
         exit 1
       fi
       if [[ -z $target || $target == --* ]] ; then
-        echo 1>&2 "ERROR: please specify a target for your upload."
+        echo 1>&2 "ERROR: please specify a target for your $command."
         exit 1
       fi
       ;;
@@ -2753,10 +2807,6 @@ api_call () {
         $debug && echo 1>&2 "User specified WebDAV door: $webdav_door"
       fi
       # We should be ready to upload now!
-      # If --debug was specified, run curl with --verbose.
-      if $debug ; then
-        curl_args+=(--verbose)
-      fi
       $debug && set -x   # If --debug is specified, show curl command (tracing on)
       curl "${curl_authorization[@]}" \
            --location --post302 \
@@ -2766,12 +2816,119 @@ api_call () {
       { set +x ; } 2>/dev/null   # Silently switch off tracing
       ;;
     download )
-      # Check source (remote)
-      #  - Is file? Then OK
+      #
       # Check target (local)
-      #  - Is file? Overwrite or not?
-      #  - Is dir? Use remote file name
-      #  - Does not exist? Check if parent is an existing dir, then OK, use remote file name
+      # We check the target first because it is local and thus faster.
+      #
+      curl_target=()
+      # If a file already exists, quit.
+      if [[ -f $target ]] ; then
+        echo 1>&2 "ERROR: Target file '$target' already exists."
+        exit 1
+      fi
+      # If the target is a directory, we tell curl to use the
+      # remote filename
+      if [[ -d $target ]] ; then
+        real_target="${target%/}/$(basename "$source")"
+        if [[ -e "$real_target" ]] ; then
+          echo 1>&2 "ERROR: destination '$real_target' already exists."
+          exit 1
+        fi
+      fi
+      # If the target does not exist, it's OK if the parent dir exists.
+      if [[ ! -e $target ]] ; then
+        parent=$(dirname "$target")
+        if [[ ! -d $parent ]] ; then
+          echo "ERROR: Can't download to '$target': directory '$parent' does not exist."
+          exit 1
+        fi
+        real_target="$target"
+      fi
+      curl_target=(--output "$real_target")
+      #
+      # Check source (remote).
+      #
+      # Has the user prepended a WebDAV door to the source? Split it.
+      if [[ "$source" == *"://"* ]]; then
+        webdav_door="${source%%//*}://${source#*//}"
+        webdav_door="${webdav_door%%/*}"
+        path="/${source#*://*/}"
+      else
+        webdav_door=""
+        path="$source"
+      fi
+      # Does the source (remote path) exist?
+      if ! exists "$path" ; then
+        echo 1>&2 "ERROR: source file '$path' does not exist."
+        exit 1
+      fi
+      # The source path should be a file, not a dir or a link.
+      case $(pathtype "$path") in
+        DIR )
+          echo 1>&2 "Source '$path' is a directory. Ada does not support downloading a directory." \
+                    "You can use Rclone for that."
+          exit 1
+          ;;
+        LINK )
+          echo 1>&2 "ERROR: target '$path' is a symlink. A symlink can point to a file or a directory." \
+                    "Ada does not support downloading from a symlink."
+          exit 1
+          ;;
+      esac
+      # If the source file is only on tape and not online,
+      # dCache would automatically stage it when we read it, but
+      # that could take a very long time (varying from minutes to days).
+      # All that time, the curl would be waiting for the file,
+      # keeping a transfer slot occupied, and confusing the user.
+      # So, we refuse to download tape files, and suggest to stage them.
+      if ! is_online "$path" ; then
+        echo 1>&2 "ERROR: file '$path' exists, but it is on tape, and not online." \
+                  "You need to stage it first before you can download it." \
+                  "You can use 'ada --stage $path' for this." 
+        exit 1
+      fi
+      # We can't download through the API; we need a WebDAV door for that.
+      # If the user hasn't prepended the WebDAV door to the source,
+      # we need to discover it.
+      if [[ -z $webdav_door ]] ; then
+        webdav_door=$(get_webdav_door "$api")
+      else
+        $debug && echo 1>&2 "User specified WebDAV door: $webdav_door"
+      fi
+      # We should be ready to download!
+      $debug && set -x   # If --debug is specified, show curl command (tracing on)
+      curl "${curl_authorization[@]}" \
+           --location --post302 \
+           "$webdav_door$source" \
+           "${curl_target[@]}"
+      { set +x ; } 2>/dev/null   # Silently switch off tracing
+      if $verify_checksum ; then
+        # For now, we just print the checksums; verify will be implemented later.
+        remote_checksum=$(get_checksum ADLER32 "$path")
+        if [[ -n $remote_checksum ]] ; then
+          local_checksum=$(get_local_checksum ADLER32 "$real_target")
+        else
+          remote_checksum=$(get_checksum MD5 "$path")
+          if [[ -z $remote_checksum ]] ; then
+            echo 1>&2 "ERROR: unable to get a checksum for file '$path'."
+            exit 1
+          fi
+          local_checksum=$(get_local_checksum MD5 "$real_target")
+        fi
+        if [[ "$local_checksum" == "$remote_checksum" ]] ; then
+          if $debug ; then
+            echo 1>&2 "Remote checksum: $remote_checksum" ;
+            echo 1>&2 "Local checksum : $local_checksum" ;
+          fi
+          echo "Checksum OK."
+        else
+          {
+            echo "ERROR: Checksum incorrect! File '$real_target' might be damaged."
+            echo "Remote checksum: $remote_checksum" ;
+            echo "Local checksum : $local_checksum" ;
+          } 1>&2
+        fi
+      fi
       ;;
     setlabel | lslabel | rmlabel )
       case $(pathtype "$path") in

From f4a592e1df849acd00ee1a0c8031c1025563d155 Mon Sep 17 00:00:00 2001
From: Onno Zweers <onno.zweers@surfsara.nl>
Date: Wed, 29 Oct 2025 13:07:40 +0100
Subject: [PATCH 3/9] If file not online, provide accurate, working command to
 stage it. Before this commit, an example command was provided, but in many
 cases copying and pasting the command wouldn't work. With this commit, a
 command is printed that will work in all cases (AFAIK). I stole some lines of
 code from Haili and adapted them a bit (to fix shellcheck warnings and
 improve layout).

---
 ada/ada | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/ada/ada b/ada/ada
index 743415a..0615244 100755
--- a/ada/ada
+++ b/ada/ada
@@ -2938,9 +2938,18 @@ api_call () {
       # keeping a transfer slot occupied, and confusing the user.
       # So, we refuse to download tape files, and suggest to stage them.
       if ! is_online "$path" ; then
+        # Construct the stage command that the user would need to stage this file.
+        cmd_args=""
+        if [[ "${args[*]}" =~ "api" ]]; then
+          cmd_args="--api $api"
+        fi
+        if [[ "${args[*]}" =~ "netrc" || "${args[*]}" =~ "proxy" || "${args[*]}" =~ "tokenfile" ]]; then
+          cmd_args="--$auth_method $auth_file $cmd_args"
+        fi
         echo 1>&2 "ERROR: file '$path' exists, but it is on tape, and not online." \
                   "You need to stage it first before you can download it." \
-                  "You can use 'ada --stage $path' for this." 
+                  "You can do that with this command:"
+        echo 1>&2 "   $0 --stage $path" "$cmd_args" | tr -d '\r'
         exit 1
       fi
       # We can't download through the API; we need a WebDAV door for that.

From 35358bf9533580faf9868c8b4a11f5eb74400f1f Mon Sep 17 00:00:00 2001
From: Onno Zweers <onno.zweers@surfsara.nl>
Date: Wed, 29 Oct 2025 14:43:16 +0100
Subject: [PATCH 4/9] Add remark about staging in bulk

---
 ada/ada | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/ada/ada b/ada/ada
index 0615244..850b8bc 100755
--- a/ada/ada
+++ b/ada/ada
@@ -2946,10 +2946,15 @@ api_call () {
         if [[ "${args[*]}" =~ "netrc" || "${args[*]}" =~ "proxy" || "${args[*]}" =~ "tokenfile" ]]; then
           cmd_args="--$auth_method $auth_file $cmd_args"
         fi
-        echo 1>&2 "ERROR: file '$path' exists, but it is on tape, and not online." \
-                  "You need to stage it first before you can download it." \
-                  "You can do that with this command:"
-        echo 1>&2 "   $0 --stage $path" "$cmd_args" | tr -d '\r'
+        {
+          echo "ERROR: file '$path' exists, but it is on tape, and not online." \
+              "You need to stage it first before you can download it." \
+              "You can do that with this command:"
+          echo
+          echo "   $0 --stage $path" "$cmd_args" | tr -d '\r'
+          echo
+          echo "Please note, that staging works most efficiently when done in bulk."
+        } 1>&2
         exit 1
       fi
       # We can't download through the API; we need a WebDAV door for that.

From c532193713c1788ae84c6e6c9c55242339b461b1 Mon Sep 17 00:00:00 2001
From: Onno Zweers <onno.zweers@surfsara.nl>
Date: Wed, 29 Oct 2025 15:01:44 +0100
Subject: [PATCH 5/9] Adding comments and debug messages

---
 ada/ada | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/ada/ada b/ada/ada
index 850b8bc..e8711ce 100755
--- a/ada/ada
+++ b/ada/ada
@@ -1820,10 +1820,12 @@ get_checksum () {
   # We output only the requested checksum, nothing else.
   # If the requested checksum for this file is not in the dCache database,
   # the result will be empty.
+  $debug && set -x
   curl "${curl_authorization[@]}" \
        "${curl_options_no_errors[@]}" \
        -X GET "$api/namespace/$encoded_path?checksum=true" \
   | jq -r ".checksums[] | select(.type == \"$type\") | .value"
+  { set +x ; } &> /dev/null
 }
 
 
@@ -2973,11 +2975,15 @@ api_call () {
            "${curl_target[@]}"
       { set +x ; } 2>/dev/null   # Silently switch off tracing
       if $verify_checksum ; then
-        # For now, we just print the checksums; verify will be implemented later.
+        $debug && echo 1>&2 "Comparing checksums..."
+        # Get the checksum of the remote file from the dCache database.
+        # First try Adler32 because it is the default in dCache,
+        # and fastest to calculate locally.
         remote_checksum=$(get_checksum ADLER32 "$path")
         if [[ -n $remote_checksum ]] ; then
           local_checksum=$(get_local_checksum ADLER32 "$real_target")
         else
+          # No Adler32 in the dCache database? Then try MD5.
           remote_checksum=$(get_checksum MD5 "$path")
           if [[ -z $remote_checksum ]] ; then
             echo 1>&2 "ERROR: unable to get a checksum for file '$path'."

From a65d03b62c7bb87c652a39a9f8e0d10df1b82c92 Mon Sep 17 00:00:00 2001
From: Onno Zweers <onno.zweers@surfsara.nl>
Date: Wed, 3 Dec 2025 13:49:20 +0100
Subject: [PATCH 6/9] With --verify-checksum and target file exists, verify
 checksum.

---
 ada/ada | 109 ++++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 75 insertions(+), 34 deletions(-)

diff --git a/ada/ada b/ada/ada
index e8711ce..c9fae7c 100755
--- a/ada/ada
+++ b/ada/ada
@@ -103,6 +103,11 @@ usage() {
 	      If the uploaded file's checksum does not match,
 	      dCache aborts the transfer and removes the corrupt file.
 	      This ensures data integrity during transfer.
+	      
+	      Ada does not overwrite files. If the destination file
+	      exists, Ada will print an error message, except when
+	      you specify --verify-checksum: then Ada will compare
+	      checksums of source and destination.
 
 	      Ada will try several ways to discover a WebDAV door
 	      to use for the transfer.
@@ -115,6 +120,11 @@ usage() {
 	      With --verify-checksum, Ada will verify the file's checksum
 	      after download, and show an error message if it doesn't match.
 
+	      Ada does not overwrite files. If the destination file
+	      exists, Ada will print an error message, except when
+	      you specify --verify-checksum: then Ada will compare
+	      checksums of source and destination.
+
 	  --setlabel <file> <label>
 	      Attach a label to a file.
 	  --rmlabel <file> <label>
@@ -1849,6 +1859,43 @@ get_local_checksum () {
 }
 
 
+compare_checksums() {
+  local local_file="$1"
+  local remote_file="$2"
+  $debug && echo 1>&2 "Comparing checksums..."
+  # Get the checksum of the remote file from the dCache database.
+  # First try Adler32 because it is the default in dCache,
+  # and fastest to calculate locally.
+  remote_checksum=$(get_checksum ADLER32 "$remote_file")
+  if [[ -n $remote_checksum ]] ; then
+    local_checksum=$(get_local_checksum ADLER32 "$local_file")
+  else
+    # No Adler32 in the dCache database? Then try MD5.
+    remote_checksum=$(get_checksum MD5 "$remote_file")
+    if [[ -z $remote_checksum ]] ; then
+      echo 1>&2 "ERROR: unable to get a checksum for file '$remote_file'."
+      exit 1
+    fi
+    local_checksum=$(get_local_checksum MD5 "$local_file")
+  fi
+  if [[ "$local_checksum" == "$remote_checksum" ]] ; then
+    if $debug ; then
+      echo 1>&2 "Remote checksum: $remote_checksum for file '$remote_file'." ;
+      echo 1>&2 "Local checksum : $local_checksum for file '$local_file'." ;
+    fi
+    echo "Checksum OK."
+    return 0
+  else
+    {
+      echo "ERROR: Checksum mismatch!"
+      echo "Remote checksum: $remote_checksum" ;
+      echo "Local checksum : $local_checksum" ;
+    } 1>&2
+    return 1
+  fi
+}
+
+
 get_channel_by_name () {
   local channel_name="$1"
   # Many other API calls depend on this one.
@@ -2810,8 +2857,17 @@ api_call () {
         # overwrites can be dangerous.
         case $(pathtype "$path") in
           REGULAR )
-            echo 1>&2 "ERROR: target '$path' is an existing file. Ada doesn't support overwriting a file."
-            exit 1
+            # Target (remote) exists and is a file.
+            # If user wants to verify checksums, we verify checksums!
+            if $verify_checksum ; then
+              echo 1>&2 "Target (remote) file '$path' already exists. Verifying checksum..."
+              compare_checksums "$source" "$path" || exit 1
+              exit 0
+            else
+              # User not interested in checksums. We show an error message.
+              echo 1>&2 "ERROR: target '$path' is an existing file. Ada doesn't support overwriting a file."
+              exit 1
+            fi
             ;;
           LINK )
             echo 1>&2 "ERROR: target '$path' is a symlink. A symlink can point to a file or a directory." \
@@ -2865,6 +2921,11 @@ api_call () {
         $debug && echo 1>&2 "User specified WebDAV door: $webdav_door"
       fi
       # We should be ready to upload now!
+      if $verify_checksum ; then
+        echo 1>&2 "Uploading with checksum verification..."
+      else
+        echo 1>&2 "Uploading..."
+      fi
       $debug && set -x   # If --debug is specified, show curl command (tracing on)
       curl "${curl_authorization[@]}" \
            --location --post302 \
@@ -2879,10 +2940,18 @@ api_call () {
       # We check the target first because it is local and thus faster.
       #
       curl_target=()
-      # If a file already exists, quit.
+      # Does the local (target) file already exist?
       if [[ -f $target ]] ; then
-        echo 1>&2 "ERROR: Target file '$target' already exists."
-        exit 1
+        # If user wants to verify checksums, we verify checksums!
+        if $verify_checksum ; then
+          echo 1>&2 "Target (local) file '$target' already exists. Verifying checksum..."
+          compare_checksums "$target" "$source" || exit 1
+          exit 0
+        else
+          # User not interested in checksums. We show an error message.
+          echo 1>&2 "ERROR: Target file '$target' already exists."
+          exit 1
+        fi
       fi
       # If the target is a directory, we tell curl to use the
       # remote filename
@@ -2975,35 +3044,7 @@ api_call () {
            "${curl_target[@]}"
       { set +x ; } 2>/dev/null   # Silently switch off tracing
       if $verify_checksum ; then
-        $debug && echo 1>&2 "Comparing checksums..."
-        # Get the checksum of the remote file from the dCache database.
-        # First try Adler32 because it is the default in dCache,
-        # and fastest to calculate locally.
-        remote_checksum=$(get_checksum ADLER32 "$path")
-        if [[ -n $remote_checksum ]] ; then
-          local_checksum=$(get_local_checksum ADLER32 "$real_target")
-        else
-          # No Adler32 in the dCache database? Then try MD5.
-          remote_checksum=$(get_checksum MD5 "$path")
-          if [[ -z $remote_checksum ]] ; then
-            echo 1>&2 "ERROR: unable to get a checksum for file '$path'."
-            exit 1
-          fi
-          local_checksum=$(get_local_checksum MD5 "$real_target")
-        fi
-        if [[ "$local_checksum" == "$remote_checksum" ]] ; then
-          if $debug ; then
-            echo 1>&2 "Remote checksum: $remote_checksum" ;
-            echo 1>&2 "Local checksum : $local_checksum" ;
-          fi
-          echo "Checksum OK."
-        else
-          {
-            echo "ERROR: Checksum incorrect! File '$real_target' might be damaged."
-            echo "Remote checksum: $remote_checksum" ;
-            echo "Local checksum : $local_checksum" ;
-          } 1>&2
-        fi
+        compare_checksums "$real_target" "$source" || exit 1
       fi
       ;;
     setlabel | lslabel | rmlabel )

From 29a3f6b7a1afc203225b41f476e95d4da03aaa79 Mon Sep 17 00:00:00 2001
From: Onno Zweers <onno.zweers@surfsara.nl>
Date: Wed, 3 Dec 2025 19:32:38 +0100
Subject: [PATCH 7/9] Prevent redirects from HTTPS to HTTP; fix webdav URL
 handling

* Use 'curl --proto =https' to prevent unsafe redirects
* Allow override with --allow-insecure-redirects
* Fix handling of https://webdavdoor/dir/file URLs
* When uploading or downloading target exists, and --verify-checksum is specified, we check the checksum of source and target. This was already implemented, but not for all cases. I hope I covered all cases now.
---
 ada/ada | 83 ++++++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 68 insertions(+), 15 deletions(-)

diff --git a/ada/ada b/ada/ada
index e975337..75b4db1 100755
--- a/ada/ada
+++ b/ada/ada
@@ -99,7 +99,7 @@ usage() {
 	      Please note, that dCache storage systems usually
 	      don't have an undelete option.
 
-	  --upload /localpath/file /remotepath/[file] [--verify-checksum]
+	  --upload /localpath/file /remotepath/[file] [--verify-checksum] [--allow-insecure-redirects]
 	      Upload a file from the local system to dCache.
 	      With --verify-checksum, Ada computes the local file's
 	      MD5 checksum and sends it along with the upload.
@@ -118,7 +118,12 @@ usage() {
 	      prepending it to the remote path, for example:
 	      https://webdav.example.org/remotepath/remotefile
 
-	  --download /remotepath/file /localpath/[file] [--verify-checksum]
+	      During uploads, Ada will prevent redirects from
+	      HTTPS to HTTP for security reasons. If you want to
+	      allow HTTP (for performance reasons), you can use
+	      --allow-insecure-redirects.
+
+	  --download /remotepath/file /localpath/[file] [--verify-checksum] [--allow-insecure-redirects]
 	      Download a file from dCache.
 	      With --verify-checksum, Ada will verify the file's checksum
 	      after download, and show an error message if it doesn't match.
@@ -128,6 +133,17 @@ usage() {
 	      you specify --verify-checksum: then Ada will compare
 	      checksums of source and destination.
 
+	      Ada will try several ways to discover a WebDAV door
+	      to use for the transfer.
+	      If that fails, you can specify the door explicitly by
+	      prepending it to the remote path, for example:
+	      https://webdav.example.org/remotepath/remotefile
+
+	      During downloads, Ada will prevent redirects from
+	      HTTPS to HTTP for security reasons. If you want to
+	      allow HTTP (for performance reasons), you can use
+	      --allow-insecure-redirects.
+
 	  --setlabel <file> <label>
 	      Attach a label to a file.
 	  --rmlabel <file> <label>
@@ -332,6 +348,7 @@ set_defaults() {
   verify_checksum=false
   force=false
   resume=false
+  allow_insecure_redirects=false
 
   # Default options to curl for various activities;
   # these can be overridden in configuration files, see below.
@@ -719,6 +736,10 @@ get_args() {
         verify_checksum=true
         shift
         ;;
+      --allow-insecure-redirects )
+        allow_insecure_redirects=true
+        shift
+        ;;
       --force )
         force=true
         shift
@@ -2837,6 +2858,8 @@ api_call () {
       esac
       ;;
     upload )
+      # Initialise
+      curl_args=()
       # Check source (local) file
       if [[ ! -f $source ]] ; then
         echo 1>&2 "ERROR: local file '$source' does not exist or is not a file."
@@ -2845,9 +2868,9 @@ api_call () {
       # Check target (remote) path.
       # Has the user prepended a WebDAV door to the target? Split it.
       if [[ "$target" == *"://"* ]]; then
-        webdav_door="${target%%//*}://${target#*//}"
-        webdav_door="${webdav_door%%/*}"
-        path="/${target#*://*/}"
+        webdav_door=$(echo "$target" | sed -nE 's#^([^:]+://[^/]+).*#\1#p')
+        path=$(echo "$target" | sed -nE 's#^[^:]+://[^/]+(/.*)#\1#p')
+        $debug && echo 1>&2 "webdav_door = $webdav_door | path = $path"
       else
         webdav_door=""
         path="$target"
@@ -2880,8 +2903,14 @@ api_call () {
           DIR )
             real_target="$target$(basename "$source")"
             if exists "$real_target" ; then
-              echo 1>&2 "Target '$real_target' already exists. Ada does not support overwriting."
-              exit 1
+              if $verify_checksum ; then
+                echo 1>&2 "Target '$real_target' already exists. Verifying checksum..."
+                compare_checksums "$source" "$real_target" || exit 1
+                exit 0
+              else
+                echo 1>&2 "Target '$real_target' already exists. Ada does not support overwriting."
+                exit 1
+              fi
             fi
             ;;
         esac
@@ -2901,7 +2930,6 @@ api_call () {
       # - An MD5 checksum will be added to the file in dCache
       # - Upload will fail when the checksum doesn't match, and
       #   the corrupt file will be removed automatically.
-      curl_args=()
       if $verify_checksum ; then
         $debug && set -x   # In debug mode, switch on tracing
         source_checksum=$(md5sum "$source" | cut -d' ' -f1 | xxd -r -p | base64)
@@ -2923,6 +2951,17 @@ api_call () {
       else
         $debug && echo 1>&2 "User specified WebDAV door: $webdav_door"
       fi
+      # Some dCache WebDAV doors may redirect from HTTPS to HTTP.
+      # By default, we prevent that with curl --proto =https.
+      # But there may be valid reasons to allow HTTP:
+      # it's faster. So we allow the user to override this
+      # with '--allow-insecure-redirects'.
+      if ! $allow_insecure_redirects ; then
+        # User does not override: limit curl to HTTPS only.
+        curl_args+=(
+          --proto '=https'
+        )
+      fi
       # We should be ready to upload now!
       if $verify_checksum ; then
         echo 1>&2 "Uploading with checksum verification..."
@@ -2934,10 +2973,12 @@ api_call () {
            --location --post302 \
            "${curl_args[@]}" \
            -T "$source" \
-           "$webdav_door$target"
+           "$webdav_door$path"
       { set +x ; } 2>/dev/null   # Silently switch off tracing
       ;;
     download )
+      # Initialise
+      curl_args=()
       #
       # Check target (local)
       # We check the target first because it is local and thus faster.
@@ -2980,9 +3021,9 @@ api_call () {
       #
       # Has the user prepended a WebDAV door to the source? Split it.
       if [[ "$source" == *"://"* ]]; then
-        webdav_door="${source%%//*}://${source#*//}"
-        webdav_door="${webdav_door%%/*}"
-        path="/${source#*://*/}"
+        webdav_door=$(echo "$source" | sed -nE 's#^([^:]+://[^/]+).*#\1#p')
+        path=$(echo "$source" | sed -nE 's#^[^:]+://[^/]+(/.*)#\1#p')
+        $debug && echo 1>&2 "webdav_door = $webdav_door | path = $path"
       else
         webdav_door=""
         path="$source"
@@ -3000,7 +3041,7 @@ api_call () {
           exit 1
           ;;
         LINK )
-          echo 1>&2 "ERROR: target '$path' is a symlink. A symlink can point to a file or a directory." \
+          echo 1>&2 "ERROR: source '$path' is a symlink. A symlink can point to a file or a directory." \
                     "Ada does not support downloading from a symlink."
           exit 1
           ;;
@@ -3039,15 +3080,27 @@ api_call () {
       else
         $debug && echo 1>&2 "User specified WebDAV door: $webdav_door"
       fi
+      # Some dCache WebDAV doors may redirect from HTTPS to HTTP.
+      # By default, we prevent that with 'curl --proto =https'.
+      # But there may be valid reasons to allow HTTP:
+      # it's faster. So we allow the user to override this
+      # with '--allow-insecure-redirects'.
+      if ! $allow_insecure_redirects ; then
+        # User does not override: limit curl to HTTPS only.
+        curl_args+=(
+          --proto '=https'
+        )
+      fi
       # We should be ready to download!
       $debug && set -x   # If --debug is specified, show curl command (tracing on)
       curl "${curl_authorization[@]}" \
+           "${curl_args[@]}" \
            --location --post302 \
-           "$webdav_door$source" \
+           "$webdav_door$path" \
            "${curl_target[@]}"
       { set +x ; } 2>/dev/null   # Silently switch off tracing
       if $verify_checksum ; then
-        compare_checksums "$real_target" "$source" || exit 1
+        compare_checksums "$real_target" "$path" || exit 1
       fi
       ;;
     setlabel | lslabel | rmlabel )

From 6538250370fcedd3c62839904ed953f0af612b4e Mon Sep 17 00:00:00 2001
From: Onno Zweers <onno.zweers@surfsara.nl>
Date: Wed, 3 Dec 2025 20:16:22 +0100
Subject: [PATCH 8/9] Update README.md

---
 README.md | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 6966162..2f0c715 100644
--- a/README.md
+++ b/README.md
@@ -16,15 +16,22 @@ ADA is a client that talks to the [dCache storage system](https://dcache.org/) [
 * Work with labels on files: set labels, remove them, and search directories for files with labels
 * Work with file metadata (extended attributes): set attributes, delete them, and search directories for files with certain attributes
 * Stage files (restore from tape), check whether they are online or not
-* Show available space in dCache
+* Show your disk and tape quotas
 * Subscribe to server-sent-events to set up automated workflows
+* Upload and download files
+
+Many of these operations can be done recursively. For authentication, ADA supports X509, tokens (macaroons and OIDC) and basic auth (username/password), depending on the dCache configuration.
+
+When something goes wrong, Ada tries to provide clear, understandable and useful error messages, so you can help yourself, or otherwise your support team will be able to help you better.
+
+With the `--debug` flag, Ada provides a wealth of information that may help you to troubleshoot problems. If you are a developer, this will help you to understand how to talk to the dCache API.
 
-Many of these operations can be done recursively. For authentication, ADA supports X509, tokens (macaroons and OIDC) and basic auth (username/password), depending on the dCache configuration. 
 
 #### Limitations
 
-* ADA does not transfer files; we suggest you use [Rclone](https://rclone.org/) for that.
+* ADA can upload and download files, but not in bulk, and it's not as efficient as [Rclone](https://rclone.org/).
 * ADA depends on dCache. The dCache system you work with may have limitations that impact ADA.
+* Ada is currently implemented in bash. Bash has limitations; for instance, trying to stage more than 1500 files in a single operation may fail.
 
 ### Installation
 

From 4ea08f5c6d42d7d68766d7ef9ac0752495bbcbcf Mon Sep 17 00:00:00 2001
From: Onno Zweers <onno.zweers@surfsara.nl>
Date: Thu, 4 Dec 2025 11:23:24 +0100
Subject: [PATCH 9/9] Document checksum behaviour during upload (Adler32 + MD5)

---
 ada/ada | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ada/ada b/ada/ada
index 75b4db1..8602f5f 100755
--- a/ada/ada
+++ b/ada/ada
@@ -101,11 +101,18 @@ usage() {
 
 	  --upload /localpath/file /remotepath/[file] [--verify-checksum] [--allow-insecure-redirects]
 	      Upload a file from the local system to dCache.
+
 	      With --verify-checksum, Ada computes the local file's
 	      MD5 checksum and sends it along with the upload.
 	      If the uploaded file's checksum does not match,
 	      dCache aborts the transfer and removes the corrupt file.
 	      This ensures data integrity during transfer.
+
+	      By default, dCache uses Adler32 as checksum.
+	      With --verify-checksum, dCache will store both the
+	      default (Adler32) and the MD5 checksum in its database.
+	      Without --verify-checksum, only the Adler32 checksum
+	      is stored, and not verified.
 	      
 	      Ada does not overwrite files. If the destination file
 	      exists, Ada will print an error message, except when