Add S3 storage and multi-hop transfer tutorial with resource files

docs/operator/_resources/rucio-playground/qt-000-up.txt

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cd `dirname $0`
+
+etc/certs/generate_minio12.sh
+
+docker compose --file etc/docker/dev/docker-compose-qt.yml --profile storage up -d

docs/operator/_resources/rucio-playground/qt-001-initialize.txt

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# rucio list-file-replicas test:dataset1
+# rucio list-file-replicas test:dataset2
+# rucio list-file-replicas test:dataset3
+# rucio list-file-replicas test:container
+
+# Step zero, get a compliant proxy. The key must NOT be group/other readable
+# (KEY=$(mktemp); cat /opt/rucio/etc/userkey.pem > "$KEY"; voms-proxy-init -valid 9999:00 -cert /opt/rucio/etc/usercert.pem -key "$KEY"; rm -f "$KEY")
+
+docker exec -i dev-rucio-1 /bin/bash tools/run_tests.sh -ir

docs/operator/_resources/rucio-playground/qt-002-xrd3-http-protocol.txt

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# XRD3 is enabled for multihop transfers
+# so add HTTP protocol to be able to talk to S3
+
+docker exec -i dev-rucio-1 /bin/bash <<END
+rucio rse protocol add XRD3 --host xrd3 --scheme https --prefix //rucio --port 1096 --impl rucio.rse.protocols.gfal.Default --domain-json '{"wan": {"read": 2, "write": 2, "delete": 2, "third_party_copy_read": 2, "third_party_copy_write": 2}, "lan": {"read": 2, "write": 2, "delete": 2}}'
+
+END

docs/operator/_resources/rucio-playground/qt-003-minio-buckets.txt

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+docker exec -i dev-minio-1 /bin/bash <<END
+export MC_INSECURE=true
+mc alias set local https://localhost:9001 admin password
+mc admin info local
+mc mb local/rucio
+mc ls local/
+END
+
+docker exec -i dev-minio-2 /bin/bash <<END
+export MC_INSECURE=true
+mc alias set local https://localhost:9002 admin password
+mc admin info local
+mc mb local/rucio
+mc ls local/
+END

docs/operator/_resources/rucio-playground/qt-004-minio-rses.txt

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+#
+# https://rucio.github.io/documentation/operator/s3_rse_config/
+#
+
+# set up RSEs
+docker exec -i dev-rucio-1 /bin/bash <<END
+rucio rse add MINIO1
+rucio rse protocol add MINIO1 --host minio1 --port 9001 --scheme https --prefix /rucio/ --impl rucio.rse.protocols.gfal.NoRename --domain-json '{"lan": {"read": 1, "write": 1, "delete": 1}, "wan": {"read": 1, "write": 1, "delete": 1, "third_party_copy_read": 1, "third_party_copy_write": 1}}'
+rucio rse attribute add MINIO1 --key sign_url --value s3
+rucio rse attribute add MINIO1 --key s3_url_style --value path
+rucio rse attribute add MINIO1 --key verify_checksum --value False
+rucio rse attribute add MINIO1 --key skip_upload_stat --value True
+rucio rse attribute add MINIO1 --key strict_copy --value True
+rucio rse attribute add MINIO1 --key fts --value https://fts:8446
+rucio account limit add root --rse MINIO1 --bytes infinity
+
+rucio rse add MINIO2
+rucio rse protocol add MINIO2 --host minio2 --port 9002 --scheme https --prefix /rucio/ --impl rucio.rse.protocols.gfal.NoRename --domain-json '{"lan": {"read": 1, "write": 1, "delete": 1}, "wan": {"read": 1, "write": 1, "delete": 1, "third_party_copy_read": 1, "third_party_copy_write": 1}}'
+rucio rse attribute add MINIO2 --key sign_url --value s3
+rucio rse attribute add MINIO2 --key s3_url_style --value path
+rucio rse attribute add MINIO2 --key verify_checksum --value False
+rucio rse attribute add MINIO2 --key skip_upload_stat --value True
+rucio rse attribute add MINIO2 --key strict_copy --value True
+rucio rse attribute add MINIO2 --key fts --value https://fts:8446
+rucio account limit add root --rse MINIO2 --bytes infinity
+
+# XRD3 has HTTP enabled, link it up to our mesh
+rucio rse distance add XRD3 MINIO1 --distance 1
+rucio rse distance add XRD3 MINIO2 --distance 1
+
+rucio rse distance add MINIO1 XRD3 --distance 1
+rucio rse distance add MINIO2 XRD3 --distance 1
+END
+
+# set up credentials for RSEs
+docker exec -i dev-rucio-1 /bin/bash <<'END'
+ID1=$(rucio rse show MINIO1 | grep '^  id:' | awk '{print$2}')
+ID2=$(rucio rse show MINIO2 | grep '^  id:' | awk '{print$2}')
+cat >/opt/rucio/etc/rse-accounts.cfg <<JSON
+{
+  "$ID1": {
+    "access_key": "admin",
+    "secret_key": "password",
+    "signature_version": "s3v4",
+    "region": "us-east-1"
+  },
+  "$ID2": {
+    "access_key": "admin",
+    "secret_key": "password",
+    "signature_version": "s3v4",
+    "region": "us-east-1"
+  }
+}
+JSON
+END

docs/operator/_resources/rucio-playground/qt-005-minio-fts-creds.txt

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+#
+# https://docs.egi.eu/users/tutorials/adhoc/data-transfer-object-storage/
+# https://fts3-docs.web.cern.ch/fts3-docs/docs/s3_support.html
+#
+
+# set up credentials for MINIO RSEs in FTS3
+docker exec -i dev-fts-1 /bin/bash <<'END'
+curl \
+  --cert /etc/grid-security/hostcert.pem \
+  --key /etc/grid-security/hostkey.pem \
+  --capath /etc/grid-security/certificates \
+  https://fts:8446/config/cloud_storage \
+  -H "Content-Type: application/json" \
+  -X POST \
+  -d '{"storage_name":"S3:minio1"}'
+
+curl \
+  --cert /etc/grid-security/hostcert.pem \
+  --key /etc/grid-security/hostkey.pem \
+  --capath /etc/grid-security/certificates \
+  https://fts:8446/config/cloud_storage \
+  -H "Content-Type: application/json" \
+  -X POST \
+  -d '{"user_dn":"/CN=Rucio User","storage_name":"S3:minio1","access_token":"admin","access_token_secret":"password"}'
+
+
+curl \
+  --cert /etc/grid-security/hostcert.pem \
+  --key /etc/grid-security/hostkey.pem \
+  --capath /etc/grid-security/certificates \
+  https://fts:8446/config/cloud_storage \
+  -H "Content-Type: application/json" \
+  -X POST \
+  -d '{"storage_name":"S3:minio2"}'
+
+curl \
+  --cert /etc/grid-security/hostcert.pem \
+  --key /etc/grid-security/hostkey.pem \
+  --capath /etc/grid-security/certificates \
+  https://fts:8446/config/cloud_storage \
+  -H "Content-Type: application/json" \
+  -X POST \
+  -d '{"user_dn":"/CN=Rucio User","storage_name":"S3:minio2","access_token":"admin","access_token_secret":"password"}'
+
+cat >/etc/gfal2.d/s3.conf <<INI
+[S3:MINIO1]
+ACCESS_KEY=admin
+SECRET_KEY=password
+REGION=us-east-1
+ALTERNATE=true
+
+[S3:MINIO2]
+ACCESS_KEY=admin
+SECRET_KEY=password
+REGION=us-east-1
+ALTERNATE=true
+INI
+END

docs/operator/_resources/rucio-playground/qt-006-simple-upload.txt

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+#
+# https://github.com/rucio/k8s-tutorial#create-initial-transfer-testing-data
+#
+
+docker exec -i dev-rucio-1 /bin/bash <<END
+dd if=/dev/urandom of=file5 bs=10M count=1
+dd if=/dev/urandom of=file6 bs=10M count=1
+
+rucio upload --rse MINIO1 --scope test file5
+rucio upload --rse MINIO2 --scope test file6
+
+rucio did add --type dataset test:dataset9
+rucio did content add -to test:dataset9 test:file5 test:file6
+END

docs/operator/_resources/rucio-playground/qt-007-replication-rules.txt

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+docker exec -i dev-rucio-1 /bin/bash <<END
+rucio rule add test:dataset9 --copies 1 --rses XRD1
+rucio rule add test:dataset2 --copies 1 --rses MINIO2
+END

docs/operator/_resources/rucio-playground/qt-008-replicte-loop.txt

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+#
+# https://github.com/rucio/k8s-tutorial#create-initial-transfer-testing-data
+#
+
+docker exec -i dev-rucio-1 /bin/bash <<END
+rucio rule list --account root
+rucio-judge-evaluator --run-once
+rucio-conveyor-submitter --run-once
+rucio-conveyor-poller --run-once  --older-than 0
+rucio-conveyor-finisher --run-once
+
+rucio rule list --account root
+END

docs/operator/_resources/rucio-playground/qt-999-down.txt

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cd `dirname $0`
+
+docker compose --file etc/docker/dev/docker-compose-qt.yml --profile storage down
+docker container prune -f
+docker volume prune -f

docs/operator/_resources/rucio-playground/qt-enter.txt

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "Run: tools/run_tests.sh -ir"
+# rucio list-file-replicas test:dataset1
+# rucio list-file-replicas test:dataset2
+# rucio list-file-replicas test:dataset3
+# rucio list-file-replicas test:container
+
+# Step zero, get a compliant proxy. The key must NOT be group/other readable
+# (KEY=$(mktemp); cat /opt/rucio/etc/userkey.pem > "$KEY"; voms-proxy-init -valid 9999:00 -cert /opt/rucio/etc/usercert.pem -key "$KEY"; rm -f "$KEY")
+
+
+docker exec -it dev-rucio-1 /bin/bash

docs/operator/_resources/rucio-playground/qt-replay.txt

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cd `dirname $0`
+for i in {0..8} ; do ./qt-00${i}-* ; done