From 9174fb663a2bb545fa91a81c3a34115b235c1ac8 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 10:40:40 -0400 Subject: [PATCH 01/13] test-sub.sh: a new testing script Setup the submissions repo and go through the entire patchset submission process, exercising all return paths of mailman/patchset.py Includes intitial, peer review, and final submissions Signed-off-by: Joel Savitz --- script-lint.sh | 1 + test-sub.sh | 503 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 504 insertions(+) create mode 100755 test-sub.sh diff --git a/script-lint.sh b/script-lint.sh index 10cf94fa..455ca2a9 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -7,6 +7,7 @@ set -ex shellcheck script-lint.sh shellcheck test.sh +shellcheck test-sub.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/test-sub.sh b/test-sub.sh new file mode 100755 index 00000000..d95648ec --- /dev/null +++ b/test-sub.sh @@ -0,0 +1,503 @@ +#!/bin/bash + +set -exuo pipefail + +SCRIPT_DIR=$(dirname "$0") + +cd "$SCRIPT_DIR" + +get_git_port() { podman port singularity_git_1 | awk -F':' '{ print $2 }' ; } ; + +# nuke grading repo inside container +# shellcheck disable=SC2016 +podman-compose exec git ash -c 'cd /var/lib/git/grading.git && for t in $(git tag); do git tag -d $t; done' + +# setup submissions repo +rm -rf repos/submissions +git/admin.sh submissions "course submissions repository" +pushd repos +git init --bare submissions +echo "course submissions repository" > submissions/description +git init submissions_init +pushd submissions_init +echo "# submissions" > README.md +git add README.md +git -c user.name=singularity -c user.email=singularity@singularity commit -m 'init submissions repo' +git push ../submissions master +popd +rm -rf submissions_init +pushd submissions +git push --mirror http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/submissions +popd +popd + +# create bob,bib,bab users or change their passwords to builder +orbit/warpdrive.sh -u bob -p builder -n || orbit/warpdrive.sh -u bob -p builder -m +orbit/warpdrive.sh -u bib -p builder -n || orbit/warpdrive.sh -u bob -p builder -m +orbit/warpdrive.sh -u bab -p builder -n || orbit/warpdrive.sh -u bob -p builder -m + +WORKDIR=$(mktemp -d) + +cat << EOF > "$WORKDIR"/setup_rubric +[{('--- /dev/null', '+++ b/bob/setup/work'): 0}, +{('--- a/bob/setup/work', '+++ b/bob/setup/work'): 0}] +EOF + +# create or recreate setup assignment +if denis/configure.sh dump | grep -q "^setup:"; then + denis/configure.sh remove -a setup +fi +denis/configure.sh create -a setup -i "$(date -d "15 secs" +%s)" -p "$(date -d "25 secs" +%s)" -f "$(date -d "30 secs" +%s)" -r "$WORKDIR"/setup_rubric +denis/configure.sh reload + +# setup assignment environment +pushd "$WORKDIR" + +mkdir certs +pushd certs +podman volume export singularity_ssl-certs > certs.tar +tar xf certs.tar +popd +git clone http://localhost:"$(get_git_port)"/submissions + +# bob env setup + +pushd submissions +git config user.name bob +git config user.email bob@localhost.localdomain +git config sendemail.smtpUser bob +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# setup_intial_bob patchsets + +# good patchset +pushd submissions +git checkout --orphan setup_initial_bob_good +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Good patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/Good patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# corrupt patchset +pushd submissions +git checkout --orphan setup_initial_bob_corrupt +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +echo "ghi" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add ghi to work' + +git format-patch --rfc --cover-letter -v1 -3 +rm ./*-0002-*.patch + +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Corrupt patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/Corrupt patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# patchset with whitespace errors +pushd submissions +git checkout --orphan setup_initial_bob_whitespace +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def " >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Patchset with whitespace errors/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/Patchset with whitespace errors/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# patchset with no cover letter +pushd submissions +git checkout --orphan setup_initial_bob_nocover +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc -v1 -2 + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# patchset with no cover letter and corrupt first patch +pushd submissions +git checkout --orphan setup_initial_bob_nocover-corrupt +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +echo "ghi" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add ghi to work' + +git format-patch --rfc -v1 -2 + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd + +# end of setup_initial_bob patchsets + +# bab env setup + +pushd submissions +git config user.name bab +git config user.email bab@localhost.localdomain +git config sendemail.smtpUser bab +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# setup_initial_bab submissions + +# bab good patchset +pushd submissions +git checkout --orphan setup_initial_bab_good +git rm -rf ./* +mkdir -p bab/setup + +echo "abc" > bab/setup/work +git add bab/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bab/setup/work +git add bab/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bab: Good patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/bab: Good patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm -rf ./*.patch +popd + +# end of bab initial submission + +# bib env setup +pushd submissions +git config user.name bib +git config user.email bib@localhost.localdomain +git config sendemail.smtpUser bib +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# bib initial submissions + +# bib good patchset +pushd submissions +git checkout --orphan setup_initial_bib_good +git rm -rf ./* +mkdir -p bib/setup + +echo "abc" > bib/setup/work +git add bib/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bib/setup/work +git add bib/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bib: Good patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/bib: Good patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# end of bib initial submission + +# investigate grading repo +git clone http://localhost:"$(get_git_port)"/grading.git +pushd grading +git fetch --tag +git tag + + +STATUSES=( +'patchset applies.' +'patch 2 failed to apply!' +'whitespace error patch 2?' +'missing cover letter!' +'missing cover letter and first patch failed to apply!' +'patchset applies.' +'patchset applies.') + +# submitted patchses should have statuses in this order +# assumption: first ID tag is first patch submitted by this script and IDs increase monotonically +i=0 +for t in $(git tag); do + git show -s --oneline "$t" | grep -q "${STATUSES[$i]}" + if [[ $i == 5 ]]; then + PEER1_ID=$t + elif [[ $i == 6 ]]; then + PEER2_ID=$t + fi + i=$((i+1)) +done + +popd + +echo "wait for initial submission deadline (10 secs)" +sleep 10 + +# bob env setup + +pushd submissions +git config user.name bob +git config user.email bob@localhost.localdomain +git config sendemail.smtpUser bob +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + + +# setup bob peer review +pushd submissions +git checkout --orphan setup_reviews_bob +git rm -rf ./* +# mkdir -p bob/setup (must be able to omit if missing) +echo ORPHAN > ORPHAN +git add ORPHAN +git commit -m 'orphan base' + +# submit peer review 1 + +cat < review1 +Subject: setup review 1 for bab +In-Reply-To: <${PEER1_ID::-1}1@localhost.localdomain> + +Looks good to me + +Acked-by: bob +PEER_REVIEW1 + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bab@localhost.localdomain review1 +rm review1 + +# end peer review 1 + +# submit peer review 2 + +cat < review2 +Subject: setup review 2 for bib +In-Reply-To: <${PEER2_ID::-1}2@localhost.localdomain> + +Looks good to me + +Acked-by: bob +PEER_REVIEW2 + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bib@localhost.localdomain review2 +rm review2 + +# end peer review 2 + +# end bob peer review +popd +sleep 1 + +# bob env setup + +pushd submissions +git config user.name bob +git config user.email bob@localhost.localdomain +git config sendemail.smtpUser bob +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# bob make final submission + +# good final patchset +pushd submissions +git checkout --orphan setup_final_bob_good +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +git format-patch --cover-letter -v2 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Good patchset final/g' v2-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/Good patchset final/g' v2-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# end bob final submission + +# bib env setup + +pushd submissions +git config user.name bib +git config user.email bib@localhost.localdomain +git config sendemail.smtpUser bib +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# bib final submission labeled RFC +pushd submissions +git checkout --orphan setup_final_bib_rfc +git rm -rf ./* +mkdir -p bib/setup + +echo "abc" > bib/setup/work +git add bib/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bib/setup/work +git add bib/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bib: RFC patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/bib: RFC patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# end of bib final submission labeled RFC + + +# bab env setup + +pushd submissions +git config user.name bab +git config user.email bab@localhost.localdomain +git config sendemail.smtpUser bab +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + + +# bab final submission in wrong directory +pushd submissions +git checkout --orphan setup_final_bab_illegal +git rm -rf ./* +mkdir -p bib/setup + +echo "abc" > work +git add work +git commit -sm 'add abc to work' + +echo "def" >> work +git add work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bab: Illegal patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/bab: Illegal patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# end of bab final submission in wrong directory + + +echo "wait for final submission deadline (10 secs)" +sleep 10 + +pushd grading +git fetch --tag +sleep 2 +git fetch -f origin refs/notes/*:refs/notes/* +git remote set-url --push origin http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/grading.git +git notes --ref=grade add setup_final_bob -m '66' +git notes --ref=grade add setup_review1_bob -m '22' +git notes --ref=grade add setup_review2_bob -m '99' +git notes --ref=feedback add setup_final_bob -m 'Good effort but try harder next time.' + +git notes --ref=grade add setup_final_bib -m '100' +git notes --ref=feedback add setup_final_bib -m 'Perfect. But no peer review!' + +git notes --ref=feedback add setup_final_bab -m 'Please review git' + +git push origin refs/notes/*:refs/notes/* + +echo "$WORKDIR" From 5b942d898ad216419778e4d68bf2d481aac407dc Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 19:24:16 -0400 Subject: [PATCH 02/13] CI: enhance PINPing by running test-sub.sh This script the project in a more integration-test-like manner. Signed-off-by: Joel Savitz --- Containerfile | 5 ++++- start.sh | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Containerfile b/Containerfile index ca3c8a18..de7b9f6e 100644 --- a/Containerfile +++ b/Containerfile @@ -11,7 +11,10 @@ RUN dnf update -y && \ python-flake8 \ python-virtualenv \ python-pip \ - git + gawk \ + socat \ + git \ + git-email # needed because for some reason newuidmap and newgidmap programs # lose their xattrs giving them caps when the container image for diff --git a/start.sh b/start.sh index 9394b8f4..99f1a3b7 100755 --- a/start.sh +++ b/start.sh @@ -18,6 +18,18 @@ podman-compose logs -f submatrix 2>&1 | sed '/Synapse now listening on TCP port if [ -f test.sh ] then ./test.sh + if [ -f test-sub.sh ] + then + podman-compose down + yes | podman volume prune + podman-compose up -d + podman-compose logs -f submatrix 2>&1 | sed '/Synapse now listening on TCP port 8008/ q' + ./dev_sockets.sh & + git config --global user.name PINP + git config --global user.email podman@podman + ./test-sub.sh + + fi else virtualenv . pip install -r requirements.txt From 76b69b2cf02f3638c86e407de245c9bf07909731 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 20:08:36 -0400 Subject: [PATCH 03/13] test-sub-check.sh: add script to check results of test-sub.sh test-sub.sh is good for interactive testing, but lacks validation when automated. Add test-sub-check.sh to run some basic checks that make sure test-sub.sh ran smoothly. Signed-off-by: Joel Savitz --- script-lint.sh | 1 + start.sh | 1 + test-sub-check.sh | 118 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100755 test-sub-check.sh diff --git a/script-lint.sh b/script-lint.sh index 455ca2a9..3e259437 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -8,6 +8,7 @@ set -ex shellcheck script-lint.sh shellcheck test.sh shellcheck test-sub.sh +shellcheck test-sub-check.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/start.sh b/start.sh index 99f1a3b7..9df6a678 100755 --- a/start.sh +++ b/start.sh @@ -28,6 +28,7 @@ then git config --global user.name PINP git config --global user.email podman@podman ./test-sub.sh + ./test-sub-check.sh fi else diff --git a/test-sub-check.sh b/test-sub-check.sh new file mode 100755 index 00000000..2e38b9a6 --- /dev/null +++ b/test-sub-check.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +CURL_OPTS=( \ +--verbose \ +--cacert test/ca_cert.pem \ +--fail \ +--no-progress-meter \ +) + +set -exuo pipefail + +HOSTNAME_FROM_DOTENV="$(sh -c ' +set -o allexport +. ./.env +exec jq -r -n "env.SINGULARITY_HOSTNAME" +')" + +DOCKER=${DOCKER:-podman} + +SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} + +# Create test dir if it does not exist yet +mkdir -p test + +# Reset the test directory +rm -f test/* + +${DOCKER} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem + +# login as bob +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies \ + --data "username=bob&password=builder" \ + | tee test/login_success \ + | grep "msg = bob authenticated by password" + + +# check for setup assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard \ + | grep "setup" + +grep "Total Score: 64.9" test/dashboard + +grep "missing cover letter and first patch failed to apply!" test/dashboard + +grep "bib Peer Review" test/dashboard + +grep "bab Peer Review" test/dashboard + +grep "patchset applies." test/dashboard + +grep "Good effort but try harder next time." test/dashboard + +# login as bab +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies_bab \ + --data "username=bab&password=builder" \ + | tee test/login_success_bab \ + | grep "msg = bab authenticated by password" + + +# check for setup assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies_bab \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard_bab \ + | grep "setup" + +grep "Total Score: 0.0" test/dashboard_bab + +grep "bib Peer Review" test/dashboard_bab + +grep "bob Peer Review" test/dashboard_bab + +grep "patchset applies." test/dashboard_bab + +grep "illegal patch 1: permission denied for path work!" test/dashboard_bab + +grep 'Please review git' test/dashboard_bab + +# login as bib +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies_bib \ + --data "username=bib&password=builder" \ + | tee test/login_success_bib \ + | grep "msg = bib authenticated by password" + + +# check for setup assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies_bib \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard_bib \ + | grep "setup" + +grep "Total Score: 80.0" test/dashboard_bib + +grep "bob Peer Review" test/dashboard_bib + +grep "bab Peer Review" test/dashboard_bib + +grep "patchset applies." test/dashboard_bib + +grep 'Perfect. But no peer review!' test/dashboard_bib + +echo "ALL SUBMISSION TESTS PASS" From 91ac6c81724c058586b2920a4c63e014f93eb012 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 22:01:18 -0400 Subject: [PATCH 04/13] treewide: s/DOCKER/PODMAN/g We use podman around these parts Signed-off-by: Joel Savitz --- git/admin.sh | 4 ++-- orbit/warpdrive.sh | 4 ++-- test-sub-check.sh | 4 ++-- test.sh | 16 ++++++++-------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/git/admin.sh b/git/admin.sh index f3a9e46e..bbd5189a 100755 --- a/git/admin.sh +++ b/git/admin.sh @@ -1,6 +1,6 @@ #!/bin/sh set -e -DOCKER_COMPOSE=${DOCKER_COMPOSE:-podman-compose} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} -$DOCKER_COMPOSE exec git ./create-repo.sh "$@" +$PODMAN_COMPOSE exec git ./create-repo.sh "$@" diff --git a/orbit/warpdrive.sh b/orbit/warpdrive.sh index 8d92572d..c78e8158 100755 --- a/orbit/warpdrive.sh +++ b/orbit/warpdrive.sh @@ -4,6 +4,6 @@ set -e -DOCKER_COMPOSE=${DOCKER_COMPOSE:-podman-compose} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} -$DOCKER_COMPOSE exec orbit ./hyperspace.py "$@" +$PODMAN_COMPOSE exec orbit ./hyperspace.py "$@" diff --git a/test-sub-check.sh b/test-sub-check.sh index 2e38b9a6..894de144 100755 --- a/test-sub-check.sh +++ b/test-sub-check.sh @@ -15,7 +15,7 @@ set -o allexport exec jq -r -n "env.SINGULARITY_HOSTNAME" ')" -DOCKER=${DOCKER:-podman} +PODMAN=${PODMAN:-podman} SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} @@ -25,7 +25,7 @@ mkdir -p test # Reset the test directory rm -f test/* -${DOCKER} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem # login as bob curl --url "https://$SINGULARITY_HOSTNAME/login" \ diff --git a/test.sh b/test.sh index 7e13f23f..2b5d9430 100755 --- a/test.sh +++ b/test.sh @@ -10,15 +10,15 @@ # of the furthest right failing command or zero if no command failed (o pipefail) set -exuo pipefail -DOCKER=${DOCKER:-podman} -DOCKER_COMPOSE=${DOCKER_COMPOSE:-podman-compose} +PODMAN=${PODMAN:-podman} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} require() { command -v "$1" > /dev/null || { echo "error: $1 command required yet absent" ; exit 1 ; } ; } require curl require jq require flake8 -require "${DOCKER}" -require "${DOCKER_COMPOSE}" +require "${PODMAN}" +require "${PODMAN_COMPOSE}" # Check for shell script style compliance with shellcheck ./script-lint.sh @@ -40,7 +40,7 @@ exec jq -r -n "env.SINGULARITY_HOSTNAME" SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} -${DOCKER} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem CURL_OPTS=( \ --verbose \ @@ -150,10 +150,10 @@ curl --url "pop3s://$SINGULARITY_HOSTNAME" \ orbit/warpdrive.sh -u resu -p ssap -n # Limit `resu`'s access to the empty inbox -${DOCKER_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -d resu +${PODMAN_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -d resu # Update list of email to include new message -${DOCKER_COMPOSE} exec denis /usr/local/bin/init_journal /var/lib/email/journal/journal /var/lib/email/journal/temp /var/lib/email/mail +${PODMAN_COMPOSE} exec denis /usr/local/bin/init_journal /var/lib/email/journal/journal /var/lib/email/journal/temp /var/lib/email/mail # Check that the user can get the most recent message sent to the server curl --url "pop3s://$SINGULARITY_HOSTNAME/1" \ @@ -173,7 +173,7 @@ curl --url "pop3s://$SINGULARITY_HOSTNAME" \ # Remove limit on `resu`'s access to the inbox -${DOCKER_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -a resu +${PODMAN_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -a resu # Check that `resu` can now get the most recent message sent to the server curl --url "pop3s://$SINGULARITY_HOSTNAME/1" \ From 10eced620998207e2d78d8c90e35bcc78d8ac44d Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 22:28:42 -0400 Subject: [PATCH 05/13] test: consolidate repetitive setup in test-lib test.sh and test-sub-check.sh now source test-lib script-lint.sh modified to follow source commands for those two files. Signed-off-by: Joel Savitz --- script-lint.sh | 4 ++-- test-lib | 29 +++++++++++++++++++++++++++++ test-sub-check.sh | 25 ++----------------------- test.sh | 26 ++------------------------ 4 files changed, 35 insertions(+), 49 deletions(-) create mode 100644 test-lib diff --git a/script-lint.sh b/script-lint.sh index 3e259437..0f68fbe6 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -6,9 +6,9 @@ require shellcheck set -ex shellcheck script-lint.sh -shellcheck test.sh +shellcheck -x test.sh shellcheck test-sub.sh -shellcheck test-sub-check.sh +shellcheck -x test-sub-check.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/test-lib b/test-lib new file mode 100644 index 00000000..a29d3963 --- /dev/null +++ b/test-lib @@ -0,0 +1,29 @@ +PODMAN=${PODMAN:-podman} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} + +HOSTNAME_FROM_DOTENV="$(sh -c ' +set -o allexport +. ./.env +exec jq -r -n "env.SINGULARITY_HOSTNAME" +')" + +SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} + +setup_testdir() { + # Create test dir if it does not exist yet + mkdir -p test + + # Reset the test directory + rm -f test/* + + # put the cert in there + ${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +} + +CURL_OPTS=( \ +--verbose \ +--cacert test/ca_cert.pem \ +--fail \ +--no-progress-meter \ +) + diff --git a/test-sub-check.sh b/test-sub-check.sh index 894de144..ae42ac92 100755 --- a/test-sub-check.sh +++ b/test-sub-check.sh @@ -1,31 +1,10 @@ #!/usr/bin/env bash -CURL_OPTS=( \ ---verbose \ ---cacert test/ca_cert.pem \ ---fail \ ---no-progress-meter \ -) - set -exuo pipefail -HOSTNAME_FROM_DOTENV="$(sh -c ' -set -o allexport -. ./.env -exec jq -r -n "env.SINGULARITY_HOSTNAME" -')" - -PODMAN=${PODMAN:-podman} - -SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} - -# Create test dir if it does not exist yet -mkdir -p test - -# Reset the test directory -rm -f test/* +source test-lib -${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +setup_testdir # login as bob curl --url "https://$SINGULARITY_HOSTNAME/login" \ diff --git a/test.sh b/test.sh index 2b5d9430..c28d13da 100755 --- a/test.sh +++ b/test.sh @@ -10,8 +10,7 @@ # of the furthest right failing command or zero if no command failed (o pipefail) set -exuo pipefail -PODMAN=${PODMAN:-podman} -PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} +source test-lib require() { command -v "$1" > /dev/null || { echo "error: $1 command required yet absent" ; exit 1 ; } ; } require curl @@ -26,28 +25,7 @@ require "${PODMAN_COMPOSE}" # Check python style compliance flake8 -# Create test dir if it does not exist yet -mkdir -p test - -# Reset the test directory -rm -f test/* - -HOSTNAME_FROM_DOTENV="$(sh -c ' -set -o allexport -. ./.env -exec jq -r -n "env.SINGULARITY_HOSTNAME" -')" - -SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} - -${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem - -CURL_OPTS=( \ ---verbose \ ---cacert test/ca_cert.pem \ ---fail \ ---no-progress-meter \ -) +setup_testdir # Check that registration fails before user creation curl --url "https://$SINGULARITY_HOSTNAME/register" \ From 7cb495e0fb10d2b44e3612df9cae2ed07740d4bc Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Sun, 1 Jun 2025 22:21:32 -0400 Subject: [PATCH 06/13] test: add more rubric tests cleanly by consolidating common testing code Add test-sub2.sh which tests out 12 invalid and 1 valid submission on a rubric for an example coding assignment and validate the resulting status values added to the email ID tags in the grading repo. Simplify the new tests by abstracting the setup section and repetitive commands, enabling quicker development of shorter future testing scripts the same flavor as the rest of test-sub*.sh. This is important as we want to make sure our automated grading features actually do what we think they will do to the fullest extent possible before this code goes into production with real students. Signed-off-by: Joel Savitz --- script-lint.sh | 5 +- start.sh | 1 + test-lib | 166 +++++++++++++++++ test-sub-check.sh | 2 - test-sub.sh | 457 ++++++++-------------------------------------- test-sub2.sh | 154 ++++++++++++++++ test.sh | 8 - 7 files changed, 400 insertions(+), 393 deletions(-) create mode 100755 test-sub2.sh diff --git a/script-lint.sh b/script-lint.sh index 0f68fbe6..2731cad9 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -5,10 +5,12 @@ require shellcheck set -ex +# -x needed to make shellcheck follow `source` command shellcheck script-lint.sh shellcheck -x test.sh -shellcheck test-sub.sh +shellcheck -x test-sub.sh shellcheck -x test-sub-check.sh +shellcheck -x test-sub2.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh @@ -18,6 +20,5 @@ shellcheck git/setup-repo.sh shellcheck git/cgi-bin/git-receive-pack shellcheck git/hooks/post-update -# -x needed to make shellcheck follow `source` command shellcheck -x backup/backup.sh shellcheck -x backup/restore.sh diff --git a/start.sh b/start.sh index 9df6a678..07b38725 100755 --- a/start.sh +++ b/start.sh @@ -29,6 +29,7 @@ then git config --global user.email podman@podman ./test-sub.sh ./test-sub-check.sh + ./test-sub2.sh fi else diff --git a/test-lib b/test-lib index a29d3963..1b9607fb 100644 --- a/test-lib +++ b/test-lib @@ -1,5 +1,15 @@ +# This line: +# - aborts the script after any pipeline returns nonzero (e) +# - shows all commands as they are run (x) +# - sets any dereference of an unset variable to trigger an error (u) +# - causes the return value of a pipeline to be the nonzero return value +# of the furthest right failing command or zero if no command failed (o pipefail) +set -exuo pipefail + PODMAN=${PODMAN:-podman} PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} +SCRIPT_DIR=$(dirname "$(readlink -f "$0")") +WORKDIR=$(mktemp -d) HOSTNAME_FROM_DOTENV="$(sh -c ' set -o allexport @@ -27,3 +37,159 @@ CURL_OPTS=( \ --no-progress-meter \ ) + +get_git_port() { ${PODMAN} port singularity_git_1 | awk -F':' '{ print $2 }' ; } ; + +# preconditions: +# - SCRIPT_DIR defined (singularity repo root) +# - WORKDIR defined (arbitrary) temp directory +setup_submissions_and_grading_repo() { + pushd "$SCRIPT_DIR" + # nuke grading repo inside container + # shellcheck disable=SC2016 + ${PODMAN_COMPOSE} exec git ash -c 'cd /var/lib/git/grading.git && for t in $(git tag); do git tag -d $t; done' + # crete and push fresh submissions repo + rm -rf repos/submissions + git/admin.sh submissions "course submissions repository" + pushd repos + git init --bare submissions + echo "course submissions repository" > submissions/description + # create a temporary workdir to push an initial commit to the submission repo + git init submissions_init + pushd submissions_init + echo "# submissions" > README.md + git add README.md + git status + git -c user.name=singularity -c user.email=singularity@singularity commit -sm 'init submissions repo' + git push ../submissions master + popd + rm -rf submissions_init + pushd submissions + git push --mirror http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/submissions + popd + popd + popd + + pushd "$WORKDIR" + mkdir certs + pushd certs + ${PODMAN} volume export singularity_ssl-certs > certs.tar + tar xf certs.tar + popd + git clone http://localhost:"$(get_git_port)"/submissions + popd +} + +create_lighting_assignment() { + test -n "$1" + test -n "$2" + test -n "$3" + test -n "$4" + + local asn="$1" + local i_s="$2" + local p_s="$3" + local f_s="$4" + set +u + local rub="$5" + set -u + + RUBRIC= + if [ -n "$rub" ] + then + RUBRIC="-r $rub" + fi + + pushd "$SCRIPT_DIR" + # create or recreate setup assignment + if denis/configure.sh dump | grep -q "^$asn:"; then + denis/configure.sh remove -a "$asn" + fi + denis/configure.sh create -a "$asn" -i "$(date -d "$i_s secs" +%s)" -p "$(date -d "$p_s secs" +%s)" -f "$(date -d "$f_s secs" +%s)" ${RUBRIC} + denis/configure.sh reload + popd +} + +# preconditions: +# - called after setup_submissions_and_grading_repo +setup_submissions_for() { + test -n "$1" + local user="$1" + + pushd "$SCRIPT_DIR" + orbit/warpdrive.sh -u "$user" -p builder -n || orbit/warpdrive.sh -u "$user" -p builder -m + popd + + pushd "$WORKDIR"/submissions + git config user.name "$user" + git config user.email "$user"@localhost.localdomain + git config sendemail.smtpUser "$user" + git config sendemail.smtpPass builder + git config sendemail.smtpserver localhost.localdomain + git config sendemail.smtpserverport 465 + git config sendemail.smtpencryption ssl + popd +} + +# preconditions: +# - no non-tracked files in submissions repo +# - called after setup_submissions_for +enter_and_checkout() { + test -n "$1" + local branch="$1" + + pushd "$WORKDIR"/submissions + git checkout --orphan "$1" + if [ -n "$(ls)" ] + then + git rm -rf ./* + fi +} + +# preconditions: +# - called after a call to enter_and_checkout +exit_after_sending() { + test -n "$1" + local asn="$1" + + git send-email \ + --confirm=never \ + --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem \ + --to "$asn"@localhost.localdomain \ + ./*.patch + rm ./*.patch + popd + sleep 1 +} + +# preconditions: +# - called after a call to enter_and_checkout +write_commit_to() { + test -n "$1" && test -n "$2" + local content="$1" + local file="$2" + set +u # necessary since $3 is unbound in 2 arg case + local opt="$3" + set -x + + mkdir -p $(dirname "$file") + + if [ "$opt" == "append" ]; then + echo "$content" >> "$file" + else + echo "$content" > "$file" + fi + + git add "$file" + git commit -sm "add $content to $file" +} + +# preconditions: +# - called after a call to write_commit_to +fixup_cover() { + test -n "$1" + sed -i "s/\*\*\* SUBJECT HERE \*\*\*/$1/g" *0000-cover-letter.patch + sed -i "s/\*\*\* BLURB HERE \*\*\*/$1/g" *0000-cover-letter.patch + sed -i "\$a\\Signed-off-by: $(git config user.name) <$(git config user.email)>" *0000-cover-letter.patch +} + diff --git a/test-sub-check.sh b/test-sub-check.sh index ae42ac92..4f49ae3d 100755 --- a/test-sub-check.sh +++ b/test-sub-check.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -set -exuo pipefail - source test-lib setup_testdir diff --git a/test-sub.sh b/test-sub.sh index d95648ec..b8ae94b0 100755 --- a/test-sub.sh +++ b/test-sub.sh @@ -1,283 +1,90 @@ #!/bin/bash -set -exuo pipefail - -SCRIPT_DIR=$(dirname "$0") - -cd "$SCRIPT_DIR" - -get_git_port() { podman port singularity_git_1 | awk -F':' '{ print $2 }' ; } ; - -# nuke grading repo inside container -# shellcheck disable=SC2016 -podman-compose exec git ash -c 'cd /var/lib/git/grading.git && for t in $(git tag); do git tag -d $t; done' - -# setup submissions repo -rm -rf repos/submissions -git/admin.sh submissions "course submissions repository" -pushd repos -git init --bare submissions -echo "course submissions repository" > submissions/description -git init submissions_init -pushd submissions_init -echo "# submissions" > README.md -git add README.md -git -c user.name=singularity -c user.email=singularity@singularity commit -m 'init submissions repo' -git push ../submissions master -popd -rm -rf submissions_init -pushd submissions -git push --mirror http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/submissions -popd -popd +# assumes singularity is running when invoked +# script is written to be as idempotent as posible -# create bob,bib,bab users or change their passwords to builder -orbit/warpdrive.sh -u bob -p builder -n || orbit/warpdrive.sh -u bob -p builder -m -orbit/warpdrive.sh -u bib -p builder -n || orbit/warpdrive.sh -u bob -p builder -m -orbit/warpdrive.sh -u bab -p builder -n || orbit/warpdrive.sh -u bob -p builder -m +source test-lib -WORKDIR=$(mktemp -d) +setup_submissions_and_grading_repo cat << EOF > "$WORKDIR"/setup_rubric [{('--- /dev/null', '+++ b/bob/setup/work'): 0}, {('--- a/bob/setup/work', '+++ b/bob/setup/work'): 0}] EOF -# create or recreate setup assignment -if denis/configure.sh dump | grep -q "^setup:"; then - denis/configure.sh remove -a setup -fi -denis/configure.sh create -a setup -i "$(date -d "15 secs" +%s)" -p "$(date -d "25 secs" +%s)" -f "$(date -d "30 secs" +%s)" -r "$WORKDIR"/setup_rubric -denis/configure.sh reload - -# setup assignment environment -pushd "$WORKDIR" - -mkdir certs -pushd certs -podman volume export singularity_ssl-certs > certs.tar -tar xf certs.tar -popd -git clone http://localhost:"$(get_git_port)"/submissions - -# bob env setup - -pushd submissions -git config user.name bob -git config user.email bob@localhost.localdomain -git config sendemail.smtpUser bob -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd +create_lighting_assignment setup 15 25 30 "$WORKDIR"/setup_rubric # setup_intial_bob patchsets +setup_submissions_for bob # good patchset -pushd submissions -git checkout --orphan setup_initial_bob_good -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bob_good +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Good patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/Good patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 +fixup_cover "Good patchset" +exit_after_sending setup # corrupt patchset -pushd submissions -git checkout --orphan setup_initial_bob_corrupt -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - -echo "ghi" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add ghi to work' - +enter_and_checkout setup_initial_bob_corrupt +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" +write_commit_to "ghi" bob/setup/work "append" git format-patch --rfc --cover-letter -v1 -3 rm ./*-0002-*.patch - -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Corrupt patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/Corrupt patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 +fixup_cover "Corrupt patchset" +exit_after_sending setup # patchset with whitespace errors -pushd submissions -git checkout --orphan setup_initial_bob_whitespace -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def " >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bob_whitespace +write_commit_to "abc" bob/setup/work +write_commit_to "def " bob/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Patchset with whitespace errors/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/Patchset with whitespace errors/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 +fixup_cover "Patchset with whitespace errors" +exit_after_sending setup # patchset with no cover letter -pushd submissions -git checkout --orphan setup_initial_bob_nocover -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bob_nocover +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" git format-patch --rfc -v1 -2 - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 +exit_after_sending setup # patchset with no cover letter and corrupt first patch -pushd submissions -git checkout --orphan setup_initial_bob_nocover-corrupt -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - -echo "ghi" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add ghi to work' - +enter_and_checkout setup_initial_bob_nocover-corrupt +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" +write_commit_to "ghi" bob/setup/work "append" git format-patch --rfc -v1 -2 - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd - -# end of setup_initial_bob patchsets - -# bab env setup - -pushd submissions -git config user.name bab -git config user.email bab@localhost.localdomain -git config sendemail.smtpUser bab -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd +exit_after_sending setup # setup_initial_bab submissions +setup_submissions_for bab # bab good patchset -pushd submissions -git checkout --orphan setup_initial_bab_good -git rm -rf ./* -mkdir -p bab/setup - -echo "abc" > bab/setup/work -git add bab/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bab/setup/work -git add bab/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bab_good +write_commit_to "abc" bab/setup/work +write_commit_to "def" bab/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bab: Good patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/bab: Good patchset/g' v1-0000-cover-letter.patch +fixup_cover "Good patchset" +exit_after_sending setup -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm -rf ./*.patch -popd - -# end of bab initial submission - -# bib env setup -pushd submissions -git config user.name bib -git config user.email bib@localhost.localdomain -git config sendemail.smtpUser bib -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd - -# bib initial submissions +# setup_initial_bib submissions +setup_submissions_for bib # bib good patchset -pushd submissions -git checkout --orphan setup_initial_bib_good -git rm -rf ./* -mkdir -p bib/setup - -echo "abc" > bib/setup/work -git add bib/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bib/setup/work -git add bib/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bib_good +write_commit_to "abc" bib/setup/work +write_commit_to "def" bib/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bib: Good patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/bib: Good patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 - -# end of bib initial submission +fixup_cover "Good patchset" +exit_after_sending setup # investigate grading repo +pushd "$WORKDIR" git clone http://localhost:"$(get_git_port)"/grading.git pushd grading git fetch --tag -git tag - STATUSES=( 'patchset applies.' @@ -300,36 +107,17 @@ for t in $(git tag); do fi i=$((i+1)) done - popd -echo "wait for initial submission deadline (10 secs)" -sleep 10 - -# bob env setup - -pushd submissions -git config user.name bob -git config user.email bob@localhost.localdomain -git config sendemail.smtpUser bob -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd +echo "wait for initial submission deadline (5 secs)" +sleep 5 +setup_submissions_for bob # setup bob peer review -pushd submissions -git checkout --orphan setup_reviews_bob -git rm -rf ./* -# mkdir -p bob/setup (must be able to omit if missing) -echo ORPHAN > ORPHAN -git add ORPHAN -git commit -m 'orphan base' +enter_and_checkout setup_reviews_bob # submit peer review 1 - cat < review1 Subject: setup review 1 for bab In-Reply-To: <${PEER1_ID::-1}1@localhost.localdomain> @@ -338,14 +126,10 @@ Looks good to me Acked-by: bob PEER_REVIEW1 - git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bab@localhost.localdomain review1 rm review1 -# end peer review 1 - # submit peer review 2 - cat < review2 Subject: setup review 2 for bib In-Reply-To: <${PEER2_ID::-1}2@localhost.localdomain> @@ -354,131 +138,39 @@ Looks good to me Acked-by: bob PEER_REVIEW2 - git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bib@localhost.localdomain review2 rm review2 -# end peer review 2 - -# end bob peer review popd sleep 1 +# end bob peer review -# bob env setup - -pushd submissions -git config user.name bob -git config user.email bob@localhost.localdomain -git config sendemail.smtpUser bob -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd - -# bob make final submission - -# good final patchset -pushd submissions -git checkout --orphan setup_final_bob_good -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - +# setup_final_bob good patchset +setup_submissions_for bob +enter_and_checkout setup_final_bob_good +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" git format-patch --cover-letter -v2 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Good patchset final/g' v2-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/Good patchset final/g' v2-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 - -# end bob final submission - -# bib env setup - -pushd submissions -git config user.name bib -git config user.email bib@localhost.localdomain -git config sendemail.smtpUser bib -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd - -# bib final submission labeled RFC -pushd submissions -git checkout --orphan setup_final_bib_rfc -git rm -rf ./* -mkdir -p bib/setup - -echo "abc" > bib/setup/work -git add bib/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bib/setup/work -git add bib/setup/work -git commit -sm 'add def to work' - +fixup_cover "Good final patchset" +exit_after_sending setup + +# setup_final_bib final submission incorrectly labeled RFC +setup_submissions_for bib +enter_and_checkout setup_final_bib_rfc +write_commit_to "abc" bib/setup/work +write_commit_to "def" bib/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bib: RFC patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/bib: RFC patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 - -# end of bib final submission labeled RFC - - -# bab env setup - -pushd submissions -git config user.name bab -git config user.email bab@localhost.localdomain -git config sendemail.smtpUser bab -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd - - -# bab final submission in wrong directory -pushd submissions -git checkout --orphan setup_final_bab_illegal -git rm -rf ./* -mkdir -p bib/setup - -echo "abc" > work -git add work -git commit -sm 'add abc to work' - -echo "def" >> work -git add work -git commit -sm 'add def to work' - +fixup_cover "RFC tagged final patchset" +exit_after_sending setup + +# setup_final_bab submission in wrong directory +setup_submissions_for bab +enter_and_checkout setup_final_bab_illegal +write_commit_to "abc" work +write_commit_to "def" work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bab: Illegal patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/bab: Illegal patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 - -# end of bab final submission in wrong directory - +fixup_cover "Illegal patchset" +exit_after_sending setup echo "wait for final submission deadline (10 secs)" sleep 10 @@ -488,14 +180,17 @@ git fetch --tag sleep 2 git fetch -f origin refs/notes/*:refs/notes/* git remote set-url --push origin http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/grading.git + git notes --ref=grade add setup_final_bob -m '66' git notes --ref=grade add setup_review1_bob -m '22' git notes --ref=grade add setup_review2_bob -m '99' git notes --ref=feedback add setup_final_bob -m 'Good effort but try harder next time.' +# automatic 0 for peer reviews git notes --ref=grade add setup_final_bib -m '100' git notes --ref=feedback add setup_final_bib -m 'Perfect. But no peer review!' +# automatic 0 for peer reviews and final submission git notes --ref=feedback add setup_final_bab -m 'Please review git' git push origin refs/notes/*:refs/notes/* diff --git a/test-sub2.sh b/test-sub2.sh new file mode 100755 index 00000000..80e5e733 --- /dev/null +++ b/test-sub2.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +# test a bunch of rubric violations + +source test-lib + +setup_submissions_and_grading_repo + +cat << EOF > "$WORKDIR"/coding_rubric +[{('--- /dev/null', '+++ b/user/coding/program.c'): 0}, +{('--- a/user/coding/program.c', '+++ b/user/coding/program.c'): 0}, +{('--- /dev/null', '+++ b/user/coding/Makefile'): 0}] +EOF + +create_lighting_assignment coding 25 28 30 "$WORKDIR"/coding_rubric + +setup_submissions_for bob + +enter_and_checkout coding_initial_bob_toomany +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Yet more C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -4 +fixup_cover "Too many patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_toofew +write_commit_to "C program" bob/coding/program.c +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -2 +fixup_cover "Too few patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_top-level-first +write_commit_to "C program" program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Top level first file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_top-level-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Top level third file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_second-level-first +write_commit_to "C program" bob/program.c +write_commit_to "More C program" bob/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Second level first file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_second-level-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Second level third file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_fourth-level-first +write_commit_to "C program" bob/coding/program/program.c +write_commit_to "More C program" bob/coding/program/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Fourth level first file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_fourth-level-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/program/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Fourth level third file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-second-dir-first +write_commit_to "C program" bob/setup/program.c +write_commit_to "More C program" bob/setup/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong second directory first patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-second-dir-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/setup/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong second directory third patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-filename-first +write_commit_to "C program" bob/coding/hello.c +write_commit_to "More C program" bob/coding/hello.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong filename first patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-filename-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/Snakefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong filename third patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_good +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Good patches" +exit_after_sending coding + +pushd "$WORKDIR" +git clone http://localhost:"$(get_git_port)"/grading.git +pushd grading + +STATUSES=( +"patch count 4 violates expected rubric patch count of 3!" +"patch count 2 violates expected rubric patch count of 3!" +"illegal patch 1: permission denied for path program.c!" +"illegal patch 3: permission denied for path Makefile!" +"patch 1 violates the assignment rubric!" +"patch 3 violates the assignment rubric!" +"patch 1 violates the assignment rubric!" +"patch 3 violates the assignment rubric!" +"patch 1 violates the assignment rubric!" +"patch 3 violates the assignment rubric!" +"patch 1 violates the assignment rubric!" +"patch 3 violates the assignment rubric!" +"patchset applies." +) + +# submitted patchses should have statuses in this order +# assumption: first ID tag is first patch submitted by this script and IDs increase monotonically +i=0 +for t in $(git tag); do + git show -s --oneline "$t" | grep -q "${STATUSES[$i]}" + i=$((i+1)) +done +popd +popd + +echo "RUBRIC CHECKS PASS" +echo "$WORKDIR" diff --git a/test.sh b/test.sh index c28d13da..f87528c9 100755 --- a/test.sh +++ b/test.sh @@ -2,14 +2,6 @@ # Testing script for singularity and orbit -# This line: -# - aborts the script after any pipeline returns nonzero (e) -# - shows all commands as they are run (x) -# - sets any dereference of an unset variable to trigger an error (u) -# - causes the return value of a pipeline to be the nonzero return value -# of the furthest right failing command or zero if no command failed (o pipefail) -set -exuo pipefail - source test-lib require() { command -v "$1" > /dev/null || { echo "error: $1 command required yet absent" ; exit 1 ; } ; } From 36430796473a93ddb53bac292a9c9f27144766cb Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Mon, 2 Jun 2025 15:02:54 -0400 Subject: [PATCH 07/13] test: add test for whether only initial submission gets autozero If a student makes an inital submission but not a final submission, they should automatically receive a zero for the assignment. Signed-off-by: Joel Savitz --- script-lint.sh | 1 + start.sh | 7 +++++++ test-sub3.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100755 test-sub3.sh diff --git a/script-lint.sh b/script-lint.sh index 2731cad9..d9823ae4 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -11,6 +11,7 @@ shellcheck -x test.sh shellcheck -x test-sub.sh shellcheck -x test-sub-check.sh shellcheck -x test-sub2.sh +shellcheck -x test-sub3.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/start.sh b/start.sh index 07b38725..b428e225 100755 --- a/start.sh +++ b/start.sh @@ -29,7 +29,14 @@ then git config --global user.email podman@podman ./test-sub.sh ./test-sub-check.sh + podman-compose down + yes | podman volume prune + podman-compose up -d ./test-sub2.sh + podman-compose down + yes | podman volume prune + podman-compose up -d + ./test-sub3.sh fi else diff --git a/test-sub3.sh b/test-sub3.sh new file mode 100755 index 00000000..55b28c30 --- /dev/null +++ b/test-sub3.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# test only initial submission results in automatic 0 + +source test-lib + +setup_submissions_and_grading_repo + +create_lighting_assignment fun 5 7 10 + +setup_submissions_for bob + +enter_and_checkout fun_initial_bob_only-initial +write_commit_to "C program" bob/fun/program.c +write_commit_to "More C program" bob/fun/program.c "append" +write_commit_to "Yet more C program" bob/fun/program.c "append" +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Only initial sub patchset" +exit_after_sending fun + +# wait for initial submission deadline +sleep 9 + +setup_testdir + +# login as bob +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies \ + --data "username=bob&password=builder" \ + | tee test/login_success \ + | grep "msg = bob authenticated by password" + + +# check for fun assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard \ + | grep "fun" + +grep "patchset applies." test/dashboard +grep "No submission" test/dashboard +grep "Total Score: 0.0" test/dashboard + +echo "INITIAL SUBMISSION ONLY GETS AUTO ZERO CONFIRMED" From d08ad18191ec9f9e2d36cee8d0ad58a7ee0480c2 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Wed, 9 Jul 2025 18:08:16 -0400 Subject: [PATCH 08/13] test: add test-sub4 to check peer review corner cases Check that total score is calculated correctly when only two students submit and only one review is required or no students submit so no reviews are required. Signed-off-by: Joel Savitz --- script-lint.sh | 1 + start.sh | 4 ++ test-sub4.sh | 116 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+) create mode 100755 test-sub4.sh diff --git a/script-lint.sh b/script-lint.sh index d9823ae4..ef2d49ad 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -12,6 +12,7 @@ shellcheck -x test-sub.sh shellcheck -x test-sub-check.sh shellcheck -x test-sub2.sh shellcheck -x test-sub3.sh +shellcheck -x test-sub4.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/start.sh b/start.sh index b428e225..9649b7bc 100755 --- a/start.sh +++ b/start.sh @@ -37,6 +37,10 @@ then yes | podman volume prune podman-compose up -d ./test-sub3.sh + podman-compose down + yes | podman volume prune + podman-compose up -d + ./test-sub4.sh fi else diff --git a/test-sub4.sh b/test-sub4.sh new file mode 100755 index 00000000..76c3f65c --- /dev/null +++ b/test-sub4.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +# test total score calculation when assigned only 0 or 1 peer reviews legitimately + +source test-lib + +setup_submissions_and_grading_repo + +# single peer review case + +cat << EOF > "$WORKDIR"/first_rubric +[{('--- /dev/null', '+++ b/bob/first/work'): 0}, +{('--- a/bob/first/work', '+++ b/bob/first/work'): 0}] +EOF + +create_lighting_assignment first 10 11 20 "$WORKDIR"/first_rubric + +setup_submissions_for bob + +enter_and_checkout first_initial_bob_good +write_commit_to "abc" bob/first/work +write_commit_to "def" bob/first/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Good patchset" +exit_after_sending first + +setup_submissions_for bab + +enter_and_checkout first_initial_bab_good +write_commit_to "abc" bab/first/work +write_commit_to "def" bab/first/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Good patchset" +exit_after_sending first + +sleep 8 + +setup_submissions_for bob +enter_and_checkout first_final_bob_good +write_commit_to "abc" bob/first/work +write_commit_to "def" bob/first/work "append" +git format-patch --cover-letter -v2 -2 +fixup_cover "Good final patchset" +exit_after_sending first + +sleep 9 + +# no peer review case + +cat << EOF > "$WORKDIR"/second_rubric +[{('--- /dev/null', '+++ b/bob/second/work'): 0}, +{('--- a/bob/second/work', '+++ b/bob/second/work'): 0}] +EOF + +create_lighting_assignment second 5 7 10 "$WORKDIR"/second_rubric + +setup_submissions_for bob + +enter_and_checkout second_initial_bob_good +write_commit_to "abc" bob/second/work +write_commit_to "def" bob/second/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Good patchset" +exit_after_sending second + +sleep 4 + +enter_and_checkout second_final_bob_good +write_commit_to "abc" bob/second/work +write_commit_to "def" bob/second/work "append" +git format-patch --cover-letter -v2 -2 +fixup_cover "Good final patchset" +exit_after_sending second + +sleep 4 + +pushd "$WORKDIR" +git clone http://localhost:"$(get_git_port)"/grading.git +pushd grading + +git fetch --tag +sleep 2 +git fetch -f origin refs/notes/*:refs/notes/* +git remote set-url --push origin http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/grading.git + +git notes --ref=grade add first_final_bob -m '100' +git notes --ref=grade add first_review1_bob -fm '100' +git notes --ref=grade add second_final_bob -m '50' +git push origin refs/notes/*:refs/notes/* + +popd +popd + +# login as bob +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies \ + --data "username=bob&password=builder" \ + | tee test/login_success \ + | grep "msg = bob authenticated by password" + + +# get bob dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies \ + "${CURL_OPTS[@]}" > test/dashboard + +# first assginment should have full score since the single required peer review was completed perfectly +grep "Total Score: 100.0" test/dashboard +# assginment should have half score since no peer review was needed and final sub got half score +grep "Total Score: 50.0" test/dashboard + +echo "PEER REVIEW CORNER CASES PASS" +echo "$WORKDIR" From 410599edc70d56a4ece0df0c08788fe93ff9992d Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Tue, 26 Aug 2025 07:50:27 -0400 Subject: [PATCH 09/13] test-lib: use unpriveleged port CI now runs the containers as a non-root user and exposes 1465 for SMTP instead of 465 Signed-off-by: Joel Savitz --- test-lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-lib b/test-lib index 1b9607fb..57a49e60 100644 --- a/test-lib +++ b/test-lib @@ -126,7 +126,7 @@ setup_submissions_for() { git config sendemail.smtpUser "$user" git config sendemail.smtpPass builder git config sendemail.smtpserver localhost.localdomain - git config sendemail.smtpserverport 465 + git config sendemail.smtpserverport 1465 git config sendemail.smtpencryption ssl popd } From 8266b5100e30d911d4624c400aa3e26022dc443e Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Tue, 26 Aug 2025 07:52:29 -0400 Subject: [PATCH 10/13] test-sub: correct bob version numbers Signed-off-by: Joel Savitz --- test-sub.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test-sub.sh b/test-sub.sh index b8ae94b0..47667481 100755 --- a/test-sub.sh +++ b/test-sub.sh @@ -30,7 +30,7 @@ enter_and_checkout setup_initial_bob_corrupt write_commit_to "abc" bob/setup/work write_commit_to "def" bob/setup/work "append" write_commit_to "ghi" bob/setup/work "append" -git format-patch --rfc --cover-letter -v1 -3 +git format-patch --rfc --cover-letter -v2 -3 rm ./*-0002-*.patch fixup_cover "Corrupt patchset" exit_after_sending setup @@ -39,7 +39,7 @@ exit_after_sending setup enter_and_checkout setup_initial_bob_whitespace write_commit_to "abc" bob/setup/work write_commit_to "def " bob/setup/work "append" -git format-patch --rfc --cover-letter -v1 -2 +git format-patch --rfc --cover-letter -v3 -2 fixup_cover "Patchset with whitespace errors" exit_after_sending setup @@ -47,7 +47,7 @@ exit_after_sending setup enter_and_checkout setup_initial_bob_nocover write_commit_to "abc" bob/setup/work write_commit_to "def" bob/setup/work "append" -git format-patch --rfc -v1 -2 +git format-patch --rfc -v4 -2 exit_after_sending setup # patchset with no cover letter and corrupt first patch @@ -55,7 +55,7 @@ enter_and_checkout setup_initial_bob_nocover-corrupt write_commit_to "abc" bob/setup/work write_commit_to "def" bob/setup/work "append" write_commit_to "ghi" bob/setup/work "append" -git format-patch --rfc -v1 -2 +git format-patch --rfc -v5 -2 exit_after_sending setup # setup_initial_bab submissions @@ -150,7 +150,7 @@ setup_submissions_for bob enter_and_checkout setup_final_bob_good write_commit_to "abc" bob/setup/work write_commit_to "def" bob/setup/work "append" -git format-patch --cover-letter -v2 -2 +git format-patch --cover-letter -v6 -2 fixup_cover "Good final patchset" exit_after_sending setup From c06b7dbe237fdcc0c6f929536cb7363b1225a4af Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Sun, 7 Sep 2025 16:39:05 -0400 Subject: [PATCH 11/13] test: remove rubrics Remove assignment rubric specificiation when invoking create_lighting_assignment since the feature is deferred to 0.8 Signed-off-by: Joel Savitz --- test-sub.sh | 2 +- test-sub2.sh | 2 +- test-sub4.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test-sub.sh b/test-sub.sh index 47667481..1a5dbc6d 100755 --- a/test-sub.sh +++ b/test-sub.sh @@ -12,7 +12,7 @@ cat << EOF > "$WORKDIR"/setup_rubric {('--- a/bob/setup/work', '+++ b/bob/setup/work'): 0}] EOF -create_lighting_assignment setup 15 25 30 "$WORKDIR"/setup_rubric +create_lighting_assignment setup 15 25 30 # setup_intial_bob patchsets setup_submissions_for bob diff --git a/test-sub2.sh b/test-sub2.sh index 80e5e733..d9b6893b 100755 --- a/test-sub2.sh +++ b/test-sub2.sh @@ -12,7 +12,7 @@ cat << EOF > "$WORKDIR"/coding_rubric {('--- /dev/null', '+++ b/user/coding/Makefile'): 0}] EOF -create_lighting_assignment coding 25 28 30 "$WORKDIR"/coding_rubric +create_lighting_assignment coding 25 28 30 setup_submissions_for bob diff --git a/test-sub4.sh b/test-sub4.sh index 76c3f65c..77350786 100755 --- a/test-sub4.sh +++ b/test-sub4.sh @@ -13,7 +13,7 @@ cat << EOF > "$WORKDIR"/first_rubric {('--- a/bob/first/work', '+++ b/bob/first/work'): 0}] EOF -create_lighting_assignment first 10 11 20 "$WORKDIR"/first_rubric +create_lighting_assignment first 10 11 20 setup_submissions_for bob @@ -52,7 +52,7 @@ cat << EOF > "$WORKDIR"/second_rubric {('--- a/bob/second/work', '+++ b/bob/second/work'): 0}] EOF -create_lighting_assignment second 5 7 10 "$WORKDIR"/second_rubric +create_lighting_assignment second 5 7 10 setup_submissions_for bob From d8279fc782c07e202fd636cfa7c60d435296beae Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Sun, 7 Sep 2025 16:47:22 -0400 Subject: [PATCH 12/13] test: remove rubric status checks Signed-off-by: Joel Savitz --- test-sub2.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test-sub2.sh b/test-sub2.sh index d9b6893b..cb97aa99 100755 --- a/test-sub2.sh +++ b/test-sub2.sh @@ -125,18 +125,18 @@ git clone http://localhost:"$(get_git_port)"/grading.git pushd grading STATUSES=( -"patch count 4 violates expected rubric patch count of 3!" -"patch count 2 violates expected rubric patch count of 3!" +"patchset applies." +"patchset applies." "illegal patch 1: permission denied for path program.c!" "illegal patch 3: permission denied for path Makefile!" -"patch 1 violates the assignment rubric!" -"patch 3 violates the assignment rubric!" -"patch 1 violates the assignment rubric!" -"patch 3 violates the assignment rubric!" -"patch 1 violates the assignment rubric!" -"patch 3 violates the assignment rubric!" -"patch 1 violates the assignment rubric!" -"patch 3 violates the assignment rubric!" +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." "patchset applies." ) From 5e098367ae26a56b7c00f39da834f28403f9b404 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Mon, 8 Sep 2025 13:54:22 -0400 Subject: [PATCH 13/13] test: start.sh: simplify volume deletion Signed-off-by: Joel Savitz --- start.sh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/start.sh b/start.sh index 9649b7bc..da5ea7e3 100755 --- a/start.sh +++ b/start.sh @@ -20,8 +20,7 @@ then ./test.sh if [ -f test-sub.sh ] then - podman-compose down - yes | podman volume prune + podman-compose down -v podman-compose up -d podman-compose logs -f submatrix 2>&1 | sed '/Synapse now listening on TCP port 8008/ q' ./dev_sockets.sh & @@ -29,16 +28,13 @@ then git config --global user.email podman@podman ./test-sub.sh ./test-sub-check.sh - podman-compose down - yes | podman volume prune + podman-compose down -v podman-compose up -d ./test-sub2.sh - podman-compose down - yes | podman volume prune + podman-compose down -v podman-compose up -d ./test-sub3.sh - podman-compose down - yes | podman volume prune + podman-compose down -v podman-compose up -d ./test-sub4.sh