diff --git a/Containerfile b/Containerfile index ca3c8a18..de7b9f6e 100644 --- a/Containerfile +++ b/Containerfile @@ -11,7 +11,10 @@ RUN dnf update -y && \ python-flake8 \ python-virtualenv \ python-pip \ - git + gawk \ + socat \ + git \ + git-email # needed because for some reason newuidmap and newgidmap programs # lose their xattrs giving them caps when the container image for diff --git a/git/admin.sh b/git/admin.sh index f3a9e46e..bbd5189a 100755 --- a/git/admin.sh +++ b/git/admin.sh @@ -1,6 +1,6 @@ #!/bin/sh set -e -DOCKER_COMPOSE=${DOCKER_COMPOSE:-podman-compose} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} -$DOCKER_COMPOSE exec git ./create-repo.sh "$@" +$PODMAN_COMPOSE exec git ./create-repo.sh "$@" diff --git a/orbit/warpdrive.sh b/orbit/warpdrive.sh index 8d92572d..c78e8158 100755 --- a/orbit/warpdrive.sh +++ b/orbit/warpdrive.sh @@ -4,6 +4,6 @@ set -e -DOCKER_COMPOSE=${DOCKER_COMPOSE:-podman-compose} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} -$DOCKER_COMPOSE exec orbit ./hyperspace.py "$@" +$PODMAN_COMPOSE exec orbit ./hyperspace.py "$@" diff --git a/script-lint.sh b/script-lint.sh index 10cf94fa..ef2d49ad 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -5,8 +5,14 @@ require shellcheck set -ex +# -x needed to make shellcheck follow `source` command shellcheck script-lint.sh -shellcheck test.sh +shellcheck -x test.sh +shellcheck -x test-sub.sh +shellcheck -x test-sub-check.sh +shellcheck -x test-sub2.sh +shellcheck -x test-sub3.sh +shellcheck -x test-sub4.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh @@ -16,6 +22,5 @@ shellcheck git/setup-repo.sh shellcheck git/cgi-bin/git-receive-pack shellcheck git/hooks/post-update -# -x needed to make shellcheck follow `source` command shellcheck -x backup/backup.sh shellcheck -x backup/restore.sh diff --git a/start.sh b/start.sh index 9394b8f4..da5ea7e3 100755 --- a/start.sh +++ b/start.sh @@ -18,6 +18,27 @@ podman-compose logs -f submatrix 2>&1 | sed '/Synapse now listening on TCP port if [ -f test.sh ] then ./test.sh + if [ -f test-sub.sh ] + then + podman-compose down -v + podman-compose up -d + podman-compose logs -f submatrix 2>&1 | sed '/Synapse now listening on TCP port 8008/ q' + ./dev_sockets.sh & + git config --global user.name PINP + git config --global user.email podman@podman + ./test-sub.sh + ./test-sub-check.sh + podman-compose down -v + podman-compose up -d + ./test-sub2.sh + podman-compose down -v + podman-compose up -d + ./test-sub3.sh + podman-compose down -v + podman-compose up -d + ./test-sub4.sh + + fi else virtualenv . pip install -r requirements.txt diff --git a/test-lib b/test-lib new file mode 100644 index 00000000..57a49e60 --- /dev/null +++ b/test-lib @@ -0,0 +1,195 @@ +# This line: +# - aborts the script after any pipeline returns nonzero (e) +# - shows all commands as they are run (x) +# - sets any dereference of an unset variable to trigger an error (u) +# - causes the return value of a pipeline to be the nonzero return value +# of the furthest right failing command or zero if no command failed (o pipefail) +set -exuo pipefail + +PODMAN=${PODMAN:-podman} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} +SCRIPT_DIR=$(dirname "$(readlink -f "$0")") +WORKDIR=$(mktemp -d) + +HOSTNAME_FROM_DOTENV="$(sh -c ' +set -o allexport +. ./.env +exec jq -r -n "env.SINGULARITY_HOSTNAME" +')" + +SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} + +setup_testdir() { + # Create test dir if it does not exist yet + mkdir -p test + + # Reset the test directory + rm -f test/* + + # put the cert in there + ${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +} + +CURL_OPTS=( \ +--verbose \ +--cacert test/ca_cert.pem \ +--fail \ +--no-progress-meter \ +) + + +get_git_port() { ${PODMAN} port singularity_git_1 | awk -F':' '{ print $2 }' ; } ; + +# preconditions: +# - SCRIPT_DIR defined (singularity repo root) +# - WORKDIR defined (arbitrary) temp directory +setup_submissions_and_grading_repo() { + pushd "$SCRIPT_DIR" + # nuke grading repo inside container + # shellcheck disable=SC2016 + ${PODMAN_COMPOSE} exec git ash -c 'cd /var/lib/git/grading.git && for t in $(git tag); do git tag -d $t; done' + # crete and push fresh submissions repo + rm -rf repos/submissions + git/admin.sh submissions "course submissions repository" + pushd repos + git init --bare submissions + echo "course submissions repository" > submissions/description + # create a temporary workdir to push an initial commit to the submission repo + git init submissions_init + pushd submissions_init + echo "# submissions" > README.md + git add README.md + git status + git -c user.name=singularity -c user.email=singularity@singularity commit -sm 'init submissions repo' + git push ../submissions master + popd + rm -rf submissions_init + pushd submissions + git push --mirror http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/submissions + popd + popd + popd + + pushd "$WORKDIR" + mkdir certs + pushd certs + ${PODMAN} volume export singularity_ssl-certs > certs.tar + tar xf certs.tar + popd + git clone http://localhost:"$(get_git_port)"/submissions + popd +} + +create_lighting_assignment() { + test -n "$1" + test -n "$2" + test -n "$3" + test -n "$4" + + local asn="$1" + local i_s="$2" + local p_s="$3" + local f_s="$4" + set +u + local rub="$5" + set -u + + RUBRIC= + if [ -n "$rub" ] + then + RUBRIC="-r $rub" + fi + + pushd "$SCRIPT_DIR" + # create or recreate setup assignment + if denis/configure.sh dump | grep -q "^$asn:"; then + denis/configure.sh remove -a "$asn" + fi + denis/configure.sh create -a "$asn" -i "$(date -d "$i_s secs" +%s)" -p "$(date -d "$p_s secs" +%s)" -f "$(date -d "$f_s secs" +%s)" ${RUBRIC} + denis/configure.sh reload + popd +} + +# preconditions: +# - called after setup_submissions_and_grading_repo +setup_submissions_for() { + test -n "$1" + local user="$1" + + pushd "$SCRIPT_DIR" + orbit/warpdrive.sh -u "$user" -p builder -n || orbit/warpdrive.sh -u "$user" -p builder -m + popd + + pushd "$WORKDIR"/submissions + git config user.name "$user" + git config user.email "$user"@localhost.localdomain + git config sendemail.smtpUser "$user" + git config sendemail.smtpPass builder + git config sendemail.smtpserver localhost.localdomain + git config sendemail.smtpserverport 1465 + git config sendemail.smtpencryption ssl + popd +} + +# preconditions: +# - no non-tracked files in submissions repo +# - called after setup_submissions_for +enter_and_checkout() { + test -n "$1" + local branch="$1" + + pushd "$WORKDIR"/submissions + git checkout --orphan "$1" + if [ -n "$(ls)" ] + then + git rm -rf ./* + fi +} + +# preconditions: +# - called after a call to enter_and_checkout +exit_after_sending() { + test -n "$1" + local asn="$1" + + git send-email \ + --confirm=never \ + --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem \ + --to "$asn"@localhost.localdomain \ + ./*.patch + rm ./*.patch + popd + sleep 1 +} + +# preconditions: +# - called after a call to enter_and_checkout +write_commit_to() { + test -n "$1" && test -n "$2" + local content="$1" + local file="$2" + set +u # necessary since $3 is unbound in 2 arg case + local opt="$3" + set -x + + mkdir -p $(dirname "$file") + + if [ "$opt" == "append" ]; then + echo "$content" >> "$file" + else + echo "$content" > "$file" + fi + + git add "$file" + git commit -sm "add $content to $file" +} + +# preconditions: +# - called after a call to write_commit_to +fixup_cover() { + test -n "$1" + sed -i "s/\*\*\* SUBJECT HERE \*\*\*/$1/g" *0000-cover-letter.patch + sed -i "s/\*\*\* BLURB HERE \*\*\*/$1/g" *0000-cover-letter.patch + sed -i "\$a\\Signed-off-by: $(git config user.name) <$(git config user.email)>" *0000-cover-letter.patch +} + diff --git a/test-sub-check.sh b/test-sub-check.sh new file mode 100755 index 00000000..4f49ae3d --- /dev/null +++ b/test-sub-check.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +source test-lib + +setup_testdir + +# login as bob +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies \ + --data "username=bob&password=builder" \ + | tee test/login_success \ + | grep "msg = bob authenticated by password" + + +# check for setup assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard \ + | grep "setup" + +grep "Total Score: 64.9" test/dashboard + +grep "missing cover letter and first patch failed to apply!" test/dashboard + +grep "bib Peer Review" test/dashboard + +grep "bab Peer Review" test/dashboard + +grep "patchset applies." test/dashboard + +grep "Good effort but try harder next time." test/dashboard + +# login as bab +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies_bab \ + --data "username=bab&password=builder" \ + | tee test/login_success_bab \ + | grep "msg = bab authenticated by password" + + +# check for setup assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies_bab \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard_bab \ + | grep "setup" + +grep "Total Score: 0.0" test/dashboard_bab + +grep "bib Peer Review" test/dashboard_bab + +grep "bob Peer Review" test/dashboard_bab + +grep "patchset applies." test/dashboard_bab + +grep "illegal patch 1: permission denied for path work!" test/dashboard_bab + +grep 'Please review git' test/dashboard_bab + +# login as bib +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies_bib \ + --data "username=bib&password=builder" \ + | tee test/login_success_bib \ + | grep "msg = bib authenticated by password" + + +# check for setup assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies_bib \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard_bib \ + | grep "setup" + +grep "Total Score: 80.0" test/dashboard_bib + +grep "bob Peer Review" test/dashboard_bib + +grep "bab Peer Review" test/dashboard_bib + +grep "patchset applies." test/dashboard_bib + +grep 'Perfect. But no peer review!' test/dashboard_bib + +echo "ALL SUBMISSION TESTS PASS" diff --git a/test-sub.sh b/test-sub.sh new file mode 100755 index 00000000..1a5dbc6d --- /dev/null +++ b/test-sub.sh @@ -0,0 +1,198 @@ +#!/bin/bash + +# assumes singularity is running when invoked +# script is written to be as idempotent as posible + +source test-lib + +setup_submissions_and_grading_repo + +cat << EOF > "$WORKDIR"/setup_rubric +[{('--- /dev/null', '+++ b/bob/setup/work'): 0}, +{('--- a/bob/setup/work', '+++ b/bob/setup/work'): 0}] +EOF + +create_lighting_assignment setup 15 25 30 + +# setup_intial_bob patchsets +setup_submissions_for bob + +# good patchset +enter_and_checkout setup_initial_bob_good +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Good patchset" +exit_after_sending setup + +# corrupt patchset +enter_and_checkout setup_initial_bob_corrupt +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" +write_commit_to "ghi" bob/setup/work "append" +git format-patch --rfc --cover-letter -v2 -3 +rm ./*-0002-*.patch +fixup_cover "Corrupt patchset" +exit_after_sending setup + +# patchset with whitespace errors +enter_and_checkout setup_initial_bob_whitespace +write_commit_to "abc" bob/setup/work +write_commit_to "def " bob/setup/work "append" +git format-patch --rfc --cover-letter -v3 -2 +fixup_cover "Patchset with whitespace errors" +exit_after_sending setup + +# patchset with no cover letter +enter_and_checkout setup_initial_bob_nocover +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" +git format-patch --rfc -v4 -2 +exit_after_sending setup + +# patchset with no cover letter and corrupt first patch +enter_and_checkout setup_initial_bob_nocover-corrupt +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" +write_commit_to "ghi" bob/setup/work "append" +git format-patch --rfc -v5 -2 +exit_after_sending setup + +# setup_initial_bab submissions +setup_submissions_for bab + +# bab good patchset +enter_and_checkout setup_initial_bab_good +write_commit_to "abc" bab/setup/work +write_commit_to "def" bab/setup/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Good patchset" +exit_after_sending setup + +# setup_initial_bib submissions +setup_submissions_for bib + +# bib good patchset +enter_and_checkout setup_initial_bib_good +write_commit_to "abc" bib/setup/work +write_commit_to "def" bib/setup/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Good patchset" +exit_after_sending setup + +# investigate grading repo +pushd "$WORKDIR" +git clone http://localhost:"$(get_git_port)"/grading.git +pushd grading +git fetch --tag + +STATUSES=( +'patchset applies.' +'patch 2 failed to apply!' +'whitespace error patch 2?' +'missing cover letter!' +'missing cover letter and first patch failed to apply!' +'patchset applies.' +'patchset applies.') + +# submitted patchses should have statuses in this order +# assumption: first ID tag is first patch submitted by this script and IDs increase monotonically +i=0 +for t in $(git tag); do + git show -s --oneline "$t" | grep -q "${STATUSES[$i]}" + if [[ $i == 5 ]]; then + PEER1_ID=$t + elif [[ $i == 6 ]]; then + PEER2_ID=$t + fi + i=$((i+1)) +done +popd + +echo "wait for initial submission deadline (5 secs)" +sleep 5 + +setup_submissions_for bob + +# setup bob peer review +enter_and_checkout setup_reviews_bob + +# submit peer review 1 +cat < review1 +Subject: setup review 1 for bab +In-Reply-To: <${PEER1_ID::-1}1@localhost.localdomain> + +Looks good to me + +Acked-by: bob +PEER_REVIEW1 +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bab@localhost.localdomain review1 +rm review1 + +# submit peer review 2 +cat < review2 +Subject: setup review 2 for bib +In-Reply-To: <${PEER2_ID::-1}2@localhost.localdomain> + +Looks good to me + +Acked-by: bob +PEER_REVIEW2 +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bib@localhost.localdomain review2 +rm review2 + +popd +sleep 1 +# end bob peer review + +# setup_final_bob good patchset +setup_submissions_for bob +enter_and_checkout setup_final_bob_good +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" +git format-patch --cover-letter -v6 -2 +fixup_cover "Good final patchset" +exit_after_sending setup + +# setup_final_bib final submission incorrectly labeled RFC +setup_submissions_for bib +enter_and_checkout setup_final_bib_rfc +write_commit_to "abc" bib/setup/work +write_commit_to "def" bib/setup/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "RFC tagged final patchset" +exit_after_sending setup + +# setup_final_bab submission in wrong directory +setup_submissions_for bab +enter_and_checkout setup_final_bab_illegal +write_commit_to "abc" work +write_commit_to "def" work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Illegal patchset" +exit_after_sending setup + +echo "wait for final submission deadline (10 secs)" +sleep 10 + +pushd grading +git fetch --tag +sleep 2 +git fetch -f origin refs/notes/*:refs/notes/* +git remote set-url --push origin http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/grading.git + +git notes --ref=grade add setup_final_bob -m '66' +git notes --ref=grade add setup_review1_bob -m '22' +git notes --ref=grade add setup_review2_bob -m '99' +git notes --ref=feedback add setup_final_bob -m 'Good effort but try harder next time.' + +# automatic 0 for peer reviews +git notes --ref=grade add setup_final_bib -m '100' +git notes --ref=feedback add setup_final_bib -m 'Perfect. But no peer review!' + +# automatic 0 for peer reviews and final submission +git notes --ref=feedback add setup_final_bab -m 'Please review git' + +git push origin refs/notes/*:refs/notes/* + +echo "$WORKDIR" diff --git a/test-sub2.sh b/test-sub2.sh new file mode 100755 index 00000000..cb97aa99 --- /dev/null +++ b/test-sub2.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +# test a bunch of rubric violations + +source test-lib + +setup_submissions_and_grading_repo + +cat << EOF > "$WORKDIR"/coding_rubric +[{('--- /dev/null', '+++ b/user/coding/program.c'): 0}, +{('--- a/user/coding/program.c', '+++ b/user/coding/program.c'): 0}, +{('--- /dev/null', '+++ b/user/coding/Makefile'): 0}] +EOF + +create_lighting_assignment coding 25 28 30 + +setup_submissions_for bob + +enter_and_checkout coding_initial_bob_toomany +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Yet more C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -4 +fixup_cover "Too many patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_toofew +write_commit_to "C program" bob/coding/program.c +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -2 +fixup_cover "Too few patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_top-level-first +write_commit_to "C program" program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Top level first file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_top-level-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Top level third file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_second-level-first +write_commit_to "C program" bob/program.c +write_commit_to "More C program" bob/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Second level first file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_second-level-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Second level third file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_fourth-level-first +write_commit_to "C program" bob/coding/program/program.c +write_commit_to "More C program" bob/coding/program/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Fourth level first file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_fourth-level-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/program/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Fourth level third file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-second-dir-first +write_commit_to "C program" bob/setup/program.c +write_commit_to "More C program" bob/setup/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong second directory first patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-second-dir-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/setup/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong second directory third patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-filename-first +write_commit_to "C program" bob/coding/hello.c +write_commit_to "More C program" bob/coding/hello.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong filename first patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-filename-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/Snakefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong filename third patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_good +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Good patches" +exit_after_sending coding + +pushd "$WORKDIR" +git clone http://localhost:"$(get_git_port)"/grading.git +pushd grading + +STATUSES=( +"patchset applies." +"patchset applies." +"illegal patch 1: permission denied for path program.c!" +"illegal patch 3: permission denied for path Makefile!" +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +) + +# submitted patchses should have statuses in this order +# assumption: first ID tag is first patch submitted by this script and IDs increase monotonically +i=0 +for t in $(git tag); do + git show -s --oneline "$t" | grep -q "${STATUSES[$i]}" + i=$((i+1)) +done +popd +popd + +echo "RUBRIC CHECKS PASS" +echo "$WORKDIR" diff --git a/test-sub3.sh b/test-sub3.sh new file mode 100755 index 00000000..55b28c30 --- /dev/null +++ b/test-sub3.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# test only initial submission results in automatic 0 + +source test-lib + +setup_submissions_and_grading_repo + +create_lighting_assignment fun 5 7 10 + +setup_submissions_for bob + +enter_and_checkout fun_initial_bob_only-initial +write_commit_to "C program" bob/fun/program.c +write_commit_to "More C program" bob/fun/program.c "append" +write_commit_to "Yet more C program" bob/fun/program.c "append" +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Only initial sub patchset" +exit_after_sending fun + +# wait for initial submission deadline +sleep 9 + +setup_testdir + +# login as bob +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies \ + --data "username=bob&password=builder" \ + | tee test/login_success \ + | grep "msg = bob authenticated by password" + + +# check for fun assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard \ + | grep "fun" + +grep "patchset applies." test/dashboard +grep "No submission" test/dashboard +grep "Total Score: 0.0" test/dashboard + +echo "INITIAL SUBMISSION ONLY GETS AUTO ZERO CONFIRMED" diff --git a/test-sub4.sh b/test-sub4.sh new file mode 100755 index 00000000..77350786 --- /dev/null +++ b/test-sub4.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +# test total score calculation when assigned only 0 or 1 peer reviews legitimately + +source test-lib + +setup_submissions_and_grading_repo + +# single peer review case + +cat << EOF > "$WORKDIR"/first_rubric +[{('--- /dev/null', '+++ b/bob/first/work'): 0}, +{('--- a/bob/first/work', '+++ b/bob/first/work'): 0}] +EOF + +create_lighting_assignment first 10 11 20 + +setup_submissions_for bob + +enter_and_checkout first_initial_bob_good +write_commit_to "abc" bob/first/work +write_commit_to "def" bob/first/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Good patchset" +exit_after_sending first + +setup_submissions_for bab + +enter_and_checkout first_initial_bab_good +write_commit_to "abc" bab/first/work +write_commit_to "def" bab/first/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Good patchset" +exit_after_sending first + +sleep 8 + +setup_submissions_for bob +enter_and_checkout first_final_bob_good +write_commit_to "abc" bob/first/work +write_commit_to "def" bob/first/work "append" +git format-patch --cover-letter -v2 -2 +fixup_cover "Good final patchset" +exit_after_sending first + +sleep 9 + +# no peer review case + +cat << EOF > "$WORKDIR"/second_rubric +[{('--- /dev/null', '+++ b/bob/second/work'): 0}, +{('--- a/bob/second/work', '+++ b/bob/second/work'): 0}] +EOF + +create_lighting_assignment second 5 7 10 + +setup_submissions_for bob + +enter_and_checkout second_initial_bob_good +write_commit_to "abc" bob/second/work +write_commit_to "def" bob/second/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Good patchset" +exit_after_sending second + +sleep 4 + +enter_and_checkout second_final_bob_good +write_commit_to "abc" bob/second/work +write_commit_to "def" bob/second/work "append" +git format-patch --cover-letter -v2 -2 +fixup_cover "Good final patchset" +exit_after_sending second + +sleep 4 + +pushd "$WORKDIR" +git clone http://localhost:"$(get_git_port)"/grading.git +pushd grading + +git fetch --tag +sleep 2 +git fetch -f origin refs/notes/*:refs/notes/* +git remote set-url --push origin http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/grading.git + +git notes --ref=grade add first_final_bob -m '100' +git notes --ref=grade add first_review1_bob -fm '100' +git notes --ref=grade add second_final_bob -m '50' +git push origin refs/notes/*:refs/notes/* + +popd +popd + +# login as bob +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies \ + --data "username=bob&password=builder" \ + | tee test/login_success \ + | grep "msg = bob authenticated by password" + + +# get bob dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies \ + "${CURL_OPTS[@]}" > test/dashboard + +# first assginment should have full score since the single required peer review was completed perfectly +grep "Total Score: 100.0" test/dashboard +# assginment should have half score since no peer review was needed and final sub got half score +grep "Total Score: 50.0" test/dashboard + +echo "PEER REVIEW CORNER CASES PASS" +echo "$WORKDIR" diff --git a/test.sh b/test.sh index 7e13f23f..f87528c9 100755 --- a/test.sh +++ b/test.sh @@ -2,23 +2,14 @@ # Testing script for singularity and orbit -# This line: -# - aborts the script after any pipeline returns nonzero (e) -# - shows all commands as they are run (x) -# - sets any dereference of an unset variable to trigger an error (u) -# - causes the return value of a pipeline to be the nonzero return value -# of the furthest right failing command or zero if no command failed (o pipefail) -set -exuo pipefail - -DOCKER=${DOCKER:-podman} -DOCKER_COMPOSE=${DOCKER_COMPOSE:-podman-compose} +source test-lib require() { command -v "$1" > /dev/null || { echo "error: $1 command required yet absent" ; exit 1 ; } ; } require curl require jq require flake8 -require "${DOCKER}" -require "${DOCKER_COMPOSE}" +require "${PODMAN}" +require "${PODMAN_COMPOSE}" # Check for shell script style compliance with shellcheck ./script-lint.sh @@ -26,28 +17,7 @@ require "${DOCKER_COMPOSE}" # Check python style compliance flake8 -# Create test dir if it does not exist yet -mkdir -p test - -# Reset the test directory -rm -f test/* - -HOSTNAME_FROM_DOTENV="$(sh -c ' -set -o allexport -. ./.env -exec jq -r -n "env.SINGULARITY_HOSTNAME" -')" - -SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} - -${DOCKER} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem - -CURL_OPTS=( \ ---verbose \ ---cacert test/ca_cert.pem \ ---fail \ ---no-progress-meter \ -) +setup_testdir # Check that registration fails before user creation curl --url "https://$SINGULARITY_HOSTNAME/register" \ @@ -150,10 +120,10 @@ curl --url "pop3s://$SINGULARITY_HOSTNAME" \ orbit/warpdrive.sh -u resu -p ssap -n # Limit `resu`'s access to the empty inbox -${DOCKER_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -d resu +${PODMAN_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -d resu # Update list of email to include new message -${DOCKER_COMPOSE} exec denis /usr/local/bin/init_journal /var/lib/email/journal/journal /var/lib/email/journal/temp /var/lib/email/mail +${PODMAN_COMPOSE} exec denis /usr/local/bin/init_journal /var/lib/email/journal/journal /var/lib/email/journal/temp /var/lib/email/mail # Check that the user can get the most recent message sent to the server curl --url "pop3s://$SINGULARITY_HOSTNAME/1" \ @@ -173,7 +143,7 @@ curl --url "pop3s://$SINGULARITY_HOSTNAME" \ # Remove limit on `resu`'s access to the inbox -${DOCKER_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -a resu +${PODMAN_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -a resu # Check that `resu` can now get the most recent message sent to the server curl --url "pop3s://$SINGULARITY_HOSTNAME/1" \