From 067256f06117cb8e9994afc9cf8d2d1ca490f9c5 Mon Sep 17 00:00:00 2001 From: BasharRadya Date: Sun, 20 Oct 2024 12:03:05 +0300 Subject: [PATCH] Add infrastructure for SMTP and POP test coverage - Integrated clang coverage tool into the project build process to enable test coverage reporting. Currently, the coverage tools operate only on pop3.c and smtp.c. To collect coverage the following commands must be run: - export COVERAGE=1 - podman-compose build - ./test.sh (Note: it is recommended to wait a bit until the servers are up and running) - ./collect_pop_smtp_coverage.sh To turn off coverage, the variable COVERAGE must not be present or be of value 0. - Note that in order to support coverage it was needed to build and execute the relevant binaries in the same build stage (the last) of the container Signed-off-by: BasharRadya Signed-off-by: Bsaileh --- collect_coverage.sh | 28 +++++++++++++++ collect_pop_smtp_coverage.sh | 5 +++ container-compose.yml | 15 ++++++++ denis/Containerfile | 10 ++++++ orbit/radius.py | 59 +++++++++++++++++++++++++++----- pop/Containerfile | 36 +++++++++++++++++--- pop/Makefile | 15 ++++++-- pop/pop_wrapper.c | 34 +++++++++++++++++++ smtp/Containerfile | 35 +++++++++++++++---- smtp/Makefile | 12 +++++-- smtp/smtp_wrapper.c | 34 +++++++++++++++++++ test.sh | 66 ++++++++++++++++++++++++++++++++++++ 12 files changed, 324 insertions(+), 25 deletions(-) create mode 100755 collect_coverage.sh create mode 100755 collect_pop_smtp_coverage.sh create mode 100644 pop/pop_wrapper.c create mode 100644 smtp/smtp_wrapper.c diff --git a/collect_coverage.sh b/collect_coverage.sh new file mode 100755 index 00000000..9912841e --- /dev/null +++ b/collect_coverage.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -ex + +CONTAINER_NAME=$1 +VOLUME_NAME=$2 +DEST_DIR=$3 +BUILD_PATH=$4 +SRC_NAME=$5 +EXC_NAME=$6 +# Ensure the local coverage directory exists + +# Execute commands inside the container to process gcov files +mkdir -p $DEST_DIR +CUR_DIR=$(pwd) +cd $DEST_DIR +rm -rf ./* +cd $CUR_DIR +podman exec -u 0 -it $CONTAINER_NAME /bin/bash -c ' + cd /coverage + llvm-profdata merge -sparse *.profraw -o coverage.profdata + mkdir -p cov + mkdir -p '$BUILD_PATH' + cp ./'$SRC_NAME' '$BUILD_PATH'/'$SRC_NAME' + llvm-cov show /usr/local/bin/'$EXC_NAME' -instr-profile=coverage.profdata -format=html -output-dir=./cov +' +cp -r /var/lib/containers/storage/volumes/$VOLUME_NAME/_data/cov/* $DEST_DIR +chown 100:100 -R $DEST_DIR +chmod 777 -R $DEST_DIR \ No newline at end of file diff --git a/collect_pop_smtp_coverage.sh b/collect_pop_smtp_coverage.sh new file mode 100755 index 00000000..83aa58af --- /dev/null +++ b/collect_pop_smtp_coverage.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -ex + +./collect_coverage.sh singularity_smtp_1 singularity_smtp_coverage ./smtp_coverage/ /smtp smtp.c smtp_ +./collect_coverage.sh singularity_pop_1 singularity_pop_coverage ./pop_coverage/ /pop pop3.c pop3_ diff --git a/container-compose.yml b/container-compose.yml index eb67057b..4ed0295b 100644 --- a/container-compose.yml +++ b/container-compose.yml @@ -78,6 +78,7 @@ services: target: smtp args: hostname: ${SINGULARITY_HOSTNAME} + coverage: 1 LISTEN_PORT: 1465 volumes: - type: volume @@ -92,6 +93,12 @@ services: source: email-patchsets target: /var/lib/email/patchsets read_only: false + - type: volume + source: smtp_coverage + target: /coverage + read_only: false + environment: + COVERAGE: ${COVERAGE-0} networks: - smtp pop: @@ -113,9 +120,15 @@ services: source: email-journal target: /var/lib/email/journal read_only: true + - type: volume + source: pop_coverage + target: /coverage + read_only: false depends_on: - smtp - denis + environment: + COVERAGE: ${COVERAGE-0} networks: - pop submatrix: @@ -246,3 +259,5 @@ volumes: submatrix-data: denis-db: git-repos: + smtp_coverage: + pop_coverage: \ No newline at end of file diff --git a/denis/Containerfile b/denis/Containerfile index cc12ade3..083e8c93 100644 --- a/denis/Containerfile +++ b/denis/Containerfile @@ -4,6 +4,16 @@ RUN apk add \ make \ ; +RUN if [ "$COVERAGE" -eq 1 ]; then \ + apk add \ + clang-dev \ + compiler-rt \ + llvm-dev \ + gettext \ + bash \ + ; \ + fi + COPY --from=run_at_source . /run-at RUN make -C /run-at CC='clang -static' diff --git a/orbit/radius.py b/orbit/radius.py index 2c480949..b5784112 100644 --- a/orbit/radius.py +++ b/orbit/radius.py @@ -293,7 +293,12 @@ def mk_form_welcome(session):
- +
''' @@ -452,7 +457,12 @@ def gradeable_row(self, item_name, gradeable, rightmost_col): {item_name} - {datetime.fromtimestamp(gradeable.timestamp).astimezone().isoformat() if gradeable else '-'} + {( + datetime.fromtimestamp(gradeable.timestamp) + .astimezone() + .isoformat() + if gradeable else '-' + )} {gradeable.submission_id if gradeable else '-'} @@ -475,7 +485,13 @@ def oopsie_button(self): def body(self): if self.oopsieness == OopsStatus.USED_HERE: return f""" - {self.gradeable_row('Final Submission', self.final, self.oopsie_button())} + {( + self.gradeable_row( + 'Final Submission', + self.final, + self.oopsie_button() + ) + )} Comments - @@ -485,14 +501,26 @@ def body(self): (int(datetime.now().timestamp()) < self.assignment.initial_due_date)): return f""" - {self.gradeable_row('Initial Submission', self.init, self.oopsie_button())} + {( + self.gradeable_row( + 'Initial Submission', + self.init, + self.oopsie_button() + ) + )} Automated Feedback - """ return f""" - {self.gradeable_row('Initial Submission', self.init, self.oopsie_button())} + {( + self.gradeable_row( + 'Initial Submission', + self.init, + self.oopsie_button() + ) + )} Automated Feedback - @@ -503,9 +531,19 @@ def body(self): Submission ID Score - {self.gradeable_row(self.peer1 + ' Peer Review', self.review1, '-') if self.peer1 else ''} - {self.gradeable_row(self.peer2 + ' Peer Review', self.review2, '-') if self.peer2 else ''} - {self.gradeable_row('Final Submission', self.final, '-')} + {( + self.gradeable_row( + self.peer1 + ' Peer Review', + self.review1, '-') if self.peer1 else '' + )} + {( + self.gradeable_row( + self.peer2 + ' Peer Review', + self.review2, '-') if self.peer2 else '' + )} + {( + self.gradeable_row('Final Submission', self.final, '-') + )} Comments - @@ -631,7 +669,10 @@ def form_respond(): username, password = creds rocket.msg('welcome to the classroom') return rocket.respond(f''' -

Save these credentials, you will not be able to access them again


+

+ Save these credentials, you will not be able to access them again +

+

Username: {username}


Password: {password}


''') diff --git a/pop/Containerfile b/pop/Containerfile index 9e65d4fa..1712ba45 100644 --- a/pop/Containerfile +++ b/pop/Containerfile @@ -1,9 +1,19 @@ -FROM alpine:3.20 AS build +FROM alpine:3.20 AS pop RUN apk add \ clang \ make \ ; +RUN if [ "$COVERAGE" -eq 1 ]; then \ + apk add \ + clang-dev \ + compiler-rt \ + llvm-dev \ + gettext \ + bash \ + ; \ + fi + COPY --from=tcp_server_source . /tcp_server ARG LISTEN_PORT=995 RUN make -C /tcp_server CC='clang -static' DEFAULT_PORT=${LISTEN_PORT} @@ -14,10 +24,26 @@ COPY . /pop RUN make -C /pop CC='clang -static' -FROM scratch as pop - -COPY --from=build /tcp_server/tcp_server /usr/local/bin/tcp_server -COPY --from=build /pop/pop3 /usr/local/bin/pop3 +RUN if [ "$COVERAGE" -eq 1 ]; then \ + make -C /pop CC='clang -static' SRVNAME=$hostname COVERAGE=1; \ + else \ + make -C /pop CC='clang -static' SRVNAME=$hostname; \ + fi + +# FROM scratch as pop + +RUN cp /tcp_server/tcp_server /usr/local/bin/tcp_server +RUN if [ "$COVERAGE" -eq 1 ]; then \ + cp /pop/pop3 /usr/local/bin/pop3_; \ + cp /pop/pop_wrapper /usr/local/bin/pop3; \ + mkdir -p /coverage; \ + chown 100:100 /coverage; \ + cp /pop/pop3.c /coverage/pop3.c; \ + else \ + cp /pop/pop3 /usr/local/bin/pop3; \ + fi + +RUN rm -rf /tcp_server /pop /journal /pop USER 100:100 diff --git a/pop/Makefile b/pop/Makefile index 95fd379e..f4b77276 100644 --- a/pop/Makefile +++ b/pop/Makefile @@ -2,16 +2,25 @@ CC = clang CFLAGS = -std=c2x -Weverything -Wno-unsafe-buffer-usage -Wno-c++98-compat -Wno-gnu-designator -Wno-gnu-case-range -Wno-initializer-overrides \ -Wno-declaration-after-statement -Wno-four-char-constants -Wno-pre-c2x-compat -Wno-disabled-macro-expansion -Wno-switch -Wno-switch-enum -D_GNU_SOURCE +# Add flags for coverage support +ADDITIONAL_CFLAGS = +ifeq ($(COVERAGE), 1) + ADDITIONAL_CFLAGS = -fprofile-instr-generate -fcoverage-mapping -mllvm -runtime-counter-relocation +endif + ifdef DEBUG CFLAGS += -DDEBUG -Og -g endif .PHONY: all clean -all: pop3 +all: pop3 pop_wrapper pop3: pop3.c journal/email.h - $(CC) $(CFLAGS) -o $@ $< + $(CC) $(CFLAGS) $(ADDITIONAL_CFLAGS) -o $@ $< + +pop_wrapper: pop_wrapper.c + $(CC) $(CFLAGS) -o $@ $^ clean: - -rm pop3 + -rm pop3 \ No newline at end of file diff --git a/pop/pop_wrapper.c b/pop/pop_wrapper.c new file mode 100644 index 00000000..58f6546f --- /dev/null +++ b/pop/pop_wrapper.c @@ -0,0 +1,34 @@ +#include +#include + +int main(int argc, char **argv) { + // Set the environment variable + int result = setenv("LLVM_PROFILE_FILE", "/coverage/coverage-%p.profraw%c", 1); + + if (result != 0) { + return 1; + } + + // Define the program to execute (predefined) + char *program ="/usr/local/bin/pop3_"; + + // Create a new array to hold the arguments for execvp, including the program name + char** exec_args = (char**) malloc(sizeof(char*) * (unsigned long) (argc + 1)); + + // Set the program ßßßname as the first argument + exec_args[0] = program; + + // Copy the rest of the arguments passed to your program to the new exec_args array + for (int i = 1; i < argc; i++) { + exec_args[i] = argv[i]; + } + + // Terminate the array with a NULL pointer (as required by execvp) + exec_args[argc] = NULL; + + // Execute the program, replacing the current process + execvp(program, exec_args); + + // If execvp returns, it means it failed + return 1; +} diff --git a/smtp/Containerfile b/smtp/Containerfile index ac17fb3c..5adac32d 100644 --- a/smtp/Containerfile +++ b/smtp/Containerfile @@ -1,9 +1,19 @@ -FROM alpine:3.20 AS build +FROM alpine:3.20 AS smtp RUN apk add \ clang \ + git \ make \ ; +RUN if [ "$COVERAGE" -eq 1 ]; then \ + apk add \ + clang-dev \ + compiler-rt \ + llvm-dev \ + bash \ + ; \ +fi + COPY --from=tcp_server_source . /tcp_server ARG LISTEN_PORT=465 RUN make -C /tcp_server CC='clang -static' DEFAULT_PORT=${LISTEN_PORT} @@ -14,18 +24,31 @@ COPY . /smtp ARG hostname RUN test -n "$hostname" || (echo 'hostname is not set' && false) -RUN make -C /smtp CC='clang -static' SRVNAME=$hostname +# RUN make -C /smtp CC='clang -static' SRVNAME=$hostname + +RUN if [ "$COVERAGE" -eq 1 ]; then \ + make -C /smtp CC='clang -static' SRVNAME=$hostname COVERAGE=1; \ + else \ + make -C /smtp CC='clang -static' SRVNAME=$hostname; \ + fi RUN mkdir -p /var/lib/email/mail /var/lib/email/logs && \ chown 100:100 /var/lib/email/mail /var/lib/email/logs && \ : -FROM scratch as smtp +RUN cp /tcp_server/tcp_server /usr/local/bin/tcp_server -COPY --from=build /var/lib/email /var/lib/email -COPY --from=build /tcp_server/tcp_server /usr/local/bin/tcp_server -COPY --from=build /smtp/smtp /usr/local/bin/smtp +RUN if [ "$COVERAGE" -eq 1 ]; then \ + cp /smtp/smtp_wrapper /usr/local/bin/smtp; \ + cp /smtp/smtp /usr/local/bin/smtp_; \ + mkdir -p /coverage; \ + chown 100:100 /coverage; \ + cp /smtp/smtp.c /coverage/smtp.c; \ + else \ + cp /smtp/smtp /usr/local/bin/smtp; \ + fi +RUN rm -rf /smtp /tcp_server USER 100:100 ARG LISTEN_PORT=465 diff --git a/smtp/Makefile b/smtp/Makefile index 8ea9e43e..8e07a882 100644 --- a/smtp/Makefile +++ b/smtp/Makefile @@ -3,20 +3,28 @@ CFLAGS = -std=c2x -Weverything -Wno-unsafe-buffer-usage -Wno-c++98-compat -Wno-g -Wno-initializer-overrides -Wno-declaration-after-statement -Wno-four-char-constants \ -Wno-pre-c2x-compat -Wno-disabled-macro-expansion -D_GNU_SOURCE -DHOSTNAME='"$(SRVNAME)"' +# Add flags for coverage support +ADDITIONAL_CFLAGS = +ifeq ($(COVERAGE), 1) + ADDITIONAL_CFLAGS = -fprofile-instr-generate -fcoverage-mapping -mllvm -runtime-counter-relocation +endif + ifdef DEBUG CFLAGS += -DDEBUG -Og -g endif .PHONY: all clean -all: smtp +all: smtp smtp_wrapper smtp: smtp.c ifndef SRVNAME $(error "you must pass SRVNAME=") endif + $(CC) $(CFLAGS) $(ADDITIONAL_CFLAGS) -o $@ $^ + +smtp_wrapper: smtp_wrapper.c $(CC) $(CFLAGS) -o $@ $^ clean: -rm smtp - diff --git a/smtp/smtp_wrapper.c b/smtp/smtp_wrapper.c new file mode 100644 index 00000000..701235ab --- /dev/null +++ b/smtp/smtp_wrapper.c @@ -0,0 +1,34 @@ +#include +#include + +int main(int argc, char **argv) { + // Set the environment variable + int result = setenv("LLVM_PROFILE_FILE", "/coverage/coverage-%p.profraw%c", 1); + + if (result != 0) { + return 1; + } + + // Define the program to execute (predefined) + char *program ="/usr/local/bin/smtp_"; + + // Create a new array to hold the arguments for execvp, including the program name + char** exec_args = (char**) malloc(sizeof(char*) * (unsigned long) (argc + 1)); + + // Set the program ßßßname as the first argument + exec_args[0] = program; + + // Copy the rest of the arguments passed to your program to the new exec_args array + for (int i = 1; i < argc; i++) { + exec_args[i] = argv[i]; + } + + // Terminate the array with a NULL pointer (as required by execvp) + exec_args[argc] = NULL; + + // Execute the program, replacing the current process + execvp(program, exec_args); + + // If execvp returns, it means it failed + return 1; +} diff --git a/test.sh b/test.sh index b34d2dea..b262744a 100755 --- a/test.sh +++ b/test.sh @@ -119,6 +119,7 @@ curl --url "pop3s://$SINGULARITY_HOSTNAME" \ | diff <(printf '\r\n') /dev/stdin CR=$(printf "\r") + # Check that the user can send a message to the server ( curl --url "smtps://$SINGULARITY_HOSTNAME" \ @@ -138,6 +139,71 @@ EOF | diff <(printf "") /dev/stdin +# Check that the user can send an email to multiple recipients +( +curl --url "smtps://$SINGULARITY_HOSTNAME" \ + --unix-socket ./socks/smtps.sock \ + "${CURL_OPTS[@]}" \ + --mail-from "user@$SINGULARITY_HOSTNAME" \ + --mail-rcpt "other@$SINGULARITY_HOSTNAME" \ + --mail-rcpt "other1@$SINGULARITY_HOSTNAME" \ + --upload-file - \ + --user "user:${REGISTER_PASS}" <$CR +To: "other1@$SINGULARITY_HOSTNAME" $CR +$CR +To whom it may concern,$CR +$CR +Bottom text$CR +EOF +) | tee test/smtp_send_email_multi_recipients \ + | diff <(printf "") /dev/stdin + + +# Check that the user can send an email with Cc (carbon copy) recipients +( +curl --url "smtps://$SINGULARITY_HOSTNAME" \ + --unix-socket ./socks/smtps.sock \ + "${CURL_OPTS[@]}" \ + --mail-from "user@$SINGULARITY_HOSTNAME" \ + --mail-rcpt "other@$SINGULARITY_HOSTNAME" \ + --mail-rcpt "other1@$SINGULARITY_HOSTNAME" \ + --upload-file - \ + --user "user:${REGISTER_PASS}" <$CR +Cc: "other1@$SINGULARITY_HOSTNAME" $CR +$CR +To whom it may concern,$CR +$CR +Bottom text$CR +EOF +) | tee test/smtp_send_email_multi_recipients \ + | diff <(printf "") /dev/stdin + +# Check that sending an email without to field fails +( +! curl --url "smtps://$SINGULARITY_HOSTNAME" \ + --unix-socket ./socks/smtps.sock \ + "${CURL_OPTS[@]}" \ + --mail-from "user@$SINGULARITY_HOSTNAME" \ + --mail-rcpt "other@$SINGULARITY_HOSTNAME" \ + --mail-rcpt "other1@$SINGULARITY_HOSTNAME" \ + --upload-file - \ + --user "user:${REGISTER_PASS}" <$CR +$CR +To whom it may concern,$CR +$CR +Bottom text$CR +EOF +) |& tee test/smtp_send_erroneous_email_wo_to \ + | grep 'Syntax error in message contents' + + # Verify that no email shows up without the journal being updated curl --url "pop3s://$SINGULARITY_HOSTNAME" \ --unix-socket ./socks/pop3s.sock \