diff --git a/.gitignore b/.gitignore index 3a4e9284..2b51df24 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ __pycache__ venv venv2 build +docker-compose.override.yml +/workspace diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41e8453c..893f2a57 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - - repo: https://github.com/psf/black - rev: 25.1.0 + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 25.11.0 hooks: - id: black - repo: https://github.com/pycqa/flake8 diff --git a/Changelog.md b/Changelog.md index 44a0011f..fc46e74f 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,6 +1,14 @@ # CHANGELOG All notable changes to this project will be documented here. +## [v2.9.0] +- Install stack with GHCup (#626) +- Fixed AI tester to report error when the specified `submission` file is not found (#663) +- Updated docker image to use Ubuntu 24.04 (#668) +- Fixed stack installation in Docker environment (#668) +- Removed `click` from server requirements.txt file (#679) +- Fixed bug in R tester setup that always triggered reinstallation of R dependencies (#680) + ## [v2.8.3] - Add troubleshooting section talking about Docker Content Trust (DCT) (#653) - Fixed Python tester to display fully qualified name when running pytest (#656) @@ -22,6 +30,7 @@ All notable changes to this project will be documented here. - Add `ai_tester` module to support AI-based autograding via `ai-autograding-feedback` (#625) - Add `ai` to list of testers (#628) - Fixed an `AttributeError` when handling exceptions in server `update_test_settings` (#629) +- Add tag functionalty to `ai_tester` (#631) - Added opt out feature to the `ai_tester` by searching for `NO_EXTERNAL_AI_FEEDBACK` (#632) - Modified R tester to always display test result messages, even when tests pass (#633) diff --git a/client/.dockerfiles/Dockerfile b/client/.dockerfiles/Dockerfile index 2a8ca916..ebc4f1ca 100644 --- a/client/.dockerfiles/Dockerfile +++ b/client/.dockerfiles/Dockerfile @@ -1,7 +1,10 @@ -ARG UBUNTU_VERSION=22.04 +ARG UBUNTU_VERSION=24.04 FROM ubuntu:$UBUNTU_VERSION +# Remove ubuntu user, added in the 23.04 image +RUN userdel -r ubuntu + RUN apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get -y install software-properties-common && \ DEBIAN_FRONTEND=noninteractive add-apt-repository -y ppa:deadsnakes/ppa && \ diff --git a/client/requirements.txt b/client/requirements.txt index 491f64bc..a9bf1f4c 100644 --- a/client/requirements.txt +++ b/client/requirements.txt @@ -1,6 +1,6 @@ -flask==3.1.1 -python-dotenv==1.1.1 -rq==2.4.1 -redis==6.2.0 -jsonschema==4.25.0 +flask==3.1.2 +python-dotenv==1.2.1 +rq==2.6.0 +redis==7.1.0 +jsonschema==4.25.1 Werkzeug==3.1.3 diff --git a/compose.yaml b/compose.yaml index 7239c6ec..39e38ec8 100644 --- a/compose.yaml +++ b/compose.yaml @@ -4,10 +4,10 @@ services: context: ./server dockerfile: ./.dockerfiles/Dockerfile args: - UBUNTU_VERSION: '22.04' + UBUNTU_VERSION: '24.04' LOGIN_USER: 'docker' WORKSPACE: '/home/docker/.autotesting' - image: markus-autotest-server-dev:1.3.0 + image: markus-autotest-server-dev:1.4.0 volumes: - ./server:/app:cached - venv_server:/home/docker/markus_venv @@ -28,8 +28,8 @@ services: context: ./client dockerfile: ./.dockerfiles/Dockerfile args: - UBUNTU_VERSION: '22.04' - image: markus-autotest-client-dev:1.3.0 + UBUNTU_VERSION: '24.04' + image: markus-autotest-client-dev:1.4.0 container_name: 'autotest-client' volumes: - ./client:/app:cached diff --git a/server/.dockerfiles/Dockerfile b/server/.dockerfiles/Dockerfile index f587b85d..60ca5a48 100644 --- a/server/.dockerfiles/Dockerfile +++ b/server/.dockerfiles/Dockerfile @@ -1,7 +1,9 @@ -ARG UBUNTU_VERSION=22.04 +ARG UBUNTU_VERSION=24.04 FROM ubuntu:$UBUNTU_VERSION AS base +# Remove ubuntu user, added in the 23.04 image +RUN userdel -r ubuntu ARG LOGIN_USER ARG WORKSPACE @@ -29,6 +31,7 @@ RUN apt-get update -y && \ libharfbuzz-dev \ libfribidi-dev \ libxml2-dev \ + libnuma-dev \ r-base RUN useradd -ms /bin/bash $LOGIN_USER && \ @@ -37,9 +40,8 @@ RUN useradd -ms /bin/bash $LOGIN_USER && \ adduser --disabled-login --no-create-home $worker && \ echo "$LOGIN_USER ALL=($worker) NOPASSWD:ALL" | EDITOR="tee -a" visudo && \ usermod -aG $worker $LOGIN_USER; \ - done - -RUN chmod a+x /home/${LOGIN_USER} + done && \ + chmod a+x /home/${LOGIN_USER} COPY . /app @@ -47,8 +49,8 @@ RUN find /app/autotest_server/testers -name requirements.system -exec {} \; RUN echo "TZ=$( cat /etc/timezone )" >> /etc/R/Renviron.site -RUN mkdir -p ${WORKSPACE} && chown ${LOGIN_USER} ${WORKSPACE} -RUN mkdir -p /home/${LOGIN_USER}/markus_venv && chown ${LOGIN_USER} /home/${LOGIN_USER}/markus_venv +RUN mkdir -p ${WORKSPACE} && chown ${LOGIN_USER} ${WORKSPACE} && \ + mkdir -p /home/${LOGIN_USER}/markus_venv && chown ${LOGIN_USER} /home/${LOGIN_USER}/markus_venv WORKDIR /home/${LOGIN_USER} diff --git a/server/autotest_server/__init__.py b/server/autotest_server/__init__.py index c11af9c4..7af410aa 100644 --- a/server/autotest_server/__init__.py +++ b/server/autotest_server/__init__.py @@ -83,6 +83,8 @@ def _create_test_group_result( result["annotations"] = res["annotations"] elif "tags" in res: result["tags"] = res["tags"] + elif "extra_marks" in res: + result["extra_marks"] = res["extra_marks"] elif "overall_comment" in res: result["overall_comment"] = res["overall_comment"] else: diff --git a/server/autotest_server/testers/ai/ai_tester.py b/server/autotest_server/testers/ai/ai_tester.py index c3fbf67f..8ecf6e04 100644 --- a/server/autotest_server/testers/ai/ai_tester.py +++ b/server/autotest_server/testers/ai/ai_tester.py @@ -53,6 +53,7 @@ def __init__( super().__init__(specs, test_class, resource_settings=resource_settings) self.annotations = [] self.overall_comments = [] + self.tags = [] def call_ai_feedback(self) -> dict: """ @@ -81,7 +82,17 @@ def call_ai_feedback(self) -> dict: return results submission_file = config.get("submission") - if self._term_in_file(submission_file): + try: + disallowed_term_in_file = self._term_in_file(submission_file) + except FileNotFoundError: + results[test_label] = { + "title": test_label, + "status": "error", + "message": f'Could not file submission file "{submission_file}"', + } + return results + + if disallowed_term_in_file: results[test_label] = { "title": test_label, "status": "success", @@ -96,15 +107,26 @@ def call_ai_feedback(self) -> dict: try: result = subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=timeout, env=env) output = result.stdout + + parsed = None + try: + parsed = json.loads(output) + except json.JSONDecodeError: + pass + if isinstance(parsed, dict): + if "tags" in parsed: + tags = parsed["tags"] + self.tags.extend(tags) + if "output" in parsed: + output = parsed["output"] + if output_mode == "overall_comment": self.overall_comments.append(output) results[test_label] = {"title": test_label, "status": "success"} elif output_mode == "annotations": - try: - annotations_data = json.loads(output) - annotations = annotations_data.get("annotations", annotations_data) - except json.JSONDecodeError as e: - raise ValueError(f"Invalid JSON in output for {test_label}: {e}") + if parsed is None: + raise ValueError(f"Unable to parse the output of '{output}'") + annotations = parsed.get("annotations", parsed) self.annotations.extend(annotations) results[test_label] = {"title": test_label, "status": "success"} elif output_mode == "message": @@ -122,23 +144,20 @@ def _term_in_file(self, file_path: str) -> bool: term = "NO_EXTERNAL_AI_FEEDBACK" path = Path(file_path) - try: - if path.suffix.lower() == ".pdf": - with open(file_path, "rb") as f: - reader = PyPDF2.PdfReader(f) - for page in reader.pages: - text = page.extract_text() or "" - if term in text: - return True - return False - else: - with open(file_path, "r", encoding="utf-8") as f: - for line in f: - if term in line: - return True - return False - except FileNotFoundError: - return True + if path.suffix.lower() == ".pdf": + with open(file_path, "rb") as f: + reader = PyPDF2.PdfReader(f) + for page in reader.pages: + text = page.extract_text() or "" + if term in text: + return True + return False + else: + with open(file_path, "r", encoding="utf-8") as f: + for line in f: + if term in line: + return True + return False @Tester.run_decorator def run(self) -> None: @@ -157,3 +176,5 @@ def after_tester_run(self) -> None: print(self.test_class.format_annotations(self.annotations)) if self.overall_comments: print(self.test_class.format_overall_comment(self.overall_comments, separator="\n\n")) + if self.tags: + print(self.test_class.format_tags(self.tags)) diff --git a/server/autotest_server/testers/haskell/haskell_tester.py b/server/autotest_server/testers/haskell/haskell_tester.py index 48674fde..78010e53 100644 --- a/server/autotest_server/testers/haskell/haskell_tester.py +++ b/server/autotest_server/testers/haskell/haskell_tester.py @@ -7,6 +7,9 @@ from ..tester import Tester, Test, TestError from ..specs import TestSpecs +home = os.getenv("HOME") +os.environ["PATH"] = f"{home}/.cabal/bin:{home}/.ghcup/bin:" + os.environ["PATH"] + class HaskellTest(Test): def __init__( diff --git a/server/autotest_server/testers/haskell/requirements.system b/server/autotest_server/testers/haskell/requirements.system index 341699d1..7b493126 100755 --- a/server/autotest_server/testers/haskell/requirements.system +++ b/server/autotest_server/testers/haskell/requirements.system @@ -1,8 +1,15 @@ #!/usr/bin/env bash -if ! dpkg -l ghc cabal-install haskell-stack &> /dev/null; then - apt-get -y update - DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' ghc cabal-install haskell-stack +apt-get -y update + +if ! dpkg -l ghc cabal-install &> /dev/null; then + DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' ghc cabal-install fi -stack update +if [ ! -x "$HOME/.ghcup/bin/ghcup" ] && [ ! -x "/usr/local/bin/stack" ]; then + BOOTSTRAP_HASKELL_NONINTERACTIVE=1 BOOTSTRAP_HASKELL_INSTALL_NO_STACK=1 curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh + $HOME/.ghcup/bin/ghcup install stack recommended + if [ "$(id -u)" -eq 0 ]; then + cp $HOME/.ghcup/bin/stack /usr/local/bin/ + fi +fi diff --git a/server/autotest_server/testers/haskell/setup.py b/server/autotest_server/testers/haskell/setup.py index 6bfb446f..48e4d3f7 100644 --- a/server/autotest_server/testers/haskell/setup.py +++ b/server/autotest_server/testers/haskell/setup.py @@ -3,7 +3,10 @@ import subprocess HASKELL_TEST_DEPS = ["tasty-discover", "tasty-quickcheck", "tasty-hunit"] -STACK_RESOLVER = "lts-16.17" +STACK_RESOLVER = "lts-21.21" + +home = os.getenv("HOME") +os.environ["PATH"] = f"{home}/.cabal/bin:{home}/.ghcup/bin:" + os.environ["PATH"] def create_environment(_settings, _env_dir, default_env_dir): @@ -16,22 +19,31 @@ def create_environment(_settings, _env_dir, default_env_dir): def install(): - subprocess.run( - os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system"), - check=True, - capture_output=True, - text=True, - ) + try: + subprocess.run( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system"), + check=True, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Error executing Haskell requirements.system: {e}") resolver = STACK_RESOLVER cmd = ["stack", "build", "--resolver", resolver, "--system-ghc", *HASKELL_TEST_DEPS] - subprocess.run(cmd, check=True, capture_output=True) - subprocess.run( - os.path.join(os.path.dirname(os.path.realpath(__file__)), "stack_permissions.sh"), - check=True, - shell=True, - capture_output=True, - text=True, - ) + try: + subprocess.run(cmd, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Error running {cmd}: {e}") + try: + subprocess.run( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "stack_permissions.sh"), + check=True, + shell=True, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Error running Haskell stack_permissions.sh: {e}") def settings(): diff --git a/server/autotest_server/testers/haskell/stack_permissions.sh b/server/autotest_server/testers/haskell/stack_permissions.sh index 22c3fd34..2bb5d2f3 100755 --- a/server/autotest_server/testers/haskell/stack_permissions.sh +++ b/server/autotest_server/testers/haskell/stack_permissions.sh @@ -2,4 +2,6 @@ echo "allow-different-user: true" >> $STACK_ROOT/config.yaml echo "recommend-stack-upgrade: false" >> $STACK_ROOT/config.yaml chmod a+w $STACK_ROOT/stack.sqlite3.pantry-write-lock chmod a+w $STACK_ROOT/global-project/.stack-work/stack.sqlite3.pantry-write-lock -chmod a+w $STACK_ROOT/pantry/pantry.sqlite3.pantry-write-lock \ No newline at end of file +chmod a+w $STACK_ROOT/pantry/pantry.sqlite3.pantry-write-lock +chmod a+r $WORKSPACE/.stack/config.yaml +chmod a+r $WORKSPACE/.stack/global-project/stack.yaml diff --git a/server/autotest_server/testers/jupyter/requirements.txt b/server/autotest_server/testers/jupyter/requirements.txt index 7857feff..5c413cae 100644 --- a/server/autotest_server/testers/jupyter/requirements.txt +++ b/server/autotest_server/testers/jupyter/requirements.txt @@ -1,3 +1,3 @@ pytest==7.1.2 -psycopg2-binary==2.9.10 +psycopg2-binary==2.9.11 git+https://github.com/MarkUsProject/autotest-helpers.git#subdirectory=notebook_helper diff --git a/server/autotest_server/testers/py/py_tester.py b/server/autotest_server/testers/py/py_tester.py index 5af506ff..61ae13c7 100644 --- a/server/autotest_server/testers/py/py_tester.py +++ b/server/autotest_server/testers/py/py_tester.py @@ -5,6 +5,7 @@ import pytest import sys from ..tester import Tester, Test +import uuid from ..specs import TestSpecs @@ -102,6 +103,7 @@ def __init__(self) -> None: self.results = {} self.tags = set() self.annotations = [] + self.extra_marks = [] self.overall_comments = [] def pytest_configure(self, config): @@ -118,6 +120,10 @@ def pytest_configure(self, config): "markers", "markus_message(text): indicate text that is displayed as part of the test output (even on success)", ) + config.addinivalue_line( + "markers", + "markus_extra_marks(mark, description, unit): add extra marks with custom description", + ) @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_runtest_makereport(self, item, call): @@ -174,6 +180,15 @@ def _process_markers(self, item): self.results[item.nodeid]["marks_total"] = marker.args[0] elif marker.name == "markus_marks_earned" and marker.args != [] and item.nodeid in self.results: self.results[item.nodeid]["marks_earned"] = marker.args[0] + elif marker.name == "markus_extra_marks" and marker.args != [] and item.nodeid in self.results: + extra_mark = { + "id": str(uuid.uuid4()), + "mark": marker.args[0], + "fn": item.nodeid, + "description": marker.args[1], + "unit": marker.args[2] if len(marker.args) > 2 else "points", + } + self.extra_marks.append(extra_mark) def pytest_collectreport(self, report): """ @@ -263,6 +278,7 @@ def __init__( """ super().__init__(specs, test_class, resource_settings=resource_settings) self.annotations = [] + self.extra_marks = [] self.overall_comments = [] self.tags = set() @@ -307,6 +323,7 @@ def _run_pytest_tests(self, test_file: str) -> List[Dict]: results.extend(plugin.results.values()) self.annotations = plugin.annotations self.overall_comments = plugin.overall_comments + self.extra_marks = plugin.extra_marks self.tags = plugin.tags finally: sys.stdout = sys.__stdout__ @@ -344,3 +361,5 @@ def after_tester_run(self) -> None: print(self.test_class.format_tags(self.tags)) if self.overall_comments: print(self.test_class.format_overall_comment(self.overall_comments, separator="\n\n")) + if self.extra_marks: + print(self.test_class.format_extra_marks(self.extra_marks)) diff --git a/server/autotest_server/testers/py/requirements.txt b/server/autotest_server/testers/py/requirements.txt index 6c2622fc..9fe9dc8f 100644 --- a/server/autotest_server/testers/py/requirements.txt +++ b/server/autotest_server/testers/py/requirements.txt @@ -1,2 +1,2 @@ pytest==7.1.2 -psycopg2-binary==2.9.10 +psycopg2-binary==2.9.11 diff --git a/server/autotest_server/testers/r/lib/r_tester_setup.R b/server/autotest_server/testers/r/lib/r_tester_setup.R index 4aef4b00..f56775cc 100644 --- a/server/autotest_server/testers/r/lib/r_tester_setup.R +++ b/server/autotest_server/testers/r/lib/r_tester_setup.R @@ -38,7 +38,7 @@ install_dep <- function(row) { } else { remote_type <- NA_character_ } - if (!('stringi' %in% rownames(installed.packages))) { + if (!('stringi' %in% rownames(installed.packages()))) { install.packages(name, configure.args="--disable-pkg-config") } diff --git a/server/autotest_server/testers/tester.py b/server/autotest_server/testers/tester.py index e581d58c..d34ee8c8 100644 --- a/server/autotest_server/testers/tester.py +++ b/server/autotest_server/testers/tester.py @@ -43,12 +43,7 @@ def get_total_points(self) -> int: @staticmethod def format_result( - test_name: str, - status: str, - output: str, - points_earned: int, - points_total: int, - time: Optional[int] = None, + test_name: str, status: str, output: str, points_earned: int, points_total: int, time: Optional[int] = None ) -> str: """ Formats a test result. @@ -115,6 +110,15 @@ def format_overall_comment(overall_comment_data: str | Iterable[str], separator: content = separator.join(overall_comment_data) return json.dumps({"overall_comment": content}) + @staticmethod + def format_extra_marks(extra_marks: List[Dict[str, Any]]) -> str: + """ + Formats extra marks data. + :param extra_marks: the contents of the extra marks + :return a json string representation of the extra marks data. + """ + return json.dumps({"extra_marks": extra_marks}) + @staticmethod def format_tags(tag_data: Iterable[str | dict[str, str]]) -> str: """ @@ -141,11 +145,7 @@ def passed_with_bonus(self, points_bonus: int, message: str = "") -> str: """ if points_bonus < 0: raise ValueError("The test bonus points must be >= 0") - result = self.format( - status=self.Status.PASS, - output=message, - points_earned=self.points_total + points_bonus, - ) + result = self.format(status=self.Status.PASS, output=message, points_earned=self.points_total + points_bonus) return result def passed(self, message: str = "") -> str: @@ -171,10 +171,7 @@ def partially_passed(self, points_earned: int, message: str) -> str: result = self.format(status=self.Status.PARTIAL, output=message, points_earned=points_earned) return result - def failed( - self, - message: str, - ) -> str: + def failed(self, message: str) -> str: """ Fails this test with 0 points earned. If a feedback file is enabled, adds feedback to it. :param message: The failure message, will be shown as test output. diff --git a/server/requirements.txt b/server/requirements.txt index bae260c1..c5c98d5d 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,8 +1,7 @@ -rq==2.4.1 -click==8.1.8 -redis==6.2.0 -pyyaml==6.0.2 -jsonschema==4.25.0 -requests==2.32.4 -psycopg2-binary==2.9.10 -supervisor==4.2.5 +rq==2.6.0 +redis==7.1.0 +pyyaml==6.0.3 +jsonschema==4.25.1 +requests==2.32.5 +psycopg2-binary==2.9.11 +supervisor==4.3.0