Dockerfile and installation updates

Changelog.md

@@ -3,6 +3,9 @@ All notable changes to this project will be documented here.
 
 ## [unreleased]
 - Fixed Haskell test results to only include the function name (#687)
+- Improved robustness of tester installation scripts and Docker configuration (#688)
+- Moved tidyverse installation steps from server Dockerfile into R tester requirements.system (#688)
+- Fixed Haskell tester installation using ghcup to install stack system-wide (#688)
 
 ## [v2.9.0]
 - Install stack with GHCup (#626)

client/.dockerfiles/Dockerfile

@@ -12,4 +12,4 @@ RUN apt-get update -y && \
 
 WORKDIR /app
 
-CMD /markus_venv/bin/python run.py
+CMD ["/markus_venv/bin/python", "run.py"]

compose.yaml

@@ -7,7 +7,7 @@ services:
         UBUNTU_VERSION: '24.04'
         LOGIN_USER: 'docker'
         WORKSPACE: '/home/docker/.autotesting'
-    image: markus-autotest-server-dev:1.4.0
+    image: markus-autotest-server-dev:1.5.0
     volumes:
       - ./server:/app:cached
       - venv_server:/home/docker/markus_venv
@@ -29,7 +29,7 @@ services:
       dockerfile: ./.dockerfiles/Dockerfile
       args:
         UBUNTU_VERSION: '24.04'
-    image: markus-autotest-client-dev:1.4.0
+    image: markus-autotest-client-dev:1.5.0
     container_name: 'autotest-client'
     volumes:
       - ./client:/app:cached

server/.dockerfiles/Dockerfile

@@ -8,7 +8,9 @@ RUN userdel -r ubuntu
 ARG LOGIN_USER
 ARG WORKSPACE
 
-RUN apt-get update -y && \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get -y install software-properties-common && \
     DEBIAN_FRONTEND=noninteractive add-apt-repository -y ppa:deadsnakes/ppa && \
     DEBIAN_FRONTEND=noninteractive apt-get -y install python3.11 \
@@ -21,18 +23,7 @@ RUN apt-get update -y && \
                        postgresql-client \
                        libpq-dev \
                        sudo \
-                       git \
-                       libfontconfig1-dev \
-                       libcurl4-openssl-dev \
-                       libfreetype6-dev \
-                       libpng-dev \
-                       libtiff5-dev \
-                       libjpeg-dev \
-                       libharfbuzz-dev \
-                       libfribidi-dev \
-                       libxml2-dev \
-                       libnuma-dev \
-                       r-base
+                       git
 
 RUN useradd -ms /bin/bash $LOGIN_USER && \
     usermod -aG sudo $LOGIN_USER && \
@@ -43,17 +34,28 @@ RUN useradd -ms /bin/bash $LOGIN_USER && \
     done && \
     chmod a+x /home/${LOGIN_USER}
 
-COPY . /app
+RUN mkdir -p ${WORKSPACE} && chown ${LOGIN_USER} ${WORKSPACE} && \
+    mkdir -p /home/${LOGIN_USER}/markus_venv && chown ${LOGIN_USER} /home/${LOGIN_USER}/markus_venv
 
-RUN find /app/autotest_server/testers -name requirements.system -exec {} \;
 
-RUN echo "TZ=$( cat /etc/timezone )" >> /etc/R/Renviron.site
+# Copy requirements.system files for all testers. These are copied separately from other files
+# to avoid Docker cache invalidation of the subsequent RUN command if any other files are changed
+# in markus-autotesting/server.
+COPY --parents \
+    ./autotest_server/testers/java/requirements.system \
+    ./autotest_server/testers/haskell/requirements.system \
+    ./autotest_server/testers/r/requirements.system \
+    ./autotest_server/testers/racket/requirements.system \
+    /app
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    --mount=type=cache,target=$WORKSPACE/.stack,sharing=locked \
+    find /app/autotest_server/testers -name requirements.system -exec {} \;
 
-RUN mkdir -p ${WORKSPACE} && chown ${LOGIN_USER} ${WORKSPACE} && \
-    mkdir -p /home/${LOGIN_USER}/markus_venv && chown ${LOGIN_USER} /home/${LOGIN_USER}/markus_venv
+COPY . /app
 
 WORKDIR /home/${LOGIN_USER}
 
 USER ${LOGIN_USER}
 
-CMD /app/.dockerfiles/cmd-dev.sh
+CMD ["/app/.dockerfiles/cmd-dev.sh"]

server/autotest_server/testers/__init__.py

@@ -10,6 +10,7 @@ def install(testers=_TESTERS):
     for tester in testers:
         mod = importlib.import_module(f".{tester}.setup", package="autotest_server.testers")
         try:
+            print(f"[AUTOTESTER] calling autotest_server.testers.{tester}.setup.install()")
             mod.install()
         except Exception as e:
             msg = (

server/autotest_server/testers/haskell/haskell_tester.py

@@ -7,9 +7,6 @@
 from ..tester import Tester, Test, TestError
 from ..specs import TestSpecs
 
-home = os.getenv("HOME")
-os.environ["PATH"] = f"{home}/.cabal/bin:{home}/.ghcup/bin:" + os.environ["PATH"]
-
 
 class HaskellTest(Test):
     def __init__(

server/autotest_server/testers/haskell/requirements.system

@@ -1,15 +1,25 @@
 #!/usr/bin/env bash
+set -euxo pipefail
 
-apt-get -y update
-
+# Install a system-wide ghc, which can be used as a default version in the Haskell tester.
+# This should be synchronized with the LTS version and dependencies in setup.py
 if ! dpkg -l ghc cabal-install &> /dev/null; then
+  apt-get -y update
   DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' ghc cabal-install
 fi
 
-if [ ! -x "$HOME/.ghcup/bin/ghcup" ] && [ ! -x "/usr/local/bin/stack" ]; then
-  BOOTSTRAP_HASKELL_NONINTERACTIVE=1 BOOTSTRAP_HASKELL_INSTALL_NO_STACK=1 curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh
-  $HOME/.ghcup/bin/ghcup install stack recommended
-  if [ "$(id -u)" -eq 0 ]; then
-    cp $HOME/.ghcup/bin/stack /usr/local/bin/
-  fi
+if [ ! -x "/usr/local/bin/stack" ]; then
+  # We use ghcup to install stack rather than relying on system packages. This enables newer versions of Stack to be installed.
+  # Install ghcup dependencies: https://www.haskell.org/ghcup/install/#linux-ubuntu
+  apt-get -y update
+  DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' \
+    build-essential \
+    curl \
+    libffi-dev \
+    libffi8ubuntu1 \
+    libgmp-dev \
+    libgmp10 \
+    libncurses-dev
+  curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 BOOTSTRAP_HASKELL_MINIMAL=1 sh
+  $HOME/.ghcup/bin/ghcup install stack recommended --isolate /usr/local/bin
 fi

server/autotest_server/testers/haskell/setup.py

@@ -5,9 +5,6 @@
 HASKELL_TEST_DEPS = ["tasty-discover", "tasty-quickcheck", "tasty-hunit"]
 STACK_RESOLVER = "lts-21.21"
 
-home = os.getenv("HOME")
-os.environ["PATH"] = f"{home}/.cabal/bin:{home}/.ghcup/bin:" + os.environ["PATH"]
-
 
 def create_environment(_settings, _env_dir, default_env_dir):
     env_data = _settings.get("env_data", {})
@@ -20,26 +17,33 @@ def create_environment(_settings, _env_dir, default_env_dir):
 
 def install():
     try:
-        subprocess.run(
-            os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system"),
-            check=True,
-            capture_output=True,
-            text=True,
-        )
+        path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system")
+        print(f"[AUTOTESTER] Running {path}", flush=True)
+        subprocess.run(path, check=True)
     except subprocess.CalledProcessError as e:
         raise RuntimeError(f"Error executing Haskell requirements.system: {e}")
     resolver = STACK_RESOLVER
-    cmd = ["stack", "build", "--resolver", resolver, "--system-ghc", *HASKELL_TEST_DEPS]
+
+    cmd_update = ["stack", "update"]
+    print(f'[AUTOTESTER] Running {" ".join(cmd_update)}', flush=True)
+    try:
+        subprocess.run(cmd_update, check=True)
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError(f"Error running {cmd_update}: {e}")
+
+    cmd_build = ["stack", "build", "--resolver", resolver, "--system-ghc", *HASKELL_TEST_DEPS]
+    print(f'[AUTOTESTER] Running {" ".join(cmd_build)}', flush=True)
     try:
-        subprocess.run(cmd, check=True, capture_output=True)
+        subprocess.run(cmd_build, check=True)
     except subprocess.CalledProcessError as e:
-        raise RuntimeError(f"Error running {cmd}: {e}")
+        raise RuntimeError(f"Error running {cmd_build}: {e}")
     try:
+        path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stack_permissions.sh")
+        print(f"[AUTOTESTER] Running {path}", flush=True)
         subprocess.run(
-            os.path.join(os.path.dirname(os.path.realpath(__file__)), "stack_permissions.sh"),
+            path,
             check=True,
             shell=True,
-            capture_output=True,
             text=True,
         )
     except subprocess.CalledProcessError as e:

server/autotest_server/testers/haskell/stack_permissions.sh

@@ -1,3 +1,6 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
 echo "allow-different-user: true" >> $STACK_ROOT/config.yaml
 echo "recommend-stack-upgrade: false" >> $STACK_ROOT/config.yaml
 chmod a+w $STACK_ROOT/stack.sqlite3.pantry-write-lock

server/autotest_server/testers/java/requirements.system

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -euxo pipefail
 
 if ! dpkg -l openjdk-8-jdk wget &> /dev/null; then
   apt-get -y update

server/autotest_server/testers/java/setup.py

@@ -10,7 +10,9 @@ def create_environment(_settings, _env_dir, default_env_dir):
 
 def install():
     this_dir = os.path.dirname(os.path.realpath(__file__))
-    subprocess.run(os.path.join(this_dir, "requirements.system"), check=True)
+    path = os.path.join(this_dir, "requirements.system")
+    print(f"[AUTOTESTER] Running {path}", flush=True)
+    subprocess.run(path, check=True)
     url = (
         "https://repo1.maven.org/maven2/org/junit/platform/junit-platform-console-standalone/1.7.0/junit-platform"
         "-console-standalone-1.7.0.jar"

server/autotest_server/testers/r/requirements.system

@@ -1,6 +1,29 @@
 #!/usr/bin/env bash
+set -euxo pipefail
 
 if ! dpkg -l r-base &> /dev/null; then
   apt-get -y update
   DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' r-base
+
+  # Set global R timezone
+  echo "TZ=$( cat /etc/timezone )" >> /etc/R/Renviron.site
+
+  # Install system requirements for tidyverse. Obtained by running `pak::pkg_system_requirements("tidyverse")`.
+  DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' \
+    libx11-dev \
+    libcurl4-openssl-dev \
+    libssl-dev \
+    make \
+    zlib1g-dev \
+    pandoc \
+    libfreetype6-dev \
+    libjpeg-dev \
+    libpng-dev \
+    libtiff-dev \
+    libwebp-dev \
+    libicu-dev \
+    libfontconfig1-dev \
+    libfribidi-dev \
+    libharfbuzz-dev \
+    libxml2-dev
 fi

server/autotest_server/testers/r/setup.py

@@ -39,4 +39,6 @@ def settings():
 
 
 def install():
-    subprocess.run(os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system"), check=True)
+    path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system")
+    print(f"[AUTOTESTER] Running {path}", flush=True)
+    subprocess.run(path, check=True)

server/autotest_server/testers/racket/requirements.system

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -euxo pipefail
 
 if ! dpkg -l racket &> /dev/null; then
   apt-get -y update

server/autotest_server/testers/racket/setup.py

@@ -13,4 +13,6 @@ def settings():
 
 
 def install():
-    subprocess.run(os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system"), check=True)
+    path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.system")
+    print(f"[AUTOTESTER] Running {path}", flush=True)
+    subprocess.run(path, check=True)