diff --git a/.github/workflows/build_and_publish.yaml b/.github/workflows/build_and_publish.yaml index 73850c1..f41632c 100644 --- a/.github/workflows/build_and_publish.yaml +++ b/.github/workflows/build_and_publish.yaml @@ -3,7 +3,8 @@ on: workflow_dispatch: push: branches: [main] - +env: + UBUNTU_PRO_TOKEN: ${{ secrets.UBUNTU_PRO_TOKEN }} jobs: publish: # note: this builds/tests all versions in serial for two reasons. Firstly we diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 045ea7d..654f35c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,6 +1,8 @@ name: Run tests on: pull_request: +env: + UBUNTU_PRO_TOKEN: ${{ secrets.UBUNTU_PRO_TOKEN }} jobs: version-tests: runs-on: ubuntu-22.04 diff --git a/.gitignore b/.gitignore index acbad03..7060b29 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ *~ venv +.venv __pycache__ +.env +.secrets diff --git a/DEVELOPERS.md b/DEVELOPERS.md index 2328c3f..749c968 100644 --- a/DEVELOPERS.md +++ b/DEVELOPERS.md @@ -29,6 +29,14 @@ just build v2 just test v2 ``` +## ESM Packages on 20.04 images + +We still support the older python:v1, but it is based on Ubuntu 20.04, which +has reached EOL for security upgrades. So we have enabled ESM via Ubuntu Pro for these images. + +This means that you do need a valid `UBUNTU_PRO_TOKEN` environment variable to +build these images. + ## Add a new package to existing version diff --git a/Dockerfile b/Dockerfile index 8bdf3af..1c3ddb3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.2 +# syntax=docker/dockerfile:1.10 ################################################# # # We need base python dependencies on both the builder and python images, so @@ -22,7 +22,12 @@ ENV ACTION_EXEC=python MAJOR_VERSION=${MAJOR_VERSION} BASE=${BASE} COPY ${MAJOR_VERSION}/dependencies.txt /opt/dependencies.txt # use space efficient utility from base image -RUN /root/docker-apt-install.sh /opt/dependencies.txt +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=bind,source=${MAJOR_VERSION}/dependencies.txt,target=/tmp/dependencies.txt \ + --mount=type=secret,id=ubuntu_pro_token,required=true \ + mkdir /workspace; \ + /root/docker-apt-install.sh /tmp/dependencies.txt # now we have python, set up a venv to install packages to, for isolation from # system python libraries @@ -42,8 +47,11 @@ FROM base-python as builder ARG MAJOR_VERSION # install build time dependencies -COPY ${MAJOR_VERSION}/build-dependencies.txt /opt/build-dependencies.txt -RUN /root/docker-apt-install.sh /opt/build-dependencies.txt +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=bind,source=${MAJOR_VERSION}/build-dependencies.txt,target=/tmp/build-dependencies.txt \ + --mount=type=secret,id=ubuntu_pro_token \ + /root/docker-apt-install.sh /tmp/build-dependencies.txt COPY ${MAJOR_VERSION}/requirements.txt /opt/requirements.txt COPY ${MAJOR_VERSION}/packages.md /opt/packages.md diff --git a/docker-compose.yml b/docker-compose.yml index e9f4f17..de4a539 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,8 @@ services: build: context: . target: base-python + secrets: + - ubuntu_pro_token cache_from: # should speed up the build in CI, where we have a cold cache - ghcr.io/opensafely-core/base-action:${BASE} - ghcr.io/opensafely-core/python:${MAJOR_VERSION} @@ -24,3 +26,7 @@ services: image: python:${MAJOR_VERSION} build: target: python + +secrets: + ubuntu_pro_token: + file: ${UBUNTU_PRO_TOKEN_FILE:-.secrets/ubuntu_pro_token} diff --git a/justfile b/justfile index 2b05e3f..bd03b18 100644 --- a/justfile +++ b/justfile @@ -1,3 +1,6 @@ +set dotenv-load := true + +export UBUNTU_PRO_TOKEN_FILE := env_var_or_default('UBUNTU_PRO_TOKEN_FILE', justfile_directory() + "/.secrets/ubuntu_pro_token") export DOCKER_BUILDKIT := "1" # technically, these could differ by 1 seconds, but thats unlikely and doesn't matter # human readable, used as label in docker image @@ -6,8 +9,20 @@ export BUILD_DATE := `date +'%y-%m-%dT%H:%M:%S.%3NZ'` export BUILD_NUMBER := `date +'%y%m%d%H%M%S'` export REVISION := `git rev-parse --short HEAD` +ensure-pro-token: + #!/bin/bash + set -euo pipefail + token_file="{{ UBUNTU_PRO_TOKEN_FILE }}" + if test -z "${UBUNTU_PRO_TOKEN:-}"; then + echo "UBUNTU_PRO_TOKEN is required to create $token_file" >&2 + exit 1 + fi + mkdir -p "$(dirname "$token_file")" + umask 077 + printf '%s' "$UBUNTU_PRO_TOKEN" > "$token_file" + # build docker image for version -build version target="python" *args="": +build version target="python" *args="": ensure-pro-token docker compose --env-file {{ version }}/env build --pull {{ args }} {{ target }} @@ -29,8 +44,9 @@ render version *args: # run linters check: - @docker pull hadolint/hadolint:v2.12.0 - @docker run --rm -i hadolint/hadolint:v2.12.0 < Dockerfile + @docker run --rm -i hadolint/hadolint:v2.14.0 < Dockerfile + @ls scripts/*.sh | xargs docker run --rm -v "$PWD:/mnt:ro" koalaman/shellcheck:v0.11.0 + @docker run --rm -v "$PWD:/repo:ro" --workdir /repo rhysd/actionlint:1.7.10 -color # publish version (dry run by default - pass "true" to perform publish) diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 0000000..faf0e89 --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,11 @@ +from pathlib import Path +import subprocess + +import pytest + +os_release = Path("/etc/os-release").read_text() + +@pytest.mark.skipif('VERSION_ID="20.04"' not in os_release, reason="20.04 only") +def test_esm(): + output = subprocess.check_output(["dpkg-query", "-W", "-f='${Package}\t${Version}\n'", "libssl1.1"], text=True) + assert "esm" in output