From 2810b7f9af865122f687bfd653654d5ebf98d4f0 Mon Sep 17 00:00:00 2001 From: nicomiguelino Date: Fri, 18 Apr 2025 13:13:52 -0700 Subject: [PATCH 1/2] fix: cache from the Docker Hub registry when building images --- tools/image_builder/__main__.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tools/image_builder/__main__.py b/tools/image_builder/__main__.py index 83f260145..d5e1b5aa9 100644 --- a/tools/image_builder/__main__.py +++ b/tools/image_builder/__main__.py @@ -34,6 +34,7 @@ def build_image( clean_build: bool, push: bool, dockerfiles_only: bool, + disable_registry_cache: bool, ) -> None: # Enable BuildKit os.environ['DOCKER_BUILDKIT'] = '1' @@ -127,10 +128,19 @@ def build_image( docker.buildx.build( context_path='.', cache=(not clean_build), - cache_from={ - 'type': 'local', - 'src': str(cache_dir), - } if not clean_build else None, + cache_from=[ + { + 'type': 'local', + 'src': str(cache_dir), + }, + { + 'type': 'registry', + 'ref': f'screenly/anthias-{service}:latest-pi4-64', + } if board == 'pi4' and target_platform == 'linux/arm64/v8' else { + 'type': 'registry', + 'ref': f'screenly/anthias-{service}:latest-{board}', + } + ] if not clean_build and not disable_registry_cache else None, cache_to={ 'type': 'local', 'dest': str(cache_dir), @@ -186,6 +196,11 @@ def build_image( '--dockerfiles-only', is_flag=True, ) +@click.option( + '--disable-registry-cache', + is_flag=True, + help='Disable caching from Docker Hub images', +) def main( clean_build: bool, build_target: str, @@ -195,6 +210,7 @@ def main( environment: str, push: bool, dockerfiles_only: bool, + disable_registry_cache: bool, ) -> None: git_branch = pygit2.Repository('.').head.shorthand git_hash = str(pygit2.Repository('.').head.target) @@ -244,6 +260,7 @@ def main( clean_build, push, dockerfiles_only, + disable_registry_cache, ) From 69664f94f8597602b828806b73ce6a95d62710c1 Mon Sep 17 00:00:00 2001 From: nicomiguelino Date: Fri, 18 Apr 2025 13:16:38 -0700 Subject: [PATCH 2/2] temporary: enable Docker image builds during pull request pushes --- .github/workflows/docker-build.yaml | 42 ++++++++++++++++++++++------- docker/Dockerfile.server.j2 | 29 +++++++++++--------- tools/image_builder/__main__.py | 1 + 3 files changed, 50 insertions(+), 22 deletions(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index de1639a49..2ebb3d9cc 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -1,6 +1,27 @@ name: Docker Image Build on: + # TODO: Remove. + pull_request: + branches: + - master + paths: + - '**' + - '!.editorconfig' + - '!webview/**' + - '!website/**' + - '!.github/workflows/deploy-website.yaml' + - '!.github/workflows/build-webview.yaml' + - '!.github/workflows/build-balena-disk-image.yaml' + - '!.github/workflows/python-lint.yaml' + - '!.github/pull_request_template.md' + - '!README.md' + - '!docs/**' + - '!bin/install.sh' + - '!bin/upgrade_containers.sh' + - '!bin/start_development_server.sh' + - '!tests/**' + - '!docker/Dockerfile.dev' push: branches: - master @@ -23,15 +44,19 @@ on: - '!docker/Dockerfile.dev' jobs: - run-tests: - uses: ./.github/workflows/docker-test.yaml + # TODO: Re-enable tests. + # run-tests: + # uses: ./.github/workflows/docker-test.yaml buildx: - needs: run-tests + # TODO: Re-enable tests. + # needs: run-tests strategy: matrix: board: ['pi1', 'pi2', 'pi3', 'pi4', 'pi4-64', 'pi5', 'x86'] - service: ['server', 'celery', 'redis', 'websocket', 'nginx', 'viewer', 'wifi-connect'] + # TODO: Uncomment all the services. + service: ['server'] + # service: ['server', 'celery', 'redis', 'websocket', 'nginx', 'viewer', 'wifi-connect'] python-version: ["3.11"] runs-on: ubuntu-24.04 @@ -101,19 +126,16 @@ jobs: poetry run python -m tools.image_builder \ --build-target=pi4 \ --target-platform=linux/arm/v8 \ - --service=${{ matrix.service }} \ - --push + --service=${{ matrix.service }} elif [ "${{ matrix.board }}" == "pi4-64" ]; then poetry run python -m tools.image_builder \ --build-target=pi4 \ --target-platform=linux/arm64/v8 \ - --service=${{ matrix.service }} \ - --push + --service=${{ matrix.service }} else poetry run python -m tools.image_builder \ --build-target=${{ matrix.board }} \ - --service=${{ matrix.service }} \ - --push + --service=${{ matrix.service }} fi - name: Inspect cache after build diff --git a/docker/Dockerfile.server.j2 b/docker/Dockerfile.server.j2 index ea58f70bc..c487aae37 100644 --- a/docker/Dockerfile.server.j2 +++ b/docker/Dockerfile.server.j2 @@ -1,23 +1,26 @@ {% if environment == 'production' %} FROM debian:bookworm AS node-builder +# Install node dependencies first to leverage cache RUN apt-get update && \ apt-get -y install \ nodejs \ - npm + npm && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* -RUN mkdir -p /app WORKDIR /app -COPY package.json \ - package-lock.json \ - webpack.prod.js \ - webpack.common.js \ - /app/ -RUN npm install +# Copy only package files first to leverage cache +COPY package.json package-lock.json ./ +RUN npm ci -COPY ./static/js/*.coffee /app/static/js/ -COPY ./static/sass/*.scss /app/static/sass/ +# Copy only the necessary files for the build +COPY webpack.prod.js webpack.common.js ./ +COPY ./static/js/*.coffee ./static/js/ +COPY ./static/sass/*.scss ./static/sass/ + +# Run the build RUN npm run build {% endif %} @@ -31,10 +34,12 @@ RUN --mount=type=cache,target=/root/.cache/pip \ {% endif %} pip3 install -r /tmp/requirements.txt --break-system-packages -RUN mkdir -p /usr/src/app -COPY . /usr/src/app/ +# Set up application directory WORKDIR /usr/src/app +# Copy application code +COPY . . + {% if environment == 'production' %} COPY --from=node-builder \ /app/static/dist/ \ diff --git a/tools/image_builder/__main__.py b/tools/image_builder/__main__.py index d5e1b5aa9..474839c92 100644 --- a/tools/image_builder/__main__.py +++ b/tools/image_builder/__main__.py @@ -38,6 +38,7 @@ def build_image( ) -> None: # Enable BuildKit os.environ['DOCKER_BUILDKIT'] = '1' + os.environ['BUILDKIT_INLINE_CACHE'] = '1' context = {}