diff --git a/.dockerignore b/.dockerignore index e63c0c184..f9ea40b62 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,4 @@ !.coveragerc !.env !.pylintrc +DOCS/** diff --git a/.gitattributes b/.gitattributes index 176a458f9..e6f5bad21 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,3 @@ -* text=auto +* text=auto eol=lf +*.ico binary +ghostwriter/static/images/** binary diff --git a/.github/workflows/push-container.yml b/.github/workflows/push-container.yml new file mode 100644 index 000000000..a322acdcc --- /dev/null +++ b/.github/workflows/push-container.yml @@ -0,0 +1,187 @@ +name: push-container + +on: + push: + branches: + - 'master' + - 'feature/push-to-container-registry' + - 'feature/push-to-container-registry-v3.0.0' + tags: + - 'v*' + schedule: + # * is a special character in YAML so you have to quote this string + # at 03:00 on the 1st and 15th of the month + - cron: '0 3 1,15 * *' + +env: + PLATFORMS: linux/amd64 # multiple platforms can be specified as: linux/amd64,linux/arm64 + +jobs: + build_and_push_container_image: + runs-on: ubuntu-latest + + permissions: + # when permissions are defined only those that are explicitly set will be enabled + # this workflow job currently only requires reading contents and writing packages. + # https://docs.github.com/en/actions/reference/authentication-in-a-workflow#modifying-the-permissions-for-the-github_token + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Validate secret defined + id: from_secrets + run: | + github_container_push="true"; + dockerhub_token_exists="false"; + dockerhub_username_exists="false"; + dockerhub_namespace_exists="false"; + + [[ -n "${{ secrets.DOCKERHUB_TOKEN }}" ]] && dockerhub_token_exists="true"; + [[ -n "${{ secrets.DOCKERHUB_USERNAME }}" ]] && dockerhub_username_exists="true"; + [[ -n "${{ secrets.DOCKERHUB_NAMESPACE }}" ]] && dockerhub_namespace_exists="true"; + [[ "true" = "${{ secrets.GITHUB_CONTAINER_PUSH_DISABLED }}" ]] && github_container_push="false"; + + echo "::set-output name=dockerhub_token_exists::${dockerhub_token_exists}"; + echo "::set-output name=dockerhub_username_exists::${dockerhub_username_exists}"; + echo "::set-output name=dockerhub_namespace_exists::${dockerhub_namespace_exists}"; + echo "::set-output name=github_container_push::${github_container_push}"; + + - name: Generate container image names + id: generate_image_names + run: | + repository_name="$(basename "${GITHUB_REPOSITORY}")"; + images=(); + + if [[ "${{ steps.from_secrets.outputs.github_container_push }}" = "true" ]]; + then + # set GITHUB_CONTAINER_PUSH_DISABLED to a value of true to disable pushing to github container registry + images+=("ghcr.io/${GITHUB_REPOSITORY}"); + fi + + if [[ -n "${{ secrets.DOCKERHUB_TOKEN }}" ]] && [[ -n "${{ secrets.DOCKERHUB_USERNAME }}" ]] && [[ -n "${{ secrets.DOCKERHUB_NAMESPACE }}" ]]; + then + # dockerhub repository should be the same as the github repository name, within the dockerhub namespace (organization or personal) + images+=("${{ secrets.DOCKERHUB_NAMESPACE }}/${repository_name}"); + fi + + # join the array for Docker meta job to produce image tags + # https://github.com/crazy-max/ghaction-docker-meta#inputs + echo "::set-output name=images::$(IFS=,; echo "${images[*]}")"; + + - name: Docker ghostwriter meta + id: meta + uses: docker/metadata-action@v3 + with: + images: ${{ steps.generate_image_names.outputs.images }} + tags: | + type=schedule,pattern={{date 'YYYYMMDD'}} + type=edge,branch=master + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - name: Docker ghostwriter:postgres meta + id: meta-postgres + uses: docker/metadata-action@v3 + with: + images: ${{ steps.generate_image_names.outputs.images }} + flavor: | + prefix=postgres- + tags: | + type=schedule,pattern={{date 'YYYYMMDD'}} + type=edge,branch=master + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + with: + version: latest + + - name: Login to DockerHub + uses: docker/login-action@v1 + # conditions do not have direct access to github secrets so we check the output of the step from_secrets + if: ${{ steps.from_secrets.outputs.dockerhub_namespace_exists == 'true' && steps.from_secrets.outputs.dockerhub_token_exists == 'true' && steps.from_secrets.outputs.dockerhub_username_exists == 'true' }} + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + if: ${{ steps.from_secrets.outputs.github_container_push == 'true' }} + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + # Caches are scoped to the current branch and parent branch. + # Cache miss can happen on first run of a new branch + # If there is a matching cache key in the default branch then that should be used. + # https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows#matching-a-cache-key + # cache key is a hash of the base and production requirements. Changes to these files will cause a full rebuild. + key: ${{ runner.os }}-buildx-${{ hashFiles('requirements/base.txt', 'requirements/production.txt') }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Build and push - ghostwriter + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: ./compose/django/Dockerfile + platforms: ${{ env.PLATFORMS }} + push: ${{ contains(fromJson('["push", "schedule"]'), github.event_name) }} + labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + # type=gha will replace type=local when a buildx release containing + # https://github.com/docker/buildx/commit/5ca0cbff8ed63450a6d4a3b32659e9521d329a43 is published + # https://github.com/docker/buildx/pull/535 + # cache-from: type=gha + # cache-to: type=gha + + - name: Build and push - ghostwriter:postgres + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: ./compose/postgres/Dockerfile + platforms: ${{ env.PLATFORMS }} + push: ${{ contains(fromJson('["push", "schedule"]'), github.event_name) }} + labels: ${{ steps.meta-postgres.outputs.labels }} + tags: ${{ steps.meta-postgres.outputs.tags }} + cache-from: type=local,src=/tmp/.buildx-cache-new + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + # type=gha will replace type=local when a buildx release containing + # https://github.com/docker/buildx/commit/5ca0cbff8ed63450a6d4a3b32659e9521d329a43 is published + # https://github.com/docker/buildx/pull/535 + # cache-from: type=gha + # cache-to: type=gha + + - name: Move cache + # This step can be removed when cache-from/cache-to have been updated to use type=gha + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + if: always() + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/compose/django/Dockerfile b/compose/django/Dockerfile new file mode 100644 index 000000000..5248f5b8f --- /dev/null +++ b/compose/django/Dockerfile @@ -0,0 +1,162 @@ +#syntax=docker/dockerfile:1 +ARG STAGE=production + +# --------------------------------------------- +# BEGIN build image stage +# --------------------------------------------- +FROM python:3.8-alpine as build +ARG STAGE=production + +# only update build build when requirements have changed +COPY ./requirements /requirements +# install build dependencies +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + apk update \ + && apk add --no-cache build-base \ + # psycopg2 dependencies + && apk add --no-cache --virtual build-deps gcc python3-dev musl-dev \ + && apk add --no-cache postgresql-dev \ + # Pillow dependencies + && apk add --no-cache jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \ + # CFFI dependencies + && apk add --no-cache libffi-dev py-cffi \ + # XLSX dependencies + && apk add --no-cache libxml2-dev libxslt-dev \ + # Rust and Cargo required by the ``cryptography`` Python package - only required during build + && apk add --no-cache rust \ + && apk add --no-cache cargo \ + # build wheels + && pip install wheel && pip wheel --wheel-dir=/tmp/wheels -r /requirements/${STAGE}.txt \ + # remove the virtual package group 'build-deps' + && apk del build-deps +# --------------------------------------------- +# END build image stage +# --------------------------------------------- + +# --------------------------------------------- +# BEGIN django image stage +# --------------------------------------------- +FROM python:3.8-alpine as django +ARG STAGE=production + +# stream python output for django logs +ENV PYTHONUNBUFFERED 1 + +ENV PYTHONPATH="$PYTHONPATH:/app/config" + +ARG USER_UID=1000 +ARG USER_GID=$USER_UID +RUN if [ -n "$(getent group ${USER_GID})" ]; \ + then \ + apk --no-cache add shadow; \ + groupmod -n "django" "${USER_GID}"; \ + else \ + addgroup --gid "${USER_GID}" "django"; \ + fi && \ + if [ -n "$(getent passwd ${USER_UID})" ]; \ + then \ + apk --no-cache add shadow; \ + usermod -l "django" -g "${USER_GID}" -d "/app"; \ + else \ + adduser \ + --home "/app" \ + --shell /bin/ash \ + --ingroup "django" \ + --system \ + --disabled-password \ + --no-create-home \ + --uid "${USER_UID}" \ + "django"; \ + fi + +# install runtime dependencies. `add --no-cache` performs an apk update, adds packages and excludes caching +# in order to not require deletion of apk cache. +RUN apk add --no-cache postgresql-dev \ + # Pillow dependencies + jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \ + # CFFI dependencies + libffi-dev py-cffi \ + # XLSX dependencies + libxml2-dev libxslt-dev + +# combine build and ${STAGE}.txt - remove --no-binary to install our own wheels +RUN --mount=type=bind,target=/tmp/wheels,source=/tmp/wheels,from=build \ + --mount=type=bind,target=/requirements,source=/requirements,from=build,readwrite \ + --mount=type=cache,mode=0755,target=/root/.cache/pip \ + ( cat /requirements/base.txt; sed -e 's/--no-binary.*//' -e 's/^-r .*//' /requirements/${STAGE}.txt ) | tee /tmp/requirements.txt >/dev/null \ + && pip install --find-links=/tmp/wheels -r /tmp/requirements.txt \ + && rm -rf /tmp/requirements.txt +# --------------------------------------------- +# END django image stage +# --------------------------------------------- + +# --------------------------------------------- +# BEGIN production stage +# --------------------------------------------- +FROM django as django-production + +# add our application +COPY --chown=django . /app + +# copy the entrypoint and run scripts +RUN for target in /app/compose/django/*; \ + do ln "$target" /"$(basename "$target")" \ + && chmod -v 0755 /"$(basename "$target")" \ + # remove all carriage returns in the case that a user checks out the files on a windows system + # and has their git core.eol set to native or crlf + && sed -i 's/\r$//g' /"$(basename "$target")"; \ + done \ + # due to volumes mounted to these locations we must created and set the ownership of the underlying directory + # so that it is correctly propagated to the named volume + && mkdir -p "/app/ghostwriter/media" "/app/staticfiles" \ + && chown -R "django": "/app/ghostwriter/media" "/app/staticfiles" +# --------------------------------------------- +# END production stage +# --------------------------------------------- + +# --------------------------------------------- +# BEGIN local stage +# --------------------------------------------- +FROM django as django-local + +# add our application CMD scripts +COPY --chown=django ./compose/django/ / + +# copy the entrypoint and run scripts +RUN find / -maxdepth 1 -type f -exec chmod -v 0755 {} \; \ + # remove all carriage returns in the case that a user checks out the files on a windows system + # and has their git core.eol set to native or crlf + && find / -maxdepth 1 -type f -exec sed -i 's/\r$//g' {} \; \ + # due to volumes mounted to these locations we must created and set the ownership of the underlying directory + # so that it is correctly propagated to the named volume + && mkdir -p "/app/ghostwriter/media" "/app/staticfiles" \ + && chown -R "django": "/app/ghostwriter/media" "/app/staticfiles" +# --------------------------------------------- +# END local stage +# --------------------------------------------- + +# --------------------------------------------- +# BEGIN conditional stage +# with buildkit/bake only referenced stages will be built starting from this stage +# --------------------------------------------- +FROM django-${STAGE} as conditional + +USER "django" + +WORKDIR /app +# --------------------------------------------- +# END conditional stage +# --------------------------------------------- + +# --------------------------------------------- +# BEGIN live stage +# --------------------------------------------- +FROM conditional as live + +VOLUME ["/app/ghostwriter/media", "/app/staticfiles"] + +CMD ["/start"] +ENTRYPOINT ["/entrypoint"] +# --------------------------------------------- +# END live stage +# --------------------------------------------- diff --git a/compose/production/django/entrypoint b/compose/django/entrypoint similarity index 100% rename from compose/production/django/entrypoint rename to compose/django/entrypoint diff --git a/compose/local/django/seed_data b/compose/django/seed_data similarity index 100% rename from compose/local/django/seed_data rename to compose/django/seed_data diff --git a/compose/django/start b/compose/django/start new file mode 100644 index 000000000..68d3dd911 --- /dev/null +++ b/compose/django/start @@ -0,0 +1,84 @@ +#!/bin/ash +# shellcheck shell=dash + +set -o errexit +set -o pipefail +set -o nounset + +readonly AVATAR_DIR=/app/ghostwriter/media/user_avatars +readonly EVIDENCE_DIR=/app/ghostwriter/media/evidence +readonly TEMPLATE_DIR=/app/ghostwriter/media/templates + +readonly TEMPLATE_PATH_DOCX=/app/ghostwriter/reporting/templates/reports/template.docx +readonly TEMPLATE_PATH_PPTX=/app/ghostwriter/reporting/templates/reports/template.pptx + +create_dir() { + local target_directory="$1" + local target_user="${2:-django}" + + if test -d "$target_directory"; then + return 0 + fi + + mkdir -p "$target_directory" + chown -R "$target_user": "$target_directory" +} + +prepare() { + create_dir "$EVIDENCE_DIR" "${USER:-django}" + create_dir "$AVATAR_DIR" "${USER:-django}" + create_dir "$TEMPLATE_DIR" "${USER:-django}" + + cp -u -p "$TEMPLATE_PATH_DOCX" "${TEMPLATE_DIR}/" + cp -u -p "$TEMPLATE_PATH_PPTX" "${TEMPLATE_DIR}/" +} + +queue() { + python manage.py qcluster +} + +live() { + python /app/manage.py collectstatic --noinput + python /app/manage.py migrate + # /usr/local/bin/daphne -b 0.0.0.0 -p 5000 config.asgi:application + uvicorn config.asgi:application --host 0.0.0.0 --port 5000 +} + +dev() { + python manage.py makemigrations + python manage.py migrate + # python manage.py runserver 0.0.0.0:8000 + uvicorn config.asgi:application --host 0.0.0.0 --reload +} + +debug() { + python -m pip install --upgrade pip + python manage.py makemigrations + python manage.py migrate + pip install debugpy -t /tmp + python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 manage.py runserver 0.0.0.0:8000 --nothreading +} + +main() { + local target="${1:-live}" + + prepare + case "$target" in + live | debug | dev) + prepare + $target + ;; + queue) + $target + ;; + *) + echo >&2 "Unknown target: $target" + echo >&2 "Usage: $0 [live|debug|dev|queue]" + echo >&2 "Defaulting to production equivalent: $0 live" + prepare + live + ;; + esac +} + +main "$@" diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile deleted file mode 100644 index 17c924e76..000000000 --- a/compose/local/django/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -FROM python:3.8-alpine3.14 - -ENV PYTHONUNBUFFERED 1 - -ENV PYTHONDONTWRITEBYTECODE 1 - -RUN apk --no-cache add build-base \ - # psycopg2 dependencies - && apk --no-cache add --virtual build-deps gcc python3-dev musl-dev \ - && apk --no-cache add postgresql-dev \ - # Pillow dependencies - && apk --no-cache add jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \ - # CFFI dependencies - && apk --no-cache add libffi-dev py-cffi \ - # XLSX dependencies - && apk --no-cache add libxml2-dev libxslt-dev \ - # Translations dependencies - && apk --no-cache add gettext \ - # https://docs.djangoproject.com/en/dev/ref/django-admin/#dbshell - && apk --no-cache add postgresql-client \ - # Rust and Cargo required by the ``cryptography`` Python package - && apk --no-cache add rust \ - && apk --no-cache add cargo \ - && pip install --no-cache-dir -U setuptools pip - -COPY ./requirements /requirements - -RUN pip install --no-cache-dir -r /requirements/local.txt - -COPY ./compose/production/django/entrypoint /entrypoint - -RUN sed -i 's/\r$//g' /entrypoint \ - && chmod +x /entrypoint - -COPY ./compose/local/django/start /start - -RUN sed -i 's/\r$//g' /start \ - && chmod +x /start - -COPY ./compose/production/django/queue/start /start-queue - -RUN sed -i 's/\r//' /start-queue \ - && chmod +x /start-queue - -COPY ./compose/local/django/seed_data /seed_data - -RUN sed -i 's/\r$//g' /seed_data \ - && chmod +x /seed_data - -WORKDIR /app - -RUN mkdir -p /app/ghostwriter/media - -VOLUME ["/app/ghostwriter/media"] - -ENTRYPOINT ["/entrypoint"] diff --git a/compose/local/django/start b/compose/local/django/start deleted file mode 100644 index 73a37561f..000000000 --- a/compose/local/django/start +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -set -o errexit -set -o pipefail -set -o nounset - -AVATAR_DIR=/app/ghostwriter/media/user_avatars -EVIDENCE_DIR=/app/ghostwriter/media/evidence -TEMPLATE_DIR=/app/ghostwriter/media/templates -ARCHIVE_DIR=/app/ghostwriter/media/archives - -TEMPLATE_PATH_DOCX=/app/ghostwriter/reporting/templates/reports/template.docx -TEMPLATE_PATH_PPTX=/app/ghostwriter/reporting/templates/reports/template.pptx - -[[ ! -d "$EVIDENCE_DIR" ]] && mkdir -p "$EVIDENCE_DIR" -[[ ! -d "$AVATAR_DIR" ]] && mkdir -p "$AVATAR_DIR" -[[ ! -d "$TEMPLATE_DIR" ]] && mkdir -p "$TEMPLATE_DIR" -[[ ! -d "$ARCHIVE_DIR" ]] && mkdir -p "$ARCHIVE_DIR" - -cp -u -p "$TEMPLATE_PATH_DOCX" "$TEMPLATE_DIR" -cp -u -p "$TEMPLATE_PATH_PPTX" "$TEMPLATE_DIR" - -python manage.py makemigrations -python manage.py migrate -# python manage.py runserver 0.0.0.0:8000 -uvicorn config.asgi:application --host 0.0.0.0 --reload diff --git a/compose/local/django/start_debug b/compose/local/django/start_debug deleted file mode 100644 index b874cf10c..000000000 --- a/compose/local/django/start_debug +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -set -o errexit -set -o pipefail -set -o nounset - -AVATAR_DIR=/app/ghostwriter/media/user_avatars -EVIDENCE_DIR=/app/ghostwriter/media/evidence -TEMPLATE_DIR=/app/ghostwriter/media/templates -ARCHIVE_DIR=/app/ghostwriter/media/archives - -TEMPLATE_PATH_DOCX=/app/ghostwriter/reporting/templates/reports/template.docx -TEMPLATE_PATH_PPTX=/app/ghostwriter/reporting/templates/reports/template.pptx - -[[ ! -d "$EVIDENCE_DIR" ]] && mkdir -p "$EVIDENCE_DIR" -[[ ! -d "$AVATAR_DIR" ]] && mkdir -p "$AVATAR_DIR" -[[ ! -d "$TEMPLATE_DIR" ]] && mkdir -p "$TEMPLATE_DIR" -[[ ! -d "$ARCHIVE_DIR" ]] && mkdir -p "$ARCHIVE_DIR" - -cp -u -p "$TEMPLATE_PATH_DOCX" "$TEMPLATE_DIR" -cp -u -p "$TEMPLATE_PATH_PPTX" "$TEMPLATE_DIR" - -python -m pip install --upgrade pip -python manage.py makemigrations -python manage.py migrate -pip install debugpy -t /tmp -python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 manage.py runserver 0.0.0.0:8000 --nothreading diff --git a/compose/nginx/Dockerfile b/compose/nginx/Dockerfile new file mode 100644 index 000000000..6210b4e6c --- /dev/null +++ b/compose/nginx/Dockerfile @@ -0,0 +1,2 @@ +FROM nginxinc/nginx-unprivileged:stable +# ADD nginx.conf /etc/nginx/conf.d/default.conf diff --git a/compose/nginx/nginx.conf b/compose/nginx/nginx.conf new file mode 100644 index 000000000..66700e27d --- /dev/null +++ b/compose/nginx/nginx.conf @@ -0,0 +1,78 @@ +upstream app { + server django:5000; +} + +# Begin redirect for port 80 +server { + listen 80 default_server; + listen [::]:80 default_server; + return 301 https://$host$request_uri; +} +# End redirect for port 80 + +# Begin SSL site setup +server { + client_max_body_size 100M; + listen 443 ssl http2 default_server; + server_name ghostwriter.local; + charset utf-8; + + root /var/www/html; + + # ssl on; + ssl_certificate /ssl/ghostwriter.crt; + ssl_certificate_key /ssl/ghostwriter.key; + #ssl_stapling on; + #ssl_stapling_verify on; + + # SSL from stock default's ssl section + ssl_session_timeout 60m; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH'; + ssl_dhparam /ssl/dhparam.pem; + ssl_prefer_server_ciphers on; + resolver 8.8.8.8; + + location /media { + alias /app/media; + } + + location /admin { + try_files $uri @proxy_to_app; + } + + location /static { + alias /app/staticfiles; + } + + location / { + try_files $uri @proxy_to_app; + } + + location @proxy_to_app { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Protocol ssl; + proxy_connect_timeout 60; + proxy_read_timeout 60; + proxy_pass http://app; + } + + location /ws/ { + proxy_pass http://app; + proxy_http_version 1.1; + + proxy_redirect off; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Protocol ssl; + proxy_set_header X-Forwarded-Host $server_name; + } + +} +# End setup for SSL site diff --git a/compose/postgres/Dockerfile b/compose/postgres/Dockerfile new file mode 100644 index 000000000..f80eda119 --- /dev/null +++ b/compose/postgres/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:11.3 + +COPY --chmod=0755 ./compose/postgres/maintenance/ /usr/local/bin/ diff --git a/compose/production/postgres/maintenance/_sourced/constants.sh b/compose/postgres/maintenance/_sourced/constants.sh similarity index 100% rename from compose/production/postgres/maintenance/_sourced/constants.sh rename to compose/postgres/maintenance/_sourced/constants.sh diff --git a/compose/production/postgres/maintenance/_sourced/countdown.sh b/compose/postgres/maintenance/_sourced/countdown.sh similarity index 100% rename from compose/production/postgres/maintenance/_sourced/countdown.sh rename to compose/postgres/maintenance/_sourced/countdown.sh diff --git a/compose/production/postgres/maintenance/_sourced/messages.sh b/compose/postgres/maintenance/_sourced/messages.sh similarity index 100% rename from compose/production/postgres/maintenance/_sourced/messages.sh rename to compose/postgres/maintenance/_sourced/messages.sh diff --git a/compose/production/postgres/maintenance/_sourced/yes_no.sh b/compose/postgres/maintenance/_sourced/yes_no.sh similarity index 100% rename from compose/production/postgres/maintenance/_sourced/yes_no.sh rename to compose/postgres/maintenance/_sourced/yes_no.sh diff --git a/compose/production/postgres/maintenance/backup b/compose/postgres/maintenance/backup similarity index 100% rename from compose/production/postgres/maintenance/backup rename to compose/postgres/maintenance/backup diff --git a/compose/production/postgres/maintenance/backups b/compose/postgres/maintenance/backups similarity index 100% rename from compose/production/postgres/maintenance/backups rename to compose/postgres/maintenance/backups diff --git a/compose/production/postgres/maintenance/restore b/compose/postgres/maintenance/restore similarity index 100% rename from compose/production/postgres/maintenance/restore rename to compose/postgres/maintenance/restore diff --git a/compose/production/django/Dockerfile b/compose/production/django/Dockerfile deleted file mode 100644 index c679aa335..000000000 --- a/compose/production/django/Dockerfile +++ /dev/null @@ -1,63 +0,0 @@ -FROM python:3.8-alpine3.14 - -ENV PYTHONUNBUFFERED 1 - -ENV PYTHONPATH="$PYTHONPATH:/app/config" - -RUN apk --no-cache add build-base \ - # psycopg2 dependencies - && apk --no-cache add --virtual build-deps gcc python3-dev musl-dev \ - && apk --no-cache add postgresql-dev \ - # Pillow dependencies - && apk --no-cache add jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \ - # CFFI dependencies - && apk --no-cache add libffi-dev py-cffi \ - # XLSX dependencies - && apk --no-cache add libxml2-dev libxslt-dev \ - # Rust and Cargo required by the ``cryptography`` Python package - && apk --no-cache add rust \ - && apk --no-cache add cargo \ - && addgroup -S django \ - && adduser -S -G django django \ - && pip install --no-cache-dir -U setuptools pip - -COPY ./requirements /requirements - -RUN pip install --no-cache-dir -r /requirements/production.txt \ - && rm -rf /requirements - -COPY ./compose/production/django/entrypoint /entrypoint - -RUN sed -i 's/\r$//g' /entrypoint \ - && chmod +x /entrypoint \ - && chown django /entrypoint - -COPY ./compose/production/django/start /start - -RUN sed -i 's/\r$//g' /start \ - && chmod +x /start \ - && chown django /start - -COPY . /app - -COPY ./compose/production/django/queue/start /start-queue - -RUN sed -i 's/\r//' /start-queue \ - && chmod +x /start-queue \ - && chown django /start-queue - -COPY ./compose/production/django/seed_data /seed_data - -RUN sed -i 's/\r$//g' /seed_data \ - && chmod +x /seed_data \ - && mkdir -p /app/staticfiles \ - && mkdir -p /app/ghostwriter/media \ - && chown -R django /app - -USER django - -WORKDIR /app - -VOLUME ["/app/ghostwriter/media", "/app/staticfiles"] - -ENTRYPOINT ["/entrypoint"] diff --git a/compose/production/django/queue/start b/compose/production/django/queue/start deleted file mode 100644 index ad167e05f..000000000 --- a/compose/production/django/queue/start +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -set -o errexit -set -o pipefail -set -o nounset - -python manage.py qcluster diff --git a/compose/production/django/seed_data b/compose/production/django/seed_data deleted file mode 100644 index 613089617..000000000 --- a/compose/production/django/seed_data +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/ash - -JSON_FILES=$(find /app/ghostwriter -mindepth 2 -type d -name fixtures -exec find {} -type f -iname "*.json" \;) - -for json in $JSON_FILES -do - echo "Loading $json..." - python manage.py loaddata $json -done diff --git a/compose/production/django/start b/compose/production/django/start deleted file mode 100644 index 442ef5346..000000000 --- a/compose/production/django/start +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -set -o errexit -set -o pipefail -set -o nounset - -AVATAR_DIR=/app/ghostwriter/media/user_avatars -EVIDENCE_DIR=/app/ghostwriter/media/evidence -TEMPLATE_DIR=/app/ghostwriter/media/templates -ARCHIVE_DIR=/app/ghostwriter/media/archives - -TEMPLATE_PATH_DOCX=/app/ghostwriter/reporting/templates/reports/template.docx -TEMPLATE_PATH_PPTX=/app/ghostwriter/reporting/templates/reports/template.pptx - -[[ ! -d "$EVIDENCE_DIR" ]] && mkdir -p "$EVIDENCE_DIR" && chown -R django "$EVIDENCE_DIR" -[[ ! -d "$AVATAR_DIR" ]] && mkdir -p "$AVATAR_DIR" && chown -R django "$AVATAR_DIR" -[[ ! -d "$TEMPLATE_DIR" ]] && mkdir -p "$TEMPLATE_DIR" && chown -R django "$TEMPLATE_DIR" -[[ ! -d "$ARCHIVE_DIR" ]] && mkdir -p "$ARCHIVE_DIR" && chown -R django "$ARCHIVE_DIR" - -cp -u -p "$TEMPLATE_PATH_DOCX" "$TEMPLATE_DIR" -cp -u -p "$TEMPLATE_PATH_PPTX" "$TEMPLATE_DIR" - -python /app/manage.py collectstatic --noinput -python /app/manage.py migrate -# /usr/local/bin/daphne -b 0.0.0.0 -p 5000 config.asgi:application -uvicorn config.asgi:application --host 0.0.0.0 --port 5000 diff --git a/compose/production/hasura/Dockerfile b/compose/production/hasura/Dockerfile deleted file mode 100644 index 0163e7d82..000000000 --- a/compose/production/hasura/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM hasura/graphql-engine:v2.7.0.cli-migrations-v3 diff --git a/compose/production/nginx/Dockerfile b/compose/production/nginx/Dockerfile deleted file mode 100644 index ba9678fcf..000000000 --- a/compose/production/nginx/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM nginx:1.21.1-alpine -# ADD nginx.conf /etc/nginx/nginx.conf diff --git a/compose/production/nginx/nginx.conf b/compose/production/nginx/nginx.conf deleted file mode 100644 index 43d1335c6..000000000 --- a/compose/production/nginx/nginx.conf +++ /dev/null @@ -1,175 +0,0 @@ -user nginx; -worker_processes 1; - -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - #tcp_nopush on; - - keepalive_timeout 65; - - #gzip on; - - upstream app { - server django:5000; - } - - upstream graphql { - server graphql_engine:8080; - } - - # Begin redirect for port 80 - server { - listen 80 default_server; - listen [::]:80 default_server; - return 301 https://$host$request_uri; - } - # End redirect for port 80 - - # Begin SSL site setup - # https://ssl-config.mozilla.org/#server=nginx&version=1.21.1&config=intermediate&openssl=1.1.1l&ocsp=false&guideline=5.6 - server { - client_max_body_size 100M; - listen 443 ssl http2 default_server; - listen [::]:443 ssl http2; - server_name ghostwriter.local; - charset utf-8; - - root /var/www/html; - - resolver 8.8.8.8; - - proxy_headers_hash_max_size 1024; - proxy_headers_hash_bucket_size 128; - - # Enable SSL - ssl_certificate /ssl/ghostwriter.crt; - ssl_certificate_key /ssl/ghostwriter.key; - ssl_session_timeout 1d; - ssl_session_cache shared:MozSSL:10m; - ssl_session_tickets off; - ssl_dhparam /ssl/dhparam.pem; - - # Intermediate configuration - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; - ssl_prefer_server_ciphers off; - - # HSTS (ngx_http_headers_module is required) (63072000 seconds) - add_header Strict-Transport-Security "max-age=63072000" always; - - # Media locations for Docker - location /media { - alias /app/media; - } - - location /admin { - try_files $uri @proxy_to_app; - } - - location /static { - alias /app/staticfiles; - } - - location / { - try_files $uri @proxy_to_app; - } - - # Proxy connections to the Django server - location @proxy_to_app { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Protocol ssl; - proxy_connect_timeout 60; - proxy_read_timeout 60; - proxy_pass http://app; - } - - location /ws/ { - proxy_pass http://app; - proxy_http_version 1.1; - proxy_redirect off; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-Protocol ssl; - proxy_set_header X-Forwarded-Host $server_name; - } - - # Proxy connections to the Hasura GraphQL server - location /v1/ { - proxy_pass http://graphql/v1/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header X-Real-IP $remote_addr; - proxy_redirect off; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-Port $server_port; - } - - location /v2/ { - proxy_pass http://graphql/v2/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header X-Real-IP $remote_addr; - proxy_redirect off; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-Port $server_port; - } - - location /graphql/ { - proxy_pass http://graphql/v1/graphql; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header X-Real-IP $remote_addr; - proxy_redirect off; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-Port $server_port; - } - - location /console/ { - proxy_pass http://graphql/console/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header X-Real-IP $remote_addr; - proxy_redirect off; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-Port $server_port; - } - } -} diff --git a/compose/production/postgres/Dockerfile b/compose/production/postgres/Dockerfile deleted file mode 100644 index 04767267e..000000000 --- a/compose/production/postgres/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM postgres:11.12 - -COPY ./compose/production/postgres/maintenance /usr/local/bin/maintenance - -RUN chmod +x /usr/local/bin/maintenance/* \ - && mv /usr/local/bin/maintenance/* /usr/local/bin \ - && rmdir /usr/local/bin/maintenance diff --git a/compose/production/redis/Dockerfile b/compose/production/redis/Dockerfile deleted file mode 100644 index 42763d4eb..000000000 --- a/compose/production/redis/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM redis:6-alpine diff --git a/config/settings/base.py b/config/settings/base.py index f724168d4..69048dc18 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -298,7 +298,7 @@ # https://django-allauth.readthedocs.io/en/latest/configuration.html ACCOUNT_EMAIL_REQUIRED = True # https://django-allauth.readthedocs.io/en/latest/configuration.html -ACCOUNT_EMAIL_VERIFICATION = env.bool("DJANGO_ACCOUNT_EMAIL_VERIFICATION", "mandatory") +ACCOUNT_EMAIL_VERIFICATION = env("DJANGO_ACCOUNT_EMAIL_VERIFICATION", default="mandatory") # https://django-allauth.readthedocs.io/en/latest/configuration.html ACCOUNT_ADAPTER = "ghostwriter.users.adapters.AccountAdapter" # https://django-allauth.readthedocs.io/en/latest/configuration.html diff --git a/local.yml b/local.yml index e950a2fec..9642877ba 100644 --- a/local.yml +++ b/local.yml @@ -8,7 +8,9 @@ services: django: &django build: context: . - dockerfile: ./compose/local/django/Dockerfile + dockerfile: ./compose/django/Dockerfile + args: + STAGE: local image: ghostwriter_local_django depends_on: - postgres @@ -17,37 +19,37 @@ services: labels: name: ghostwriter_django environment: - - USE_DOCKER=${USE_DOCKER} - - IPYTHONDIR=${IPYTHONDIR} - - DJANGO_ACCOUNT_ALLOW_REGISTRATION=${DJANGO_ACCOUNT_ALLOW_REGISTRATION} - - DJANGO_ACCOUNT_EMAIL_VERIFICATION=${DJANGO_ACCOUNT_EMAIL_VERIFICATION} - - DJANGO_ADMIN_URL=${DJANGO_ADMIN_URL} - - DJANGO_ALLOWED_HOSTS=${DJANGO_ALLOWED_HOSTS} - - DJANGO_DATE_FORMAT=${DJANGO_DATE_FORMAT} - - DJANGO_JWT_SECRET_KEY=${DJANGO_JWT_SECRET_KEY} - - DJANGO_QCLUSTER_NAME=${DJANGO_QCLUSTER_NAME} + - USE_DOCKER=${USE_DOCKER:-yes} + - IPYTHONDIR=${IPYTHONDIR:-/app/.ipython} + - DJANGO_ACCOUNT_ALLOW_REGISTRATION=${DJANGO_ACCOUNT_ALLOW_REGISTRATION:-false} + - DJANGO_ACCOUNT_EMAIL_VERIFICATION=${DJANGO_ACCOUNT_EMAIL_VERIFICATION:-none} + - DJANGO_ADMIN_URL=${DJANGO_ADMIN_URL:-admin/} + - DJANGO_ALLOWED_HOSTS=${DJANGO_ALLOWED_HOSTS:-} + - DJANGO_DATE_FORMAT=${DJANGO_DATE_FORMAT:-d M Y} + - DJANGO_JWT_SECRET_KEY=${DJANGO_JWT_SECRET_KEY:-} + - DJANGO_QCLUSTER_NAME=${DJANGO_QCLUSTER_NAME:-} - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY} - - DJANGO_SECURE_SSL_REDIRECT=${DJANGO_SECURE_SSL_REDIRECT} - - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} - - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL} + - DJANGO_SECURE_SSL_REDIRECT=${DJANGO_SECURE_SSL_REDIRECT:-false} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-config.settings.local} + - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL:-admin@ghostwriter.local} - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD} - - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME} + - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME:-admin} - HASURA_ACTION_SECRET=${HASURA_GRAPHQL_ACTION_SECRET} - - POSTGRES_DB=${POSTGRES_DB} - - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_DB=${POSTGRES_DB:-ghostwriter} + - POSTGRES_HOST=${POSTGRES_HOST:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - POSTGRES_PORT=${POSTGRES_PORT} - - POSTGRES_USER=${POSTGRES_USER} - - REDIS_URL=redis://${REDIS_HOST}:${REDIS_PORT}/0 - - WEB_CONCURRENCY=${DJANGO_WEB_CONCURRENCY} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/0 + - WEB_CONCURRENCY=${DJANGO_WEB_CONCURRENCY:-4} ports: - "8000:8000" - command: /start + command: /start dev postgres: build: context: . - dockerfile: ./compose/production/postgres/Dockerfile + dockerfile: ./compose/postgres/Dockerfile image: ghostwriter_production_postgres volumes: - local_postgres_data:/var/lib/postgresql/data @@ -55,19 +57,16 @@ services: labels: name: ghostwriter_postgres environment: - - POSTGRES_DB=${POSTGRES_DB} - - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_DB=${POSTGRES_DB:-ghostwriter} + - POSTGRES_HOST=${POSTGRES_HOST:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - POSTGRES_PORT=${POSTGRES_PORT} - - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_USER=${POSTGRES_USER:-postgres} ports: - - "${POSTGRES_PORT}:5432" + - "${POSTGRES_PORT:-5432}:5432" redis: - build: - context: . - dockerfile: ./compose/production/redis/Dockerfile - image: ghostwriter_local_redis + image: redis:6-alpine labels: name: ghostwriter_redis @@ -80,19 +79,16 @@ services: labels: name: ghostwriter_queue ports: [] - command: /start-queue + command: /start queue graphql_engine: - build: - context: . - dockerfile: ./compose/production/hasura/Dockerfile - image: ghostwriter_local_graphql + image: hasura/graphql-engine:v2.7.0.cli-migrations-v3 depends_on: - postgres - django restart: always ports: - - "${HASURA_GRAPHQL_SERVER_PORT}:8080" + - "${HASURA_GRAPHQL_SERVER_PORT:-8080}:8080" - "9691:9691" volumes: - ./hasura-docker/metadata:/metadata @@ -104,17 +100,17 @@ services: labels: name: ghostwriter_graphql environment: - - ACTIONS_URL_BASE=http://${DJANGO_HOST}:${DJANGO_PORT}/api + - ACTIONS_URL_BASE=http://${DJANGO_HOST:-django}:${DJANGO_PORT:-8000}/api - HASURA_ACTION_SECRET=${HASURA_GRAPHQL_ACTION_SECRET} - HASURA_GRAPHQL_ADMIN_SECRET=${HASURA_GRAPHQL_ADMIN_SECRET} - - HASURA_GRAPHQL_AUTH_HOOK=http://${DJANGO_HOST}:${DJANGO_PORT}/api/webhook - - HASURA_GRAPHQL_DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - - HASURA_GRAPHQL_DEV_MODE=${HASURA_GRAPHQL_DEV_MODE} - - HASURA_GRAPHQL_ENABLE_CONSOLE=${HASURA_GRAPHQL_ENABLE_CONSOLE} - - HASURA_GRAPHQL_ENABLED_LOG_TYPES=${HASURA_GRAPHQL_ENABLED_LOG_TYPES} - - HASURA_GRAPHQL_ENABLE_TELEMETRY=${HASURA_GRAPHQL_ENABLE_TELEMETRY} - - HASURA_GRAPHQL_INSECURE_SKIP_TLS_VERIFY=${HASURA_GRAPHQL_INSECURE_SKIP_TLS_VERIFY} - - HASURA_GRAPHQL_LOG_LEVEL=${HASURA_GRAPHQL_LOG_LEVEL} - - HASURA_GRAPHQL_METADATA_DIR=${HASURA_GRAPHQL_METADATA_DIR} - - HASURA_GRAPHQL_MIGRATIONS_DIR=${HASURA_GRAPHQL_MIGRATIONS_DIR} - - HASURA_GRAPHQL_SERVER_PORT=${HASURA_GRAPHQL_SERVER_PORT} + - HASURA_GRAPHQL_AUTH_HOOK=http://${DJANGO_HOST:-django}:${DJANGO_PORT:-8000}/api/webhook + - HASURA_GRAPHQL_DATABASE_URL=postgres://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-ghostwriter} + - HASURA_GRAPHQL_DEV_MODE=${HASURA_GRAPHQL_DEV_MODE:-true} + - HASURA_GRAPHQL_ENABLE_CONSOLE=${HASURA_GRAPHQL_ENABLE_CONSOLE:-true} + - HASURA_GRAPHQL_ENABLED_LOG_TYPES=${HASURA_GRAPHQL_ENABLED_LOG_TYPES:-startup, http-log, webhook-log, websocket-log, query-log} + - HASURA_GRAPHQL_ENABLE_TELEMETRY=${HASURA_GRAPHQL_ENABLE_TELEMETRY:-false} + - HASURA_GRAPHQL_INSECURE_SKIP_TLS_VERIFY=${HASURA_GRAPHQL_INSECURE_SKIP_TLS_VERIFY:-true} + - HASURA_GRAPHQL_LOG_LEVEL=${HASURA_GRAPHQL_LOG_LEVEL:-warn} + - HASURA_GRAPHQL_METADATA_DIR=${HASURA_GRAPHQL_METADATA_DIR:-/metadata} + - HASURA_GRAPHQL_MIGRATIONS_DIR=${HASURA_GRAPHQL_MIGRATIONS_DIR:-/migrations} + - HASURA_GRAPHQL_SERVER_PORT=${HASURA_GRAPHQL_SERVER_PORT:-8080} diff --git a/production.yml b/production.yml index 96c853639..66bd4f920 100644 --- a/production.yml +++ b/production.yml @@ -1,4 +1,4 @@ -version: "3" +version: "3.2" volumes: production_postgres_data: {} @@ -8,10 +8,7 @@ volumes: services: django: &django - build: - context: . - dockerfile: ./compose/production/django/Dockerfile - image: ghostwriter_production_django + image: ghcr.io/quantum-sec/ghostwriter:feature-push-to-container-registry-v3.0.0 restart: unless-stopped depends_on: - postgres @@ -19,41 +16,47 @@ services: labels: name: ghostwriter_django environment: - - USE_DOCKER=${USE_DOCKER} - - IPYTHONDIR=${IPYTHONDIR} - - DJANGO_ACCOUNT_ALLOW_REGISTRATION=${DJANGO_ACCOUNT_ALLOW_REGISTRATION} - - DJANGO_ACCOUNT_EMAIL_VERIFICATION=${DJANGO_ACCOUNT_EMAIL_VERIFICATION} - - DJANGO_ADMIN_URL=${DJANGO_ADMIN_URL} - - DJANGO_ALLOWED_HOSTS=${DJANGO_ALLOWED_HOSTS} - - DJANGO_DATE_FORMAT=${DJANGO_DATE_FORMAT} + - USE_DOCKER=${USE_DOCKER:-yes} + - IPYTHONDIR=${IPYTHONDIR:-/app/.ipython} + - DJANGO_ACCOUNT_ALLOW_REGISTRATION=${DJANGO_ACCOUNT_ALLOW_REGISTRATION:-false} + - DJANGO_ACCOUNT_EMAIL_VERIFICATION=${DJANGO_ACCOUNT_EMAIL_VERIFICATION:-} + - DJANGO_ADMIN_URL=${DJANGO_ADMIN_URL:-admin/} + - DJANGO_ALLOWED_HOSTS=${DJANGO_ALLOWED_HOSTS:-} + - DJANGO_DATE_FORMAT=${DJANGO_DATE_FORMAT:-d M Y} - DJANGO_JWT_SECRET_KEY=${DJANGO_JWT_SECRET_KEY} - - DJANGO_QCLUSTER_NAME=${DJANGO_QCLUSTER_NAME} + - DJANGO_QCLUSTER_NAME=${DJANGO_QCLUSTER_NAME:-} - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY} - - DJANGO_SECURE_SSL_REDIRECT=${DJANGO_SECURE_SSL_REDIRECT} - - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} - - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL} + - DJANGO_SECURE_SSL_REDIRECT=${DJANGO_SECURE_SSL_REDIRECT:-} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-config.settings.production} + - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL:-admin@ghostwriter.local} - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD} - - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME} + - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME:-admin} - HASURA_ACTION_SECRET=${HASURA_GRAPHQL_ACTION_SECRET} - - MAILGUN_API_KEY=${DJANGO_MAILGUN_API_KEY} - - MAILGUN_DOMAIN=${DJANGO_MAILGUN_DOMAIN} - - POSTGRES_DB=${POSTGRES_DB} - - POSTGRES_HOST=${POSTGRES_HOST} + - MAILGUN_API_KEY=${DJANGO_MAILGUN_API_KEY:-} + - MAILGUN_DOMAIN=${DJANGO_MAILGUN_DOMAIN:-} + - POSTGRES_DB=${POSTGRES_DB:-ghostwriter} + - POSTGRES_HOST=${POSTGRES_HOST:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - POSTGRES_PORT=${POSTGRES_PORT} - - POSTGRES_USER=${POSTGRES_USER} - - REDIS_URL=redis://${REDIS_HOST}:${REDIS_PORT}/0 - - WEB_CONCURRENCY=${DJANGO_WEB_CONCURRENCY} - command: /start + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/0 + - WEB_CONCURRENCY=${DJANGO_WEB_CONCURRENCY:-4} volumes: - - production_staticfiles:/app/staticfiles - - production_data:/app/ghostwriter/media + - # production_staticfiles is where collectstatic will collect static files to + # https://docs.djangoproject.com/en/dev/ref/settings/#static-root + type: volume + source: production_staticfiles + target: /app/staticfiles + read_only: false + - # media is where ghostwriter will write to and serve reports as well as other local media from + type: volume + source: production_data + target: /app/ghostwriter/media + read_only: false + command: /start live postgres: - build: - context: . - dockerfile: ./compose/production/postgres/Dockerfile - image: ghostwriter_production_postgres + image: ghcr.io/quantum-sec/ghostwriter:postgres-feature-push-to-container-registry-v3.0.0 restart: unless-stopped volumes: - production_postgres_data:/var/lib/postgresql/data @@ -61,43 +64,57 @@ services: labels: name: ghostwriter_postgres environment: - - POSTGRES_DB=${POSTGRES_DB} - - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_DB=${POSTGRES_DB:-ghostwriter} + - POSTGRES_HOST=${POSTGRES_HOST:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - POSTGRES_PORT=${POSTGRES_PORT} - - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_USER=${POSTGRES_USER:-postgres} ports: - - "${POSTGRES_PORT}:5432" + - "${POSTGRES_PORT:-5432}:5432" nginx: - build: ./compose/production/nginx - image: ghostwriter_production_nginx + image: nginxinc/nginx-unprivileged:1.21 restart: unless-stopped depends_on: - django - volumes: - - production_data:/app/media - - production_staticfiles:/app/staticfiles - - ./compose/production/nginx/nginx.conf:/etc/nginx/nginx.conf - - ./ssl:/ssl labels: name: ghostwriter_nginx + environment: + NGINX_ENTRYPOINT_QUIET_LOGS: 1 + volumes: + - type: volume + source: production_data + target: /app/media + read_only: true + volume: + nocopy: true + - type: volume + source: production_staticfiles + target: /app/staticfiles + read_only: true + volume: + nocopy: true + - type: bind + source: ./compose/nginx/nginx.conf + target: /etc/nginx/conf.d/default.conf + # target: /etc/nginx/nginx.conf + read_only: true + - type: bind + source: ./ssl + target: /ssl + read_only: true ports: - - "0.0.0.0:80:80" - - "0.0.0.0:443:443" + - '0.0.0.0:80:80' + - '0.0.0.0:443:443' redis: - build: - context: . - dockerfile: ./compose/production/redis/Dockerfile - image: ghostwriter_production_redis + image: redis:6-alpine labels: name: ghostwriter_redis restart: unless-stopped queue: <<: *django - image: ghostwriter_production_queue restart: unless-stopped depends_on: - redis @@ -106,19 +123,16 @@ services: ports: [] labels: name: ghostwriter_queue - command: /start-queue + command: /start queue graphql_engine: - build: - context: . - dockerfile: ./compose/production/hasura/Dockerfile - image: ghostwriter_production_graphql + image: hasura/graphql-engine:v2.7.0.cli-migrations-v3 depends_on: - postgres - django restart: always ports: - - "${HASURA_GRAPHQL_SERVER_PORT}:8080" + - "${HASURA_GRAPHQL_SERVER_PORT:-8080}:8080" - "9691:9691" volumes: - ./hasura-docker/metadata:/metadata @@ -130,17 +144,17 @@ services: labels: name: ghostwriter_graphql environment: - - ACTIONS_URL_BASE=http://${NGINX_HOST}:${NGINX_PORT}/api + - ACTIONS_URL_BASE=http://${NGINX_HOST}:${NGINX_PORT:-443}/api - HASURA_ACTION_SECRET=${HASURA_GRAPHQL_ACTION_SECRET} - HASURA_GRAPHQL_ADMIN_SECRET=${HASURA_GRAPHQL_ADMIN_SECRET} - - HASURA_GRAPHQL_AUTH_HOOK=http://${NGINX_HOST}:${NGINX_PORT}/api/webhook - - HASURA_GRAPHQL_DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - - HASURA_GRAPHQL_DEV_MODE=${HASURA_GRAPHQL_DEV_MODE} - - HASURA_GRAPHQL_ENABLE_CONSOLE=${HASURA_GRAPHQL_ENABLE_CONSOLE} - - HASURA_GRAPHQL_ENABLED_LOG_TYPES=${HASURA_GRAPHQL_ENABLED_LOG_TYPES} - - HASURA_GRAPHQL_ENABLE_TELEMETRY=${HASURA_GRAPHQL_ENABLE_TELEMETRY} - - HASURA_GRAPHQL_INSECURE_SKIP_TLS_VERIFY=${HASURA_GRAPHQL_INSECURE_SKIP_TLS_VERIFY} - - HASURA_GRAPHQL_LOG_LEVEL=${HASURA_GRAPHQL_LOG_LEVEL} - - HASURA_GRAPHQL_METADATA_DIR=${HASURA_GRAPHQL_METADATA_DIR} - - HASURA_GRAPHQL_MIGRATIONS_DIR=${HASURA_GRAPHQL_MIGRATIONS_DIR} - - HASURA_GRAPHQL_SERVER_PORT=${HASURA_GRAPHQL_SERVER_PORT} + - HASURA_GRAPHQL_AUTH_HOOK=http://${NGINX_HOST}:${NGINX_PORT:-443}/api/webhook + - HASURA_GRAPHQL_DATABASE_URL=postgres://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-ghostwriter} + - HASURA_GRAPHQL_DEV_MODE=${HASURA_GRAPHQL_DEV_MODE:-false} + - HASURA_GRAPHQL_ENABLE_CONSOLE=${HASURA_GRAPHQL_ENABLE_CONSOLE:-true} + - HASURA_GRAPHQL_ENABLED_LOG_TYPES=${HASURA_GRAPHQL_ENABLED_LOG_TYPES:-startup, http-log, webhook-log, websocket-log, query-log} + - HASURA_GRAPHQL_ENABLE_TELEMETRY=${HASURA_GRAPHQL_ENABLE_TELEMETRY:-false} + - HASURA_GRAPHQL_INSECURE_SKIP_TLS_VERIFY=${HASURA_GRAPHQL_INSECURE_SKIP_TLS_VERIFY:-false} + - HASURA_GRAPHQL_LOG_LEVEL=${HASURA_GRAPHQL_LOG_LEVEL:-warn} + - HASURA_GRAPHQL_METADATA_DIR=${HASURA_GRAPHQL_METADATA_DIR:-/metadata} + - HASURA_GRAPHQL_MIGRATIONS_DIR=${HASURA_GRAPHQL_MIGRATIONS_DIR:-/migrations} + - HASURA_GRAPHQL_SERVER_PORT=${HASURA_GRAPHQL_SERVER_PORT:-8080} diff --git a/ssl/readme.md b/ssl/readme.md index 55c1131e0..cc978c986 100644 --- a/ssl/readme.md +++ b/ssl/readme.md @@ -2,7 +2,7 @@ Before running in production, it is necessary to setup a SSL certificate. A self-signed certificate can be created using the following commands. Other options include purchasing a certificate or using [LetsEncrypt](https://letsencrypt.org/) for a free certificate. -Certificates should be placed in the `ssl/` folder. The files referenced in `compose/production/nginx/nginx.conf` use the following files names: +Certificates should be placed in the `ssl/` folder. The files referenced in `compose/nginx/nginx.conf` use the following files names: - ghostwriter.crt - ghostwriter.key