diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..a866da9 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @pomerium/dev-backend diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 758deee..0f1cca3 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -1,5 +1,15 @@ version: 2 updates: + - package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: "monthly" + groups: + gitsubmodule: + patterns: + - "*" + exclude-patterns: + - "deps/github.com/pomerium/*" - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..e3eba25 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,16 @@ +name: Build + +on: + pull_request: + branches-ignore: + - "dependabot/**" + push: + branches: + - "main" + +jobs: + build: + name: Build + uses: ./.github/workflows/reusable-build.yaml + with: + ref: ${{ github.head_ref }} diff --git a/.github/workflows/dependabot.yaml b/.github/workflows/dependabot.yaml new file mode 100644 index 0000000..747fb3c --- /dev/null +++ b/.github/workflows/dependabot.yaml @@ -0,0 +1,34 @@ +name: Dependabot + +on: + pull_request: + branches: + - "dependabot/**" + +jobs: + generate: + name: Generate + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + ref: ${{ github.head_ref }} + submodules: "true" + token: ${{ secrets.APPARITOR_GITHUB_TOKEN }} + + - name: Generate + run: make generate + + - name: Commit + uses: devops-infra/action-commit-push@e6a24fad602d1f92e46432c89a7e0c7fdd45d62d + with: + github_token: ${{ secrets.APPARITOR_GITHUB_TOKEN }} + commit_message: "dependabot: generate" + + build: + name: Build + needs: generate + uses: ./.github/workflows/reusable-build.yaml + with: + ref: ${{ github.head_ref }} diff --git a/.github/workflows/reusable-build.yaml b/.github/workflows/reusable-build.yaml new file mode 100644 index 0000000..af8257e --- /dev/null +++ b/.github/workflows/reusable-build.yaml @@ -0,0 +1,28 @@ +name: Build + +on: + workflow_call: + inputs: + ref: + required: true + type: string + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + ref: ${{ inputs.ref }} + + - name: Setup Python + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c + with: + python-version: ${{ matrix.python-version }} + + - name: Test + run: make test diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml deleted file mode 100644 index 9c830d1..0000000 --- a/.github/workflows/test.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: test - -on: - pull_request: - push: - branches: - - main -jobs: - tests: - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c - with: - python-version: ${{ matrix.python-version }} - - - run: pip install . - - - run: make test diff --git a/.github/workflows/update-pomerium.yaml b/.github/workflows/update-pomerium.yaml new file mode 100644 index 0000000..f2961ea --- /dev/null +++ b/.github/workflows/update-pomerium.yaml @@ -0,0 +1,42 @@ +name: Update Pomerium + +on: + schedule: + - cron: "40 1 * * *" + workflow_dispatch: + +jobs: + update: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + submodules: "true" + token: ${{ secrets.APPARITOR_GITHUB_TOKEN }} + + - name: Update Pomerium + run: make update-pomerium + + - name: Generate + run: make generate + + - name: Check for changes + id: git-diff + run: | + git config --global user.email "apparitor@users.noreply.github.com" + git config --global user.name "GitHub Actions" + git add deps/github.com/pomerium + git diff --cached --exit-code || echo "changed=true" >> $GITHUB_OUTPUT + + - name: Create Pull Request + if: ${{ steps.git-diff.outputs.changed }} == 'true' + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e + with: + author: GitHub Actions + body: "This PR updates Pomerium Dependencies" + commit-message: "ci: update pomerium dependencies" + delete-branch: true + labels: ci + title: "ci: update pomerium dependencies" + token: ${{ secrets.APPARITOR_GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 652725f..8321e3f 100644 --- a/.gitignore +++ b/.gitignore @@ -146,5 +146,3 @@ cython_debug/ # End of https://www.toptal.com/developers/gitignore/api/python # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) - -deps/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e93d210 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,12 @@ +[submodule "deps/github.com/pomerium/enterprise-client"] + path = deps/github.com/pomerium/enterprise-client + url = git@github.com:pomerium/enterprise-client +[submodule "deps/github.com/envoyproxy/protoc-gen-validate"] + path = deps/github.com/envoyproxy/protoc-gen-validate + url = git@github.com:envoyproxy/protoc-gen-validate +[submodule "deps/github.com/census-instrumentation/opencensus-proto"] + path = deps/github.com/census-instrumentation/opencensus-proto + url = git@github.com:census-instrumentation/opencensus-proto +[submodule "deps/github.com/open-telemetry/opentelemetry-proto"] + path = deps/github.com/open-telemetry/opentelemetry-proto + url = git@github.com:open-telemetry/opentelemetry-proto diff --git a/Makefile b/Makefile index 50c4d09..cd80524 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,29 @@ PYTHON:=python3 .PHONY: all -all: test pkg +all: generate test pkg .PHONY: pkg -pkg: +pkg: install @echo "==> $@" + $(PYTHON) -m pip install build $(PYTHON) -m build . -.PHONY: update -update: +.PHONY: generate +generate: install @echo "==> $@" - @scripts/update + @scripts/generate + +.PHONY: install +install: + $(PYTHON) -m pip install . .PHONY: test -test: +test: install @echo "==> $@" PYTHONPATH=src/ $(PYTHON) -m unittest discover -s src -v + +.PHONY: update-pomerium +update-pomerium: + @echo "==> $@" + git submodule update --remote deps/github.com/pomerium diff --git a/deps/github.com/census-instrumentation/opencensus-proto b/deps/github.com/census-instrumentation/opencensus-proto new file mode 160000 index 0000000..e53624a --- /dev/null +++ b/deps/github.com/census-instrumentation/opencensus-proto @@ -0,0 +1 @@ +Subproject commit e53624a87b9b9b919147a9b4626c669a869ebb34 diff --git a/deps/github.com/envoyproxy/protoc-gen-validate b/deps/github.com/envoyproxy/protoc-gen-validate new file mode 160000 index 0000000..7b06248 --- /dev/null +++ b/deps/github.com/envoyproxy/protoc-gen-validate @@ -0,0 +1 @@ +Subproject commit 7b06248484ceeaa947e93ca2747eccf336a88ecc diff --git a/deps/github.com/open-telemetry/opentelemetry-proto b/deps/github.com/open-telemetry/opentelemetry-proto new file mode 160000 index 0000000..c0a98a1 --- /dev/null +++ b/deps/github.com/open-telemetry/opentelemetry-proto @@ -0,0 +1 @@ +Subproject commit c0a98a1847d3124ac5f9ecd02d0e2d2732bbb590 diff --git a/deps/github.com/pomerium/enterprise-client b/deps/github.com/pomerium/enterprise-client new file mode 160000 index 0000000..82b4168 --- /dev/null +++ b/deps/github.com/pomerium/enterprise-client @@ -0,0 +1 @@ +Subproject commit 82b4168693e371f30407fe0bf989c1f30d0e01cb diff --git a/scripts/generate b/scripts/generate new file mode 100755 index 0000000..bc0d0a5 --- /dev/null +++ b/scripts/generate @@ -0,0 +1,96 @@ +#!/bin/bash +set -euo pipefail + +_scripts_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +_root_dir="$(dirname "$_scripts_dir")" +_deps_dir="$_root_dir/deps" + +function replace-in-file() { + if [ "$(uname)" = 'Darwin' ]; then + # for MacOS + sed -i '' -E "$1" "$2" + else + # for Linux and Windows + sed -i'' -E "$1" "$2" + fi +} + +function download() { + local _url="$1" + local _dst="$2" + + echo "downloading $_url to $_dst" + curl \ + --silent \ + --compressed \ + --fail \ + --location \ + --time-cond "$_dst" \ + --output "$_dst" \ + "$_url" +} + +function join_by() { + local IFS="$1" + shift + echo "$*" +} + +_proto_files=( + activity_log.proto + clusters.proto + devices.proto + external_data_sources.proto + key_chain.proto + namespaces.proto + policy.proto + report.proto + route_health_check.proto + route_redirect_action.proto + routes.proto + settings.proto + types.proto + users.proto +) + +echo "installing python dependencies" +pip install \ + 'wheel==0.45.1' \ + 'protobuf==5.29.4' \ + 'grpcio==1.71.0' \ + 'grpcio-tools==1.71.0' \ + 'mypy-protobuf==3.6.0' + +( + cd "$_root_dir" + + echo "generating dependency protobuf code" + python -m grpc_tools.protoc \ + -I "$_deps_dir/github.com/envoyproxy/protoc-gen-validate" \ + -I "$_deps_dir/github.com/census-instrumentation/opencensus-proto/src" \ + -I "$_deps_dir/github.com/open-telemetry/opentelemetry-proto" \ + -I "$_deps_dir/github.com/prometheus/client_model" \ + --python_out="src" \ + $(cd deps/github.com/envoyproxy/protoc-gen-validate && find . -regex '.*\.proto' | sed 's|^./||') + + echo "generating enterprise-client protobuf code" + mkdir -p src/pomerium/pb + python -m grpc_tools.protoc \ + -I "$_deps_dir" \ + -I "$_deps_dir/github.com/envoyproxy/protoc-gen-validate" \ + -I "$_deps_dir/github.com/pomerium/enterprise-client/protos/pomerium-console" \ + --mypy_out="src/pomerium/pb" \ + --mypy_grpc_out="src/pomerium/pb" \ + --python_out="src/pomerium/pb" \ + --grpc_python_out="src/pomerium/pb" \ + "${_proto_files[@]}" + + # fix package names for grpc + for _file in src/pomerium/pb/*.py; do + replace-in-file 's/^import ([a-z_]+)_pb2/import pomerium.pb.\1_pb2/' "$_file" + done + + # add __init__ files + find src -type d -exec touch __init__.py \; + rm __init__.py +) diff --git a/scripts/update b/scripts/update deleted file mode 100755 index 5dbfae3..0000000 --- a/scripts/update +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash -set -euo pipefail - -_scripts_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -_root_dir="$(dirname "$_scripts_dir")" -_deps_dir="$_root_dir/deps" - -readonly _git_deps=( - "pomerium/enterprise-client|4acce8472bc4b6ff031203adf0a4f2f76cb6542d" - 'envoyproxy/protoc-gen-validate|v1.0.2' - 'census-instrumentation/opencensus-proto|v0.4.1' - 'open-telemetry/opentelemetry-proto|v1.0.0' -) - -function replace-in-file() { - if [ "$(uname)" = 'Darwin' ]; then - # for MacOS - sed -i '' -E "$1" "$2" - else - # for Linux and Windows - sed -i'' -E "$1" "$2" - fi -} - -function clone() { - local _name="$1" - local _tag="$2" - local _dst="$3" - - echo "cloning $_name:$_tag to $_dst" - if [ ! -d "$_dst" ]; then - mkdir -p "$_dst" - git clone "git@github.com:$_name" "$_dst" - fi - ( - cd "$_dst" - git fetch - git checkout "$_tag" - ) -} - -function clone_all() { - local _name _tag _dst - for _dep in "${_git_deps[@]}"; do - IFS=$'|' read -r _name _tag <<<"$_dep" - _dst="$_deps_dir/github.com/$_name" - clone "$_name" "$_tag" "$_dst" - done -} - -function download() { - local _url="$1" - local _dst="$2" - - echo "downloading $_url to $_dst" - curl \ - --silent \ - --compressed \ - --fail \ - --location \ - --time-cond "$_dst" \ - --output "$_dst" \ - "$_url" -} - -function join_by() { - local IFS="$1" - shift - echo "$*" -} - -clone_all - -_proto_files=( - activity_log.proto - clusters.proto - devices.proto - external_data_sources.proto - key_chain.proto - namespaces.proto - policy.proto - report.proto - route_health_check.proto - route_redirect_action.proto - routes.proto - settings.proto - types.proto - users.proto -) - -echo "installing python dependencies" -pip install \ - 'wheel==0.45.1' \ - 'protobuf==5.29.4' \ - 'grpcio==1.71.0' \ - 'grpcio-tools==1.71.0' \ - 'mypy-protobuf==3.6.0' - -( - cd "$_root_dir" - - echo "generating dependency protobuf code" - python -m grpc_tools.protoc \ - -I "$_deps_dir/github.com/envoyproxy/protoc-gen-validate" \ - -I "$_deps_dir/github.com/census-instrumentation/opencensus-proto/src" \ - -I "$_deps_dir/github.com/open-telemetry/opentelemetry-proto" \ - -I "$_deps_dir/github.com/prometheus/client_model" \ - --python_out="src" \ - $(cd deps/github.com/envoyproxy/protoc-gen-validate && find . -regex '.*\.proto' | sed 's|^./||') - - echo "generating enterprise-client protobuf code" - mkdir -p src/pomerium/pb - python -m grpc_tools.protoc \ - -I "$_deps_dir" \ - -I "$_deps_dir/github.com/envoyproxy/protoc-gen-validate" \ - -I "$_deps_dir/github.com/pomerium/enterprise-client/protos/pomerium-console" \ - --mypy_out="src/pomerium/pb" \ - --mypy_grpc_out="src/pomerium/pb" \ - --python_out="src/pomerium/pb" \ - --grpc_python_out="src/pomerium/pb" \ - "${_proto_files[@]}" - - # fix package names for grpc - for _file in src/pomerium/pb/*.py; do - replace-in-file 's/^import ([a-z_]+)_pb2/import pomerium.pb.\1_pb2/' "$_file" - done - - # add __init__ files - find src -type d -exec touch __init__.py \; - rm __init__.py -)