From 4bec54f0549cd6861d777c40ea856a7bc83d2a67 Mon Sep 17 00:00:00 2001 From: Chad Pritchett Date: Mon, 17 Nov 2025 03:08:28 -0600 Subject: [PATCH] Enhance project documentation and configuration files - Updated CONTRIBUTING.md to improve clarity and structure. - Cleaned up dependabot.yml by removing unnecessary whitespace. - Modified release-drafter.yml for consistent quotation style and formatting. - Adjusted workflows (black.yml, codeql.yml, deploy-PROD.yml, deploy-RELEASE.yml, docker-publish.yml) for consistent quotation marks and formatting. - Refined .pre-commit-config.yaml by removing redundant entries and ensuring proper formatting. - Improved launch.json formatting in .vscode for better readability. - Enhanced README.md with clearer formatting and additional spacing for improved readability. - Refactored app.py and various GitHub app modules (azuread.py, core.py, googleworkspace.py, keycloak.py, ldap.py, okta.py, onelogin.py) for consistent import ordering and formatting. --- .automation/upload-docker.sh | 750 +++++++++++++-------------- .github/CONTRIBUTING.md | 5 + .github/dependabot.yml | 1 - .github/release-drafter.yml | 50 +- .github/workflows/black.yml | 10 +- .github/workflows/codeql.yml | 6 +- .github/workflows/deploy-PROD.yml | 2 +- .github/workflows/deploy-RELEASE.yml | 2 +- .github/workflows/docker-publish.yml | 1 - .pre-commit-config.yaml | 29 +- .vscode/launch.json | 30 +- README.md | 85 +-- app.py | 24 +- githubapp/azuread.py | 333 ++++++------ githubapp/core.py | 8 +- githubapp/googleworkspace.py | 11 +- githubapp/keycloak.py | 33 +- githubapp/ldap.py | 11 +- githubapp/okta.py | 174 +++---- githubapp/onelogin.py | 3 +- 20 files changed, 801 insertions(+), 767 deletions(-) diff --git a/.automation/upload-docker.sh b/.automation/upload-docker.sh index 9cb15ac..91dfc3c 100755 --- a/.automation/upload-docker.sh +++ b/.automation/upload-docker.sh @@ -41,392 +41,392 @@ source "${GITHUB_WORKSPACE}/lib/log.sh" # Source the function script(s) ################################################################################ #### Function Header ########################################################### Header() { - info "-------------------------------------------------------" - info "---- GitHub Actions Upload image to [${REGISTRY}] ----" - info "-------------------------------------------------------" + info "-------------------------------------------------------" + info "---- GitHub Actions Upload image to [${REGISTRY}] ----" + info "-------------------------------------------------------" } ################################################################################ #### Function ValidateInput #################################################### ValidateInput() { - # Need to validate we have the basic variables - ################ - # Print header # - ################ - info "----------------------------------------------" - info "Gathering variables..." - info "----------------------------------------------" - - ############################# - # Validate GITHUB_WORKSPACE # - ############################# - if [ -z "${GITHUB_WORKSPACE}" ]; then - error "Failed to get [GITHUB_WORKSPACE]!" - fatal "[${GITHUB_WORKSPACE}]" - else - info "Successfully found:${F[W]}[GITHUB_WORKSPACE]${F[B]}, value:${F[W]}[${GITHUB_WORKSPACE}]" - fi - - ##################### - # Validate REGISTRY # - ##################### - if [ -z "${REGISTRY}" ]; then - error "Failed to get [REGISTRY]!" - fatal "[${REGISTRY}]" - else - info "Successfully found:${F[W]}[REGISTRY]${F[B]}, value:${F[W]}[${REGISTRY}]" - fi - - ##################################################### - # See if we need values for GitHub package Registry # - ##################################################### - if [[ ${REGISTRY} == "GPR" ]]; then - ######################### - # Validate GPR_USERNAME # - ######################### - if [ -z "${GPR_USERNAME}" ]; then - error "Failed to get [GPR_USERNAME]!" - fatal "[${GPR_USERNAME}]" - else - info "Successfully found:${F[W]}[GPR_USERNAME]${F[B]}, value:${F[W]}[${GPR_USERNAME}]" - fi - - ###################### - # Validate GPR_TOKEN # - ###################### - if [ -z "${GPR_TOKEN}" ]; then - error "Failed to get [GPR_TOKEN]!" - fatal "[${GPR_TOKEN}]" - else - info "Successfully found:${F[W]}[GPR_TOKEN]${F[B]}, value:${F[W]}[********]" - fi - ######################################## - # See if we need values for Ducker hub # - ######################################## - elif [[ ${REGISTRY} == "Docker" ]]; then - ############################ - # Validate DOCKER_USERNAME # - ############################ - if [ -z "${DOCKER_USERNAME}" ]; then - error "Failed to get [DOCKER_USERNAME]!" - fatal "[${DOCKER_USERNAME}]" - else - info "Successfully found:${F[W]}[DOCKER_USERNAME]${F[B]}, value:${F[W]}[${DOCKER_USERNAME}]" - fi - - ############################ - # Validate DOCKER_PASSWORD # - ############################ - if [ -z "${DOCKER_PASSWORD}" ]; then - error "Failed to get [DOCKER_PASSWORD]!" - fatal "[${DOCKER_PASSWORD}]" - else - info "Successfully found:${F[W]}[DOCKER_PASSWORD]${F[B]}, value:${F[B]}[********]" - fi - ########################################### - # We were not passed a registry to update # - ########################################### - else - error "Failed to find a valid registry!" - fatal "Registry:[${REGISTRY}]" - fi - - ####################### - # Validate IMAGE_REPO # - ####################### - if [ -z "${IMAGE_REPO}" ]; then - error "Failed to get [IMAGE_REPO]!" - fatal "[${IMAGE_REPO}]" - else - info "Successfully found:${F[W]}[IMAGE_REPO]${F[B]}, value:${F[W]}[${IMAGE_REPO}]" - ############################################### - # Need to see if GPR registry and update name # - ############################################### - if [[ ${REGISTRY} == "GPR" ]]; then - NAME="docker.pkg.github.com/${IMAGE_REPO}/super-linter" - IMAGE_REPO="${NAME}" - info "Updated [IMAGE_REPO] to:[${IMAGE_REPO}] for GPR" - fi - fi - - ########################## - # Validate IMAGE_VERSION # - ########################## - if [ -z "${IMAGE_VERSION}" ]; then - warn "Failed to get [IMAGE_VERSION]!" - info "Pulling from Branch Name..." - ############################## - # Get the name of the branch # - ############################## - BRANCH_NAME=$(git -C "${GITHUB_WORKSPACE}" branch --contains "${GITHUB_SHA}" | awk '{print ${2}}' 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ ${ERROR_CODE} -ne 0 ]; then - error "Failed to get branch name!" - fatal "[${BRANCH_NAME}]" - fi - - ################################## - # Remove non alpha-numeric chars # - ################################## - BRANCH_NAME=$(echo "${BRANCH_NAME}" | tr -cd '[:alnum:]') - - ############################################ - # Set the IMAGE_VERSION to the BRANCH_NAME # - ############################################ - IMAGE_VERSION="${BRANCH_NAME}" - info "Tag:[${IMAGE_VERSION}]" - else - info "Successfully found:${F[W]}[IMAGE_VERSION]${F[B]}, value:${F[W]}[${IMAGE_VERSION}]" - fi - - ################################## - # Set regex for getting tag info # - ################################## - REGEX='(v[0-9]+\.[0-9]+\.[0-9]+)' # Matches 'v1.2.3' - - ###################################################################### - # Check if this is a latest to a versioned release at create new tag # - ###################################################################### - if [[ ${IMAGE_VERSION} =~ ${REGEX} ]]; then - # Need to get the major version, and set flag to update - - ##################### - # Set the major tag # - ##################### - MAJOR_TAG=$(echo "${IMAGE_VERSION}" | cut -d '.' -f1) - - ################################### - # Set flag for updating major tag # - ################################### - UPDATE_MAJOR_TAG=1 - - info "- Also deploying a major tag of:[${MAJOR_TAG}]" - fi - - ############################ - # Validate DOCKERFILE_PATH # - ############################ - if [ -z "${DOCKERFILE_PATH}" ]; then - error "Failed to get [DOCKERFILE_PATH]!" - fatal "[${DOCKERFILE_PATH}]" - else - info "Successfully found:${F[W]}[DOCKERFILE_PATH]${F[B]}, value:${F[W]}[${DOCKERFILE_PATH}]" - fi + # Need to validate we have the basic variables + ################ + # Print header # + ################ + info "----------------------------------------------" + info "Gathering variables..." + info "----------------------------------------------" + + ############################# + # Validate GITHUB_WORKSPACE # + ############################# + if [ -z "${GITHUB_WORKSPACE}" ]; then + error "Failed to get [GITHUB_WORKSPACE]!" + fatal "[${GITHUB_WORKSPACE}]" + else + info "Successfully found:${F[W]}[GITHUB_WORKSPACE]${F[B]}, value:${F[W]}[${GITHUB_WORKSPACE}]" + fi + + ##################### + # Validate REGISTRY # + ##################### + if [ -z "${REGISTRY}" ]; then + error "Failed to get [REGISTRY]!" + fatal "[${REGISTRY}]" + else + info "Successfully found:${F[W]}[REGISTRY]${F[B]}, value:${F[W]}[${REGISTRY}]" + fi + + ##################################################### + # See if we need values for GitHub package Registry # + ##################################################### + if [[ ${REGISTRY} == "GPR" ]]; then + ######################### + # Validate GPR_USERNAME # + ######################### + if [ -z "${GPR_USERNAME}" ]; then + error "Failed to get [GPR_USERNAME]!" + fatal "[${GPR_USERNAME}]" + else + info "Successfully found:${F[W]}[GPR_USERNAME]${F[B]}, value:${F[W]}[${GPR_USERNAME}]" + fi + + ###################### + # Validate GPR_TOKEN # + ###################### + if [ -z "${GPR_TOKEN}" ]; then + error "Failed to get [GPR_TOKEN]!" + fatal "[${GPR_TOKEN}]" + else + info "Successfully found:${F[W]}[GPR_TOKEN]${F[B]}, value:${F[W]}[********]" + fi + ######################################## + # See if we need values for Ducker hub # + ######################################## + elif [[ ${REGISTRY} == "Docker" ]]; then + ############################ + # Validate DOCKER_USERNAME # + ############################ + if [ -z "${DOCKER_USERNAME}" ]; then + error "Failed to get [DOCKER_USERNAME]!" + fatal "[${DOCKER_USERNAME}]" + else + info "Successfully found:${F[W]}[DOCKER_USERNAME]${F[B]}, value:${F[W]}[${DOCKER_USERNAME}]" + fi + + ############################ + # Validate DOCKER_PASSWORD # + ############################ + if [ -z "${DOCKER_PASSWORD}" ]; then + error "Failed to get [DOCKER_PASSWORD]!" + fatal "[${DOCKER_PASSWORD}]" + else + info "Successfully found:${F[W]}[DOCKER_PASSWORD]${F[B]}, value:${F[B]}[********]" + fi + ########################################### + # We were not passed a registry to update # + ########################################### + else + error "Failed to find a valid registry!" + fatal "Registry:[${REGISTRY}]" + fi + + ####################### + # Validate IMAGE_REPO # + ####################### + if [ -z "${IMAGE_REPO}" ]; then + error "Failed to get [IMAGE_REPO]!" + fatal "[${IMAGE_REPO}]" + else + info "Successfully found:${F[W]}[IMAGE_REPO]${F[B]}, value:${F[W]}[${IMAGE_REPO}]" + ############################################### + # Need to see if GPR registry and update name # + ############################################### + if [[ ${REGISTRY} == "GPR" ]]; then + NAME="docker.pkg.github.com/${IMAGE_REPO}/super-linter" + IMAGE_REPO="${NAME}" + info "Updated [IMAGE_REPO] to:[${IMAGE_REPO}] for GPR" + fi + fi + + ########################## + # Validate IMAGE_VERSION # + ########################## + if [ -z "${IMAGE_VERSION}" ]; then + warn "Failed to get [IMAGE_VERSION]!" + info "Pulling from Branch Name..." + ############################## + # Get the name of the branch # + ############################## + BRANCH_NAME=$(git -C "${GITHUB_WORKSPACE}" branch --contains "${GITHUB_SHA}" | awk '{print ${2}}' 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ ${ERROR_CODE} -ne 0 ]; then + error "Failed to get branch name!" + fatal "[${BRANCH_NAME}]" + fi + + ################################## + # Remove non alpha-numeric chars # + ################################## + BRANCH_NAME=$(echo "${BRANCH_NAME}" | tr -cd '[:alnum:]') + + ############################################ + # Set the IMAGE_VERSION to the BRANCH_NAME # + ############################################ + IMAGE_VERSION="${BRANCH_NAME}" + info "Tag:[${IMAGE_VERSION}]" + else + info "Successfully found:${F[W]}[IMAGE_VERSION]${F[B]}, value:${F[W]}[${IMAGE_VERSION}]" + fi + + ################################## + # Set regex for getting tag info # + ################################## + REGEX='(v[0-9]+\.[0-9]+\.[0-9]+)' # Matches 'v1.2.3' + + ###################################################################### + # Check if this is a latest to a versioned release at create new tag # + ###################################################################### + if [[ ${IMAGE_VERSION} =~ ${REGEX} ]]; then + # Need to get the major version, and set flag to update + + ##################### + # Set the major tag # + ##################### + MAJOR_TAG=$(echo "${IMAGE_VERSION}" | cut -d '.' -f1) + + ################################### + # Set flag for updating major tag # + ################################### + UPDATE_MAJOR_TAG=1 + + info "- Also deploying a major tag of:[${MAJOR_TAG}]" + fi + + ############################ + # Validate DOCKERFILE_PATH # + ############################ + if [ -z "${DOCKERFILE_PATH}" ]; then + error "Failed to get [DOCKERFILE_PATH]!" + fatal "[${DOCKERFILE_PATH}]" + else + info "Successfully found:${F[W]}[DOCKERFILE_PATH]${F[B]}, value:${F[W]}[${DOCKERFILE_PATH}]" + fi } ################################################################################ #### Function Authenticate ##################################################### Authenticate() { - ################ - # Pull in Vars # - ################ - USERNAME="${1}" # Name to auth with - PASSWORD="${2}" # Password to auth with - URL="${3}" # Url to auth towards - NAME="${4}" # name of the service - - ################ - # Print header # - ################ - info "----------------------------------------------" - info "Login to ${NAME}..." - info "----------------------------------------------" - - ################### - # Auth to service # - ################### - LOGIN_CMD=$(docker login "${URL}" --username "${USERNAME}" --password "${PASSWORD}" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ ${ERROR_CODE} -ne 0 ]; then - # ERROR - error "Failed to authenticate to ${NAME}!" - fatal "[${LOGIN_CMD}]" - else - # SUCCESS - info "Successfully authenticated to ${F[C]}${NAME}${F[B]}!" - fi + ################ + # Pull in Vars # + ################ + USERNAME="${1}" # Name to auth with + PASSWORD="${2}" # Password to auth with + URL="${3}" # Url to auth towards + NAME="${4}" # name of the service + + ################ + # Print header # + ################ + info "----------------------------------------------" + info "Login to ${NAME}..." + info "----------------------------------------------" + + ################### + # Auth to service # + ################### + LOGIN_CMD=$(docker login "${URL}" --username "${USERNAME}" --password "${PASSWORD}" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ ${ERROR_CODE} -ne 0 ]; then + # ERROR + error "Failed to authenticate to ${NAME}!" + fatal "[${LOGIN_CMD}]" + else + # SUCCESS + info "Successfully authenticated to ${F[C]}${NAME}${F[B]}!" + fi } ################################################################################ #### Function BuildImage ####################################################### BuildImage() { - ################ - # Print header # - ################ - info "----------------------------------------------" - info "Building the DockerFile image..." - info "----------------------------------------------" - - ################################ - # Validate the DOCKERFILE_PATH # - ################################ - if [ ! -f "${DOCKERFILE_PATH}" ]; then - # No file found - error "failed to find Dockerfile at:[${DOCKERFILE_PATH}]" - error "Please make sure you give full path!" - fatal "Example:[/configs/Dockerfile] or [Dockerfile] if at root directory" - fi - - ################### - # Build the image # - ################### - docker build --no-cache -t "${IMAGE_REPO}:${IMAGE_VERSION}" -f "${DOCKERFILE_PATH}" . 2>&1 - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ ${ERROR_CODE} -ne 0 ]; then - # ERROR - fatal "failed to [build] Dockerfile!" - else - # SUCCESS - info "Successfully Built image!" - fi - - ######################################################## - # Need to see if we need to tag a major update as well # - ######################################################## - if [ ${UPDATE_MAJOR_TAG} -eq 1 ]; then - # Tag the image with the major tag as well - docker build -t "${IMAGE_REPO}:${MAJOR_TAG}" -f "${DOCKERFILE_PATH}" . 2>&1 - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ ${ERROR_CODE} -ne 0 ]; then - # ERROR - fatal "failed to [tag] Dockerfile!" - else - # SUCCESS - info "Successfully tagged image!" - fi - fi + ################ + # Print header # + ################ + info "----------------------------------------------" + info "Building the DockerFile image..." + info "----------------------------------------------" + + ################################ + # Validate the DOCKERFILE_PATH # + ################################ + if [ ! -f "${DOCKERFILE_PATH}" ]; then + # No file found + error "failed to find Dockerfile at:[${DOCKERFILE_PATH}]" + error "Please make sure you give full path!" + fatal "Example:[/configs/Dockerfile] or [Dockerfile] if at root directory" + fi + + ################### + # Build the image # + ################### + docker build --no-cache -t "${IMAGE_REPO}:${IMAGE_VERSION}" -f "${DOCKERFILE_PATH}" . 2>&1 + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ ${ERROR_CODE} -ne 0 ]; then + # ERROR + fatal "failed to [build] Dockerfile!" + else + # SUCCESS + info "Successfully Built image!" + fi + + ######################################################## + # Need to see if we need to tag a major update as well # + ######################################################## + if [ ${UPDATE_MAJOR_TAG} -eq 1 ]; then + # Tag the image with the major tag as well + docker build -t "${IMAGE_REPO}:${MAJOR_TAG}" -f "${DOCKERFILE_PATH}" . 2>&1 + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ ${ERROR_CODE} -ne 0 ]; then + # ERROR + fatal "failed to [tag] Dockerfile!" + else + # SUCCESS + info "Successfully tagged image!" + fi + fi } ################################################################################ #### Function UploadImage ###################################################### UploadImage() { - ################ - # Print header # - ################ - info "----------------------------------------------" - info "Uploading the DockerFile image to ${REGISTRY}..." - info "----------------------------------------------" - - ############################################ - # Upload the docker image that was created # - ############################################ - docker push "${IMAGE_REPO}:${IMAGE_VERSION}" 2>&1 - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ ${ERROR_CODE} -ne 0 ]; then - # ERROR - fatal "failed to [upload] Dockerfile!" - else - # SUCCESS - info "Successfully Uploaded Docker image:${F[W]}[${IMAGE_VERSION}]${F[B]} to ${F[C]}${REGISTRY}${F[B]}!" - fi - - ######################### - # Get Image information # - ######################### - IFS=$'\n' # Set the delimit to newline - GET_INFO_CMD=$(docker images | grep "${IMAGE_REPO}" | grep "${IMAGE_VERSION}" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ ${ERROR_CODE} -ne 0 ]; then - # ERROR - error "Failed to get information about built Image!" - fatal "[${GET_INFO_CMD}]" - else - ################ - # Get the data # - ################ - REPO=$(echo "${GET_INFO_CMD}" | awk '{print ${1}}') - TAG=$(echo "${GET_INFO_CMD}" | awk '{print ${2}}') - IMAGE_ID=$(echo "${GET_INFO_CMD}" | awk '{print ${3}}') - SIZE="${GET_INFO_CMD##* }" - - ################### - # Print the goods # - ################### - info "----------------------------------------------" - info "Docker Image Details:" - info "Repository:[${REPO}]" - info "Tag:[${TAG}]" - info "Image_ID:[${IMAGE_ID}]" - info "Size:[${SIZE}]" - info "----------------------------------------------" - fi - - ############################################################### - # Check if we need to upload the major tagged version as well # - ############################################################### - if [ ${UPDATE_MAJOR_TAG} -eq 1 ]; then - ############################################ - # Upload the docker image that was created # - ############################################ - docker push "${IMAGE_REPO}:${MAJOR_TAG}" 2>&1 - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ ${ERROR_CODE} -ne 0 ]; then - # ERROR - fatal "failed to [upload] MAJOR_TAG:[${MAJOR_TAG}] Dockerfile!" - else - # SUCCESS - info "Successfully Uploaded TAG:${F[W]}[${MAJOR_TAG}]${F[B]} of Docker image to ${F[C]}${REGISTRY}${F[B]}!" - fi - fi + ################ + # Print header # + ################ + info "----------------------------------------------" + info "Uploading the DockerFile image to ${REGISTRY}..." + info "----------------------------------------------" + + ############################################ + # Upload the docker image that was created # + ############################################ + docker push "${IMAGE_REPO}:${IMAGE_VERSION}" 2>&1 + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ ${ERROR_CODE} -ne 0 ]; then + # ERROR + fatal "failed to [upload] Dockerfile!" + else + # SUCCESS + info "Successfully Uploaded Docker image:${F[W]}[${IMAGE_VERSION}]${F[B]} to ${F[C]}${REGISTRY}${F[B]}!" + fi + + ######################### + # Get Image information # + ######################### + IFS=$'\n' # Set the delimit to newline + GET_INFO_CMD=$(docker images | grep "${IMAGE_REPO}" | grep "${IMAGE_VERSION}" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ ${ERROR_CODE} -ne 0 ]; then + # ERROR + error "Failed to get information about built Image!" + fatal "[${GET_INFO_CMD}]" + else + ################ + # Get the data # + ################ + REPO=$(echo "${GET_INFO_CMD}" | awk '{print ${1}}') + TAG=$(echo "${GET_INFO_CMD}" | awk '{print ${2}}') + IMAGE_ID=$(echo "${GET_INFO_CMD}" | awk '{print ${3}}') + SIZE="${GET_INFO_CMD##* }" + + ################### + # Print the goods # + ################### + info "----------------------------------------------" + info "Docker Image Details:" + info "Repository:[${REPO}]" + info "Tag:[${TAG}]" + info "Image_ID:[${IMAGE_ID}]" + info "Size:[${SIZE}]" + info "----------------------------------------------" + fi + + ############################################################### + # Check if we need to upload the major tagged version as well # + ############################################################### + if [ ${UPDATE_MAJOR_TAG} -eq 1 ]; then + ############################################ + # Upload the docker image that was created # + ############################################ + docker push "${IMAGE_REPO}:${MAJOR_TAG}" 2>&1 + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ ${ERROR_CODE} -ne 0 ]; then + # ERROR + fatal "failed to [upload] MAJOR_TAG:[${MAJOR_TAG}] Dockerfile!" + else + # SUCCESS + info "Successfully Uploaded TAG:${F[W]}[${MAJOR_TAG}]${F[B]} of Docker image to ${F[C]}${REGISTRY}${F[B]}!" + fi + fi } ################################################################################ #### Function Footer ########################################################### Footer() { - info "-------------------------------------------------------" - info "The step has completed" - info "-------------------------------------------------------" + info "-------------------------------------------------------" + info "The step has completed" + info "-------------------------------------------------------" } ################################################################################ ################################## MAIN ######################################## @@ -451,22 +451,22 @@ BuildImage # Login to DockerHub # ###################### if [[ ${REGISTRY} == "Docker" ]]; then - # Authenticate "Username" "Password" "Url" "Name" - Authenticate "${DOCKER_USERNAME}" "${DOCKER_PASSWORD}" "" "Dockerhub" + # Authenticate "Username" "Password" "Url" "Name" + Authenticate "${DOCKER_USERNAME}" "${DOCKER_PASSWORD}" "" "Dockerhub" #################################### # Login to GitHub Package Registry # #################################### elif [[ ${REGISTRY} == "GPR" ]]; then - # Authenticate "Username" "Password" "Url" "Name" - Authenticate "${GPR_USERNAME}" "${GPR_TOKEN}" "https://docker.pkg.github.com" "GitHub Package Registry" + # Authenticate "Username" "Password" "Url" "Name" + Authenticate "${GPR_USERNAME}" "${GPR_TOKEN}" "https://docker.pkg.github.com" "GitHub Package Registry" else - ######### - # ERROR # - ######### - error "Registry not set correctly!" - fatal "Registry:[${REGISTRY}]" + ######### + # ERROR # + ######### + error "Registry not set correctly!" + fatal "Registry:[${REGISTRY}]" fi #################### diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 00baa4c..b54135c 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,11 +1,14 @@ # Contributing + :wave: Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great. ## Submitting a pull request + [Pull Requests][pulls] are used for adding new functionality, fixing bugs, improving documentation, and improving the developer experience overall. **With write access** + 1. Clone the repository (only if you do not have write access) 1. Run `pipenv install` 1. Create a new branch: `git checkout -b /` @@ -16,6 +19,7 @@ We're thrilled that you'd like to contribute to this project. Your help is essen 1. Pat yourself on the back and wait for your pull request to be reviewed and merged. **Without write access** + 1. [Fork][fork] and clone the repository 1. Run `pipenv install` 1. Create a new branch: `git checkout -b /` @@ -38,6 +42,7 @@ Work in Progress pull requests are also welcome to get feedback early on, or if - Open a pull request and request a review from a member of the appropriate `@github/ps-delivery` and/or `@github/ps-devops-engineering` teams ## Resources + - [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) - [Using Pull Requests](https://help.github.com/articles/about-pull-requests/) - [GitHub Help](https://help.github.com) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 284a288..e5c60c1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,7 +16,6 @@ updates: interval: "daily" open-pull-requests-limit: 10 - # Maintain dependencies for docker - package-ecosystem: "docker" directory: "/" diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index b7a9a4f..513b6e2 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -1,40 +1,40 @@ -name-template: 'v$RESOLVED_VERSION' -tag-template: 'v$RESOLVED_VERSION' +name-template: "v$RESOLVED_VERSION" +tag-template: "v$RESOLVED_VERSION" template: | # Changelog $CHANGES - - See details of [all code changes](https://github.com/github/github-team-sync/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION) since previous release - + + See details of [all code changes](https://github.com/github/github-team-sync/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION) since previous release + categories: - - title: '🚀 Features' + - title: "🚀 Features" labels: - - 'feature' - - 'enhancement' - - title: '🐛 Bug Fixes' + - "feature" + - "enhancement" + - title: "🐛 Bug Fixes" labels: - - 'fix' - - 'bugfix' - - 'bug' - - title: '🧰 Maintenance' + - "fix" + - "bugfix" + - "bug" + - title: "🧰 Maintenance" labels: - - 'infrastructure' - - 'automation' - - 'documentation' - - 'dependencies' - - title: '🏎 Performance' - label: 'performance' -change-template: '- $TITLE @$AUTHOR (#$NUMBER)' + - "infrastructure" + - "automation" + - "documentation" + - "dependencies" + - title: "🏎 Performance" + label: "performance" +change-template: "- $TITLE @$AUTHOR (#$NUMBER)" version-resolver: major: labels: - - 'type: breaking' + - "type: breaking" minor: labels: - - 'type: enhancement' + - "type: enhancement" patch: labels: - - 'type: bug' - - 'type: maintenance' - - 'type: documentation' + - "type: bug" + - "type: maintenance" + - "type: documentation" default: patch diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index c16e502..ae8dd3a 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -2,27 +2,27 @@ name: Code Formatting on: push: - branches-ignore: + branches-ignore: - main pull_request: - types: + types: - opened - ready_for_review - reopened jobs: ci: - runs-on: 'ubuntu-latest' + runs-on: "ubuntu-latest" name: PR Formatter steps: - name: Checkout repo uses: actions/checkout@v3 - + - name: Install python uses: actions/setup-python@v3 with: - python-version: '3.9' + python-version: "3.9" - name: Format the code uses: psf/black@stable diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 845f1ef..e811d30 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -5,7 +5,7 @@ on: push: branches: [main] schedule: - - cron: '30 1 * * 0' # Weekly scan + - cron: "30 1 * * 0" # Weekly scan jobs: codeql-analyze: @@ -18,8 +18,8 @@ jobs: strategy: fail-fast: false matrix: - language: ['python'] - build-mode: ['none'] # Use manual here if needed + language: ["python"] + build-mode: ["none"] # Use manual here if needed steps: - name: Checkout repository diff --git a/.github/workflows/deploy-PROD.yml b/.github/workflows/deploy-PROD.yml index dde3f0b..653fb04 100644 --- a/.github/workflows/deploy-PROD.yml +++ b/.github/workflows/deploy-PROD.yml @@ -16,7 +16,7 @@ on: push: branches: - - 'master' + - "master" ############### # Set the Job # diff --git a/.github/workflows/deploy-RELEASE.yml b/.github/workflows/deploy-RELEASE.yml index b20416e..6e3778e 100644 --- a/.github/workflows/deploy-RELEASE.yml +++ b/.github/workflows/deploy-RELEASE.yml @@ -16,7 +16,7 @@ on: release: # Want to run the automation when a release is created - types: ['published'] + types: ["published"] ############### # Set the Job # diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 07a99e5..7c3ed67 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -16,7 +16,6 @@ env: jobs: push: - runs-on: ubuntu-latest if: github.event_name == 'push' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5df9d3a..a7a262c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,18 +1,17 @@ repos: + - repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black -- repo: https://github.com/psf/black - rev: 20.8b1 - hooks: - - id: black + - repo: https://github.com/pre-commit/mirrors-isort + rev: v5.7.0 + hooks: + - id: isort + args: ["--profile", "black", "--filter-files"] -- repo: https://github.com/pre-commit/mirrors-isort - rev: v5.7.0 - hooks: - - id: isort - args: ["--profile", "black", "--filter-files"] - -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 - hooks: - - id: check-added-large-files - args: ['--maxkb=1500'] \ No newline at end of file + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + hooks: + - id: check-added-large-files + args: ["--maxkb=1500"] diff --git a/.vscode/launch.json b/.vscode/launch.json index 306f58e..07a5188 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,16 +1,16 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Python: Current File", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "integratedTerminal", - "justMyCode": true - } - ] -} \ No newline at end of file + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": true + } + ] +} diff --git a/README.md b/README.md index 4faf5b9..3471f67 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@ # GitHub Team Sync + This utility is intended to enable synchronization between GitHub and various LDAP and SAML providers. This is particularly useful for large organizations with many teams that either use GitHub Enterprise Cloud, do not use LDAP for authentication, or use a SAML provider other than what is natively supported. It supports both GitHub.com, GitHub Enterprise Server (GHES) and GitHub, but it will need to live in a location that can access your LDAP servers. ## Supported user directories + - LDAP - Active Directory - Azure AD @@ -14,56 +16,60 @@ It supports both GitHub.com, GitHub Enterprise Server (GHES) and GitHub, but it - Keycloak ## Features + This utility provides the following functionality: -| Feature | Supported | Description | -| --- | --- | --- | -| Sync Users | Yes | Add or remove users from `Teams` in GitHub to keep in sync with Active Directory groups | -| Dynamic Config | Yes | Utilize a `settings` file to derive Active Directory and GitHub settings | -| LDAP SSL | Yes | SSL or TLS connections. | -| Failure notifications | Yes | Presently supports opening a GitHub issue when sync failed. The repo is configurable. | -| Sync on new team | Yes | Synchronize users when a new team is created | -| Sync on team edit | No | This event is not processed currently | -| Custom team/group maps | Yes | The team `slug` and group name will be matched automatically, unless you define a custom mapping with `syncmap.yml` | -| Force custom map | Yes | Sync only team defined in `syncmap.yml` | -| Dry run / Test mode | Yes | Run and print the differences, but make no changes | -| Nested teams/groups | No | Synchronize groups within groups. Presently, if a group is a member of another group, it is skipped | +| Feature | Supported | Description | +| ---------------------- | --------- | ------------------------------------------------------------------------------------------------------------------- | +| Sync Users | Yes | Add or remove users from `Teams` in GitHub to keep in sync with Active Directory groups | +| Dynamic Config | Yes | Utilize a `settings` file to derive Active Directory and GitHub settings | +| LDAP SSL | Yes | SSL or TLS connections. | +| Failure notifications | Yes | Presently supports opening a GitHub issue when sync failed. The repo is configurable. | +| Sync on new team | Yes | Synchronize users when a new team is created | +| Sync on team edit | No | This event is not processed currently | +| Custom team/group maps | Yes | The team `slug` and group name will be matched automatically, unless you define a custom mapping with `syncmap.yml` | +| Force custom map | Yes | Sync only team defined in `syncmap.yml` | +| Dry run / Test mode | Yes | Run and print the differences, but make no changes | +| Nested teams/groups | No | Synchronize groups within groups. Presently, if a group is a member of another group, it is skipped | ## Creating the GitHub App on your GitHub instance + 1. On your GitHub instance, visit the `settings` page on the organization that you want to own the **GitHub** App, and navigate to the `GitHub Apps` section. - - You can access this page by visiting the following url: - `https:///organizations//settings/apps` + - You can access this page by visiting the following url: + `https:///organizations//settings/apps` 2. Create a new **GitHub App** with the following settings: - - **Webhook URL**: URL of the machine on which this app has been deployed (Example: `http://ip.of.machine:3000`) - - **Homepage URL**: URL of the machine on which this app has been deployed (Example: `http://ip.of.machine:3000`) - - **Webhook Secret**: The webhook secret that will be or has been defined as an environment variable in your deployment environment as `WEBHOOK_SECRET` - - **Permissions and Events**: This application will need to be able to manage teams on GitHub, so the `events` and `permissions` listed below will be required. For more information on how to create a GitHub App, please visit [https://developer.github.com/apps/building-github-apps/creating-a-github-app](https://developer.github.com/apps/building-github-apps/creating-a-github-app) + - **Webhook URL**: URL of the machine on which this app has been deployed (Example: `http://ip.of.machine:3000`) + - **Homepage URL**: URL of the machine on which this app has been deployed (Example: `http://ip.of.machine:3000`) + - **Webhook Secret**: The webhook secret that will be or has been defined as an environment variable in your deployment environment as `WEBHOOK_SECRET` + - **Permissions and Events**: This application will need to be able to manage teams on GitHub, so the `events` and `permissions` listed below will be required. For more information on how to create a GitHub App, please visit [https://developer.github.com/apps/building-github-apps/creating-a-github-app](https://developer.github.com/apps/building-github-apps/creating-a-github-app) 3. Once these have been configured, select the `Create GitHub App` button at the bottom of the page to continue 4. Make a note of the `APP ID` on your newly-created **GitHub App**. You will need to set this as an environment variable when configuring the app. 5. Generate and download a private key from the new App page, and store it in your deployment environment. You can either do this by saving the file directly in the environment and specifying its path with the environment variable `PRIVATE_KEY_PATH` 6. After you have created the **GitHub** App, you will need to install it to the desired **GitHub** Organizations. - - Select `Install App` - - Select `All Repositories` or the desired repositories you wish to watch + - Select `Install App` + - Select `All Repositories` or the desired repositories you wish to watch ### Permissions and Events #### Permissions -| Category | Attribute | Permission | -| --- | --- | --- | -| Repository permissions | `Issues` | `Read & write` | -| Repository permissions | `Metadata` | `Read-only` | -| Organization permissions | `Members` | `Read & write` | -| User permissions | `Email addresses` | `Read-only` | +| Category | Attribute | Permission | +| ------------------------ | ----------------- | -------------- | +| Repository permissions | `Issues` | `Read & write` | +| Repository permissions | `Metadata` | `Read-only` | +| Organization permissions | `Members` | `Read & write` | +| User permissions | `Email addresses` | `Read-only` | #### Events -| Event | Required? | Description | -| --- | --- | --- | -| `Team` | Optional | Trigger when a new team is `created`, `deleted`, `edited`, `renamed`, etc. | +| Event | Required? | Description | +| ------ | --------- | -------------------------------------------------------------------------- | +| `Team` | Optional | Trigger when a new team is `created`, `deleted`, `edited`, `renamed`, etc. | #### Azure AD Permissions + **Authentication methods** + - [ ] Username/Password - [x] Service Principal - [ ] Certificate @@ -75,11 +81,15 @@ This app requires the following Azure permissions: - `User.Read.All` #### Keycloak Permissions + If you have `ADMIN_FINE_GRAINED_AUTHZ` enabled, you only need the following permission for the user realm: + - `view-users` #### Google Workspace Permissions + You must delegate domain-wide authority to the service account with the following scopes: + - `https://www.googleapis.com/auth/admin.directory.group.readonly` - `https://www.googleapis.com/auth/admin.directory.group.member.readonly` - `https://www.googleapis.com/auth/admin.directory.user.readonly` @@ -88,6 +98,7 @@ You must provide a Google Workspace Admin account for the service account to imp It must have Admin API permissions greater or equal to the scopes listed above. ## Getting Started + To get started, ensure that you are using **Python 3.9** (or update your `Pipfile` to the version you're running, 3.4+). The following additional libraries are required: - [ ] Flask @@ -111,6 +122,7 @@ pipenv install Once you have all of the requirements installed, be sure to edit the `.env` to match your environment. ### Sample `.env` for GitHub App settings + ```env ## GitHub App settings WEBHOOK_SECRET=development @@ -120,6 +132,7 @@ GHE_HOST=github.example.com ``` ### Sample `.env` for choosing your backend + ```env ## AzureAD = AAD ## AD/LDAP = LDAP @@ -152,6 +165,7 @@ LDAP_SEARCH_PAGE_SIZE=1000 ``` ### Sample `.env` for OpenLDAP + ```env LDAP_SERVER_HOST=dc1.example.com LDAP_SERVER_PORT=389 @@ -169,6 +183,7 @@ LDAP_SEARCH_PAGE_SIZE=1000 ``` ### Sample `.env` for AzureAD + ```env AZURE_TENANT_ID="" AZURE_CLIENT_ID="" @@ -183,6 +198,7 @@ AZURE_USE_TRANSITIVE_GROUP_MEMBERS=false ``` ### Sample `.env` for Okta + ```env OKTA_ORG_URL=https://example.okta.com OKTA_USERNAME_ATTRIBUTE=github_username @@ -198,6 +214,7 @@ OKTA_PRIVATE_KEY='{"kty": "RSA", ...}' ``` ### Sample `.env` for Keycloak + ```env KEYCLOAK_USERNAME=api-account KEYCLOAK_PASSWORD=ExamplePassword @@ -207,6 +224,7 @@ KEYCLOAK_USE_GITHUB_IDP=true ``` ### Sample `.env` for OneLogin + ```env ONELOGIN_CLIENT_ID='asdafsflkjlk13q33433445wee' ONELOGIN_CLIENT_SECRET='ca3a86f982fjjkjjkfkhls' @@ -214,6 +232,7 @@ REGION=US ``` ### Sample `.env` for Google Workspace + ```env GOOGLE_WORKSPACE_SA_CREDS_FILE=googleAuth.json GOOGLE_WORKSPACE_ADMIN_EMAIL=admin@example.com @@ -222,6 +241,7 @@ GOOGLE_WORKSPACE_USERNAME_FIELD=field-name ``` ### Sample `.env` settings for additional settings + ```env ## Additional settings CHANGE_THRESHOLD=25 @@ -240,6 +260,7 @@ REMOVE_ORG_MEMBERS_WITHOUT_TEAM=false ``` ### Sample `.env` setting for flask app + ```env #################### ## Flask Settings ## @@ -256,6 +277,7 @@ FLASK_RUN_HOST=0.0.0.0 ``` ### Sample `syncmap.yml` custom mapping file + ```yaml --- mapping: @@ -266,11 +288,12 @@ mapping: directory: some other group ``` -The custom map uses slugs that are lowercase. If you don't specify organization name, it will synchronize all teams with same name in any organization. +The custom map uses slugs that are lowercase. If you don't specify organization name, it will synchronize all teams with same name in any organization. ## Usage Examples ### Start the application from Pipenv + This example runs the app in a standard Flask environment. ```bash @@ -288,7 +311,9 @@ pipenv run python app.py ⚠️ This is free and open-source software that is supported by the open-source community, and is not included as part of GitHub's official platform support. ## Credits + This project draws much from: + - [Flask-GitHubApp](https://github.com/bradshjg/flask-githubapp) - [github3.py](https://github.com/sigmavirus24/github3.py) - [msal](https://github.com/AzureAD/microsoft-authentication-library-for-python) diff --git a/app.py b/app.py index 2d84d41..9c926c5 100644 --- a/app.py +++ b/app.py @@ -1,28 +1,28 @@ import atexit -from operator import truediv -import os -import time import json -import github3 -from distutils.util import strtobool -import threading +import os import sys +import threading +import time import traceback from concurrent.futures import ThreadPoolExecutor +from distutils.util import strtobool +from operator import truediv +import github3 from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.cron import CronTrigger from flask import Flask from githubapp import ( - GitHubApp, - DirectoryClient, - CRON_INTERVAL, - TEST_MODE, ADD_MEMBER, + CRON_INTERVAL, REMOVE_ORG_MEMBERS_WITHOUT_TEAM, - USER_SYNC_ATTRIBUTE, SYNCMAP_ONLY, + TEST_MODE, + USER_SYNC_ATTRIBUTE, + DirectoryClient, + GitHubApp, ) app = Flask(__name__) @@ -258,7 +258,7 @@ def load_custom_map(file="syncmap.yml"): ignore_users = [] group_prefix = [] if os.path.isfile(file): - from yaml import load, Loader + from yaml import Loader, load with open(file, "r") as f: data = load(f, Loader=Loader) diff --git a/githubapp/azuread.py b/githubapp/azuread.py index 651ac47..81cadf1 100644 --- a/githubapp/azuread.py +++ b/githubapp/azuread.py @@ -1,166 +1,167 @@ -import os -import json -import logging -from distutils.util import strtobool -import requests -import msal - -# Optional logging -# logging.basicConfig(level=logging.DEBUG) # Enable DEBUG log for entire script -# logging.getLogger("msal").setLevel(logging.INFO) # Optionally disable MSAL DEBUG logs - -LOG = logging.getLogger(__name__) - - -class AzureAD: - def __init__(self): - self.AZURE_TENANT_ID = os.environ["AZURE_TENANT_ID"] - self.AZURE_CLIENT_ID = os.environ["AZURE_CLIENT_ID"] - self.AZURE_CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"] - self.AZURE_APP_SCOPE = [ - f"https://graph.microsoft.com/{x}" - for x in os.environ["AZURE_APP_SCOPE"].split(" ") - ] - self.AZURE_API_ENDPOINT = os.environ.get( - "AZURE_API_ENDPOINT", "https://graph.microsoft.com/v1.0" - ) - self.USERNAME_ATTRIBUTE = os.environ.get( - "AZURE_USERNAME_ATTRIBUTE", "userPrincipalName" - ) - self.AZURE_USER_IS_UPN = strtobool(os.environ.get("AZURE_USER_IS_UPN", "False")) - self.AZURE_USE_TRANSITIVE_GROUP_MEMBERS = strtobool( - os.environ.get("AZURE_USE_TRANSITIVE_GROUP_MEMBERS", "False") - ) - - def get_access_token(self): - """ - Get the access token for this Azure Service Principal - :return access_token: - """ - app = msal.ConfidentialClientApplication( - self.AZURE_CLIENT_ID, - authority=f"https://login.microsoftonline.com/{self.AZURE_TENANT_ID}", - client_credential=self.AZURE_CLIENT_SECRET, - ) - - # Lookup the token in cache - result = app.acquire_token_silent(self.AZURE_APP_SCOPE, account=None) - - if not result: - logging.info( - "No suitable token exists in cache. Let's get a new one from AAD." - ) - result = app.acquire_token_for_client(scopes=self.AZURE_APP_SCOPE) - - if "access_token" in result: - # print("Successfully authenticated!") - return result["access_token"] - - else: - print(result.get("error")) - print(result.get("error_description")) - print( - result.get("correlation_id") - ) # You may need this when reporting a bug - - def get_group_members(self, token=None, group_name=None): - """ - Get a list of members for a given group - :param token: - :param group_name: - :return: - """ - token = self.get_access_token() if not token else token - member_list = [] - # Calling graph using the access token - # url encode the group name - group_name = requests.utils.quote(group_name) - graph_data = requests.get( # Use token to call downstream service - f"{self.AZURE_API_ENDPOINT}/groups?$filter=displayName eq '{group_name}'", - headers={"Authorization": f"Bearer {token}"}, - ).json() - # print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) - try: - group_info = json.loads(json.dumps(graph_data, indent=2))["value"][0] - members_endpoint = ( - "transitiveMembers" - if self.AZURE_USE_TRANSITIVE_GROUP_MEMBERS - else "members" - ) - members = self.get_group_members_pages( - token, - f'{self.AZURE_API_ENDPOINT}/groups/{group_info["id"]}/{members_endpoint}', - ) - except IndexError as e: - members = [] - for member in members: - if member["@odata.type"] == "#microsoft.graph.group": - print("Nested group: ", member["displayName"]) - else: - user_info = self.get_user_info(token=token, user=member["id"]) - if self.USERNAME_ATTRIBUTE.startswith("extensionAttribute"): - username = user_info["onPremisesExtensionAttributes"][ - self.USERNAME_ATTRIBUTE - ] - if username is None: - continue - else: - username = user_info[self.USERNAME_ATTRIBUTE] - if self.AZURE_USER_IS_UPN: - if r"\\" in username: - username = username.split(r"\\")[1] - username = username.split("@")[0].split("#")[0].split("_")[0] - username = username.translate(str.maketrans("._!#^~", "------")) - username = username.lower() - if "EMU_SHORTCODE" in os.environ: - username = username + "_" + os.environ["EMU_SHORTCODE"] - user = { - "username": username, - "email": user_info["mail"], - } - member_list.append(user) - return member_list - - def get_group_members_pages(self, token=None, url=None): - """ - Get group members - :param token: - :param url: - :return members: - :rtype members: dict - """ - members_data = requests.get(url, headers={"Authorization": f"Bearer {token}"}) - if members_data.ok != True: - print( - f"[GetMembers]: Error getting members data error code {members_data.status_code}" - ) - return [] - - members_data_content = members_data.json() - members = members_data_content["value"] - if "@odata.nextLink" in members_data_content: - members.extend( - self.get_group_members_pages( - token, members_data_content["@odata.nextLink"] - ) - ) - return members - - def get_user_info(self, token=None, user=None): - """ - Get user info - :param token: - :param user: - :return user_info: - :rtype user_info: dict - """ - token = self.get_access_token() if not token else token - attribute = self.USERNAME_ATTRIBUTE - if self.USERNAME_ATTRIBUTE.startswith("extensionAttribute"): - attribute = "onPremisesExtensionAttributes" - graph_data = requests.get( # Use token to call downstream service - f"{self.AZURE_API_ENDPOINT}/users/{user}?$select=id,mail,{attribute}", - headers={"Authorization": f"Bearer {token}"}, - ).json() - user_info = json.loads(json.dumps(graph_data, indent=2)) - return user_info +import json +import logging +import os +from distutils.util import strtobool + +import msal +import requests + +# Optional logging +# logging.basicConfig(level=logging.DEBUG) # Enable DEBUG log for entire script +# logging.getLogger("msal").setLevel(logging.INFO) # Optionally disable MSAL DEBUG logs + +LOG = logging.getLogger(__name__) + + +class AzureAD: + def __init__(self): + self.AZURE_TENANT_ID = os.environ["AZURE_TENANT_ID"] + self.AZURE_CLIENT_ID = os.environ["AZURE_CLIENT_ID"] + self.AZURE_CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"] + self.AZURE_APP_SCOPE = [ + f"https://graph.microsoft.com/{x}" + for x in os.environ["AZURE_APP_SCOPE"].split(" ") + ] + self.AZURE_API_ENDPOINT = os.environ.get( + "AZURE_API_ENDPOINT", "https://graph.microsoft.com/v1.0" + ) + self.USERNAME_ATTRIBUTE = os.environ.get( + "AZURE_USERNAME_ATTRIBUTE", "userPrincipalName" + ) + self.AZURE_USER_IS_UPN = strtobool(os.environ.get("AZURE_USER_IS_UPN", "False")) + self.AZURE_USE_TRANSITIVE_GROUP_MEMBERS = strtobool( + os.environ.get("AZURE_USE_TRANSITIVE_GROUP_MEMBERS", "False") + ) + + def get_access_token(self): + """ + Get the access token for this Azure Service Principal + :return access_token: + """ + app = msal.ConfidentialClientApplication( + self.AZURE_CLIENT_ID, + authority=f"https://login.microsoftonline.com/{self.AZURE_TENANT_ID}", + client_credential=self.AZURE_CLIENT_SECRET, + ) + + # Lookup the token in cache + result = app.acquire_token_silent(self.AZURE_APP_SCOPE, account=None) + + if not result: + logging.info( + "No suitable token exists in cache. Let's get a new one from AAD." + ) + result = app.acquire_token_for_client(scopes=self.AZURE_APP_SCOPE) + + if "access_token" in result: + # print("Successfully authenticated!") + return result["access_token"] + + else: + print(result.get("error")) + print(result.get("error_description")) + print( + result.get("correlation_id") + ) # You may need this when reporting a bug + + def get_group_members(self, token=None, group_name=None): + """ + Get a list of members for a given group + :param token: + :param group_name: + :return: + """ + token = self.get_access_token() if not token else token + member_list = [] + # Calling graph using the access token + # url encode the group name + group_name = requests.utils.quote(group_name) + graph_data = requests.get( # Use token to call downstream service + f"{self.AZURE_API_ENDPOINT}/groups?$filter=displayName eq '{group_name}'", + headers={"Authorization": f"Bearer {token}"}, + ).json() + # print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) + try: + group_info = json.loads(json.dumps(graph_data, indent=2))["value"][0] + members_endpoint = ( + "transitiveMembers" + if self.AZURE_USE_TRANSITIVE_GROUP_MEMBERS + else "members" + ) + members = self.get_group_members_pages( + token, + f'{self.AZURE_API_ENDPOINT}/groups/{group_info["id"]}/{members_endpoint}', + ) + except IndexError as e: + members = [] + for member in members: + if member["@odata.type"] == "#microsoft.graph.group": + print("Nested group: ", member["displayName"]) + else: + user_info = self.get_user_info(token=token, user=member["id"]) + if self.USERNAME_ATTRIBUTE.startswith("extensionAttribute"): + username = user_info["onPremisesExtensionAttributes"][ + self.USERNAME_ATTRIBUTE + ] + if username is None: + continue + else: + username = user_info[self.USERNAME_ATTRIBUTE] + if self.AZURE_USER_IS_UPN: + if r"\\" in username: + username = username.split(r"\\")[1] + username = username.split("@")[0].split("#")[0].split("_")[0] + username = username.translate(str.maketrans("._!#^~", "------")) + username = username.lower() + if "EMU_SHORTCODE" in os.environ: + username = username + "_" + os.environ["EMU_SHORTCODE"] + user = { + "username": username, + "email": user_info["mail"], + } + member_list.append(user) + return member_list + + def get_group_members_pages(self, token=None, url=None): + """ + Get group members + :param token: + :param url: + :return members: + :rtype members: dict + """ + members_data = requests.get(url, headers={"Authorization": f"Bearer {token}"}) + if members_data.ok != True: + print( + f"[GetMembers]: Error getting members data error code {members_data.status_code}" + ) + return [] + + members_data_content = members_data.json() + members = members_data_content["value"] + if "@odata.nextLink" in members_data_content: + members.extend( + self.get_group_members_pages( + token, members_data_content["@odata.nextLink"] + ) + ) + return members + + def get_user_info(self, token=None, user=None): + """ + Get user info + :param token: + :param user: + :return user_info: + :rtype user_info: dict + """ + token = self.get_access_token() if not token else token + attribute = self.USERNAME_ATTRIBUTE + if self.USERNAME_ATTRIBUTE.startswith("extensionAttribute"): + attribute = "onPremisesExtensionAttributes" + graph_data = requests.get( # Use token to call downstream service + f"{self.AZURE_API_ENDPOINT}/users/{user}?$select=id,mail,{attribute}", + headers={"Authorization": f"Bearer {token}"}, + ).json() + user_info = json.loads(json.dumps(graph_data, indent=2)) + return user_info diff --git a/githubapp/core.py b/githubapp/core.py index 2684192..4ce2d70 100644 --- a/githubapp/core.py +++ b/githubapp/core.py @@ -1,12 +1,13 @@ """ Flask extension for rapid GitHub app development """ -import os.path + +import distutils import hmac import logging -import distutils +import os.path -from flask import abort, current_app, jsonify, request, _app_ctx_stack +from flask import _app_ctx_stack, abort, current_app, jsonify, request from github3 import GitHub, GitHubEnterprise LOG = logging.getLogger(__name__) @@ -90,6 +91,7 @@ def init_app(self, app): ) app.add_url_rule("/health_check", endpoint="health_check") + @app.endpoint("health_check") def health_check(): return "Web server is running.", 200 diff --git a/githubapp/googleworkspace.py b/githubapp/googleworkspace.py index de6452d..cdd23d0 100644 --- a/githubapp/googleworkspace.py +++ b/githubapp/googleworkspace.py @@ -1,12 +1,13 @@ -import os -import traceback -import sys import json import logging -from google.oauth2 import service_account -import googleapiclient.discovery +import os +import sys +import traceback from pprint import pprint +import googleapiclient.discovery +from google.oauth2 import service_account + LOG = logging.getLogger(__name__) SCOPES = [ diff --git a/githubapp/keycloak.py b/githubapp/keycloak.py index 85dbfb4..db543be 100644 --- a/githubapp/keycloak.py +++ b/githubapp/keycloak.py @@ -1,10 +1,10 @@ import asyncio import collections -import os import logging +import os import re -from keycloak import KeycloakAdmin +from keycloak import KeycloakAdmin LOG = logging.getLogger(__name__) @@ -22,7 +22,7 @@ def __init__(self): if not os.environ.get("KEYCLOAK_REALM"): os.environ["KEYCLOAK_REALM"] = "master" - + if not os.environ.get("KEYCLOAK_ADMIN_REALM"): os.environ["KEYCLOAK_ADMIN_REALM"] = os.environ.get("KEYCLOAK_REALM") @@ -33,7 +33,7 @@ def __init__(self): username=os.environ["KEYCLOAK_USERNAME"], password=os.environ["KEYCLOAK_PASSWORD"], realm_name=os.environ["KEYCLOAK_REALM"], - user_realm_name=os.environ["KEYCLOAK_ADMIN_REALM"] + user_realm_name=os.environ["KEYCLOAK_ADMIN_REALM"], ) def get_group_members(self, group_name: str = None): @@ -57,7 +57,13 @@ def get_group_id(client: KeycloakAdmin = None): :return: The group's UUID in Keycloak """ - group = client.get_groups(query={"search": group_name, "briefRepresentation": "true", "exact": "true"}) + group = client.get_groups( + query={ + "search": group_name, + "briefRepresentation": "true", + "exact": "true", + } + ) if not group: raise Exception(f"Cannot find group {group_name} in Keycloak") else: @@ -80,15 +86,13 @@ def get_members(client: KeycloakAdmin = None, group_id: str = None): page_size = 100 members = [] group_members = client.get_group_members( - group_id=group_id, - query={"first": page_start, "max": page_size} + group_id=group_id, query={"first": page_start, "max": page_size} ) members += group_members while len(group_members) == page_size: page_start += page_size group_members = client.get_group_members( - group_id=group_id, - query={"first": page_start, "max": page_size} + group_id=group_id, query={"first": page_start, "max": page_size} ) members += group_members return members @@ -118,19 +122,16 @@ def get_github_username(client: KeycloakAdmin = None, user_id: str = None): for user in users: try: if self.UseGithubIDP: - username = get_github_username(client=self.client, user_id=user["id"]) + username = get_github_username( + client=self.client, user_id=user["id"] + ) else: username = user["username"] if not username: raise Exception("Unable to find username in profile") if "EMU_SHORTCODE" in os.environ: username = username + "_" + os.environ["EMU_SHORTCODE"] - member_list.append( - { - "username": username, - "email": user["email"] - } - ) + member_list.append({"username": username, "email": user["email"]}) except Exception as e: user_info = f'{user["username"]} ({user["email"]})' print(f"User {user_info}: {e}") diff --git a/githubapp/ldap.py b/githubapp/ldap.py index 8af89f3..30b06a0 100644 --- a/githubapp/ldap.py +++ b/githubapp/ldap.py @@ -1,13 +1,14 @@ -import os -import traceback -import sys import json import logging +import os import ssl -from ldap3 import Server, Connection, Tls, ALL -from ldap3.utils.conv import escape_filter_chars +import sys +import traceback from pprint import pprint +from ldap3 import ALL, Connection, Server, Tls +from ldap3.utils.conv import escape_filter_chars + LOG = logging.getLogger(__name__) diff --git a/githubapp/okta.py b/githubapp/okta.py index e186c0a..6818b1d 100644 --- a/githubapp/okta.py +++ b/githubapp/okta.py @@ -1,87 +1,87 @@ -import asyncio -import os -import logging -import re -from okta.client import Client as OktaClient - - -LOG = logging.getLogger(__name__) - - -class Okta: - def __init__(self): - self.USERNAME_ATTRIBUTE = os.environ.get("OKTA_USERNAME_ATTRIBUTE", "login") - auth_method = os.environ.get("OKTA_AUTH_METHOD", "token") - config = {"orgUrl": os.environ["OKTA_ORG_URL"]} - if auth_method == "oauth": - config["authorizationMode"] = "PrivateKey" - config["clientId"] = os.environ["OKTA_CLIENT_ID"] - config["scopes"] = os.environ["OKTA_SCOPES"].split(" ") - config["privateKey"] = os.environ["OKTA_PRIVATE_KEY"] - else: - config["token"] = os.environ["OKTA_ACCESS_TOKEN"] - self.client = OktaClient(config) - - def get_group_members(self, group_name=None): - """ - Get a list of users that are part of a given group in Okta - :param group_name: Group name to look up - :type group_name: str - :return member_list: A list of dictionaries containing usernames and emails - :rtype member_list: list - """ - member_list = [] - - async def get_group_id(client=None): - """ - Get the group ID - :return: - """ - group = await client.list_groups(query_params={"q": group_name}) - return group[0][0].id - - async def get_members(client=None, groupId=None): - """ - Get the users that belong to this group - :param groupId: - :return: - """ - members = await client.list_group_users(groupId=groupId) - return members[0] - - def get_or_create_eventloop(): - """ - Create an async loop if we're in a child thread - :return: - """ - try: - return asyncio.get_event_loop() - except RuntimeError as ex: - if "There is no current event loop in thread" in str(ex): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - return asyncio.get_event_loop() - - loop = get_or_create_eventloop() - gid = loop.run_until_complete(get_group_id(client=self.client)) - users = loop.run_until_complete(get_members(client=self.client, groupId=gid)) - for user in users: - try: - username = getattr(user.profile, self.USERNAME_ATTRIBUTE) - username = username.split("@")[0] - username = re.sub("[^0-9a-zA-Z-]+", "-", username) - if "EMU_SHORTCODE" in os.environ: - username = username + "_" + os.environ["EMU_SHORTCODE"] - member_list.append( - { - "username": username, - "email": user.profile.email, - } - ) - except AttributeError as e: - if user.links: - user_info = user.links["self"]["href"] - else: - user_info = user - print(f"User {user_info}: {e}") - return member_list +import asyncio +import logging +import os +import re + +from okta.client import Client as OktaClient + +LOG = logging.getLogger(__name__) + + +class Okta: + def __init__(self): + self.USERNAME_ATTRIBUTE = os.environ.get("OKTA_USERNAME_ATTRIBUTE", "login") + auth_method = os.environ.get("OKTA_AUTH_METHOD", "token") + config = {"orgUrl": os.environ["OKTA_ORG_URL"]} + if auth_method == "oauth": + config["authorizationMode"] = "PrivateKey" + config["clientId"] = os.environ["OKTA_CLIENT_ID"] + config["scopes"] = os.environ["OKTA_SCOPES"].split(" ") + config["privateKey"] = os.environ["OKTA_PRIVATE_KEY"] + else: + config["token"] = os.environ["OKTA_ACCESS_TOKEN"] + self.client = OktaClient(config) + + def get_group_members(self, group_name=None): + """ + Get a list of users that are part of a given group in Okta + :param group_name: Group name to look up + :type group_name: str + :return member_list: A list of dictionaries containing usernames and emails + :rtype member_list: list + """ + member_list = [] + + async def get_group_id(client=None): + """ + Get the group ID + :return: + """ + group = await client.list_groups(query_params={"q": group_name}) + return group[0][0].id + + async def get_members(client=None, groupId=None): + """ + Get the users that belong to this group + :param groupId: + :return: + """ + members = await client.list_group_users(groupId=groupId) + return members[0] + + def get_or_create_eventloop(): + """ + Create an async loop if we're in a child thread + :return: + """ + try: + return asyncio.get_event_loop() + except RuntimeError as ex: + if "There is no current event loop in thread" in str(ex): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + return asyncio.get_event_loop() + + loop = get_or_create_eventloop() + gid = loop.run_until_complete(get_group_id(client=self.client)) + users = loop.run_until_complete(get_members(client=self.client, groupId=gid)) + for user in users: + try: + username = getattr(user.profile, self.USERNAME_ATTRIBUTE) + username = username.split("@")[0] + username = re.sub("[^0-9a-zA-Z-]+", "-", username) + if "EMU_SHORTCODE" in os.environ: + username = username + "_" + os.environ["EMU_SHORTCODE"] + member_list.append( + { + "username": username, + "email": user.profile.email, + } + ) + except AttributeError as e: + if user.links: + user_info = user.links["self"]["href"] + else: + user_info = user + print(f"User {user_info}: {e}") + return member_list diff --git a/githubapp/onelogin.py b/githubapp/onelogin.py index 09edcff..bf29bc6 100644 --- a/githubapp/onelogin.py +++ b/githubapp/onelogin.py @@ -1,6 +1,7 @@ -from onelogin.api.client import OneLoginClient import os +from onelogin.api.client import OneLoginClient + class OneLogin: def __init__(self):