From 34f1425f5a105b7b110931010493d44cd42d9718 Mon Sep 17 00:00:00 2001 From: ggtrd <103002419+ggtrd@users.noreply.github.com> Date: Fri, 13 Feb 2026 16:42:29 +0100 Subject: [PATCH 1/2] superset --- main.tf | 18 +- .../kube_objects/superset_config.py | 153 ++++ modules/chart_superset/main.tf | 77 ++ modules/chart_superset/values.yaml | 687 ++++++++++++++++++ modules/chart_superset/variables.tf | 20 + 5 files changed, 954 insertions(+), 1 deletion(-) create mode 100644 modules/chart_superset/kube_objects/superset_config.py create mode 100644 modules/chart_superset/main.tf create mode 100644 modules/chart_superset/values.yaml create mode 100644 modules/chart_superset/variables.tf diff --git a/main.tf b/main.tf index 7081c14..3500d95 100644 --- a/main.tf +++ b/main.tf @@ -101,7 +101,8 @@ module "kube_namespaces" { "monitoring", "redis", "keycloak", - "harbor" + "harbor", + "superset" ] } @@ -215,6 +216,21 @@ module "chart_cert_manager" { } +module "chart_superset" { + source = "./modules/chart_superset" + + namespace = "superset" + cluster_domain = "superset-${local.cluster_domain}" + helm_repo = "https://charts.bitnami.com/bitnami" + helm_chart = "superset" + helm_chart_version = "5.0.0" + + depends_on = [ + module.kube_namespaces + ] +} + + module "chart_harbor" { source = "./modules/chart_harbor" diff --git a/modules/chart_superset/kube_objects/superset_config.py b/modules/chart_superset/kube_objects/superset_config.py new file mode 100644 index 0000000..1ef7cae --- /dev/null +++ b/modules/chart_superset/kube_objects/superset_config.py @@ -0,0 +1,153 @@ +import os +import logging +from logging import getLogger +from flask_appbuilder.security.manager import AUTH_OAUTH +from superset import SupersetSecurityManager + +logger = logging.getLogger() +log = getLogger(__name__) +log.setLevel(logging.DEBUG) + +### This file is based on this one https://github.com/apache/superset/blob/master/superset/config.py + +# Start Utility functions +def is_boolean_yes(var): + if var == 1 or var == "yes" or var == "true": + return True + return False + +def env(key, default=None): + return os.getenv(key, default) +# End Utility functions + +# Start custom_sso_security_manager (https://superset.apache.org/docs/configuration/configuring-superset/#custom-oauth2-configuration) +class KeycloakSecurity(SupersetSecurityManager): + """ + Create a new SecurityManager with own oauth_user_info to handle the information from Keycloak + """ + + def oauth_user_info(self, provider, resp=None): + log.debug("Keycloak response received : {0}".format(resp)) + id_token = resp["id_token"] + log.debug("ID Token: %s", id_token) + userinfo = resp["userinfo"] + log.debug("Token userinfo: %s", userinfo) + issuer = userinfo["iss"] + log.debug("User info issuer: %s", issuer) + me = self.appbuilder.sm.oauth_remotes[provider].get( + f'{issuer}/protocol/openid-connect/userinfo' + ) + me.raise_for_status() + data = me.json() + log.debug("User info from Keycloak: %s", data) + return { + "name": data["name"], + "email": data["email"], + "first_name": data["given_name"], + "last_name": data["family_name"], + "id": data["preferred_username"], + "username": data["preferred_username"], + "role_keys": data.get("userRoles", []) + } +# End custom_sso_security_manager + +## URLs config +ROOT_URL = 'https://superset-warp.api.cosmotech.com' +LOGOUT_REDIRECT_URL = 'https://superset-warp.api.cosmotech.com' + +# Auth config +AUTH_USER_REGISTRATION = True +AUTH_TYPE = AUTH_OAUTH +CUSTOM_SECURITY_MANAGER = KeycloakSecurity +# https://github.com/apache/superset/blob/5.0/docs/docs/configuration/configuring-superset.mdx#mapping-oauth-groups-to-superset-roles) +AUTH_ROLES_MAPPING = { + "Platform.Admin": ["Admin"], + "Organization.User": ["Gamma"], +} +AUTH_ROLES_SYNC_AT_LOGIN = True + +OAUTH_PROVIDERS = [ + { + "name": "sphinx", + "icon": "fa-key", + "token_key": "access_token", + "remote_app": { + "client_id": "cosmotech-superset-client", + "client_secret": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "client_kwargs": {"scope": "openid profile email"}, + "server_metadata_url": "https://warp.api.cosmotech.com/keycloak/realms/sphinx/.well-known/openid-configuration" + } + }, + { + "name": "eng-api-ci", + "icon": "fa-key", + "token_key": "access_token", + "remote_app": { + "client_id": "cosmotech-superset-client", + "client_secret": "xxxxxxxxxxxxxxxxxxxxxxxxx", + "client_kwargs": {"scope": "openid profile email"}, + "server_metadata_url": "https://warp.api.cosmotech.com/keycloak/realms/eng-api-ci/.well-known/openid-configuration" + } + } +] + +# Flask App Builder configuration +# Your App secret key will be used for securely signing the session cookie +# and encrypting sensitive information on the database +# Make sure you are changing this key for your deployment with a strong key. +# Alternatively you can set it with `SUPERSET_SECRET_KEY` environment variable. +# You MUST set this for production environments or the server will refuse +# to start and you will see an error in the logs accordingly. +# SECRET_KEY = 'xxxxxxxxxxxxxxxxxxxxxxx' + +# Database config +## +DB_DIALECT = env("SUPERSET_DATABASE_DIALECT", "postgresql+psycopg2") +DB_USER = env("SUPERSET_DATABASE_USER") +DB_PASSWORD = env("SUPERSET_DATABASE_PASSWORD") +DB_HOST = env("SUPERSET_DATABASE_HOST", "postgresql") +DB_PORT = env("SUPERSET_DATABASE_PORT_NUMBER", "5432") +DB_NAME = env("SUPERSET_DATABASE_NAME") +DB_PARAMS = "?sslmode=require" if is_boolean_yes(env("SUPERSET_DATABASE_USE_SSL", "no")) else "" +DB_AUTH = f"{DB_USER}:{DB_PASSWORD}@" if DB_PASSWORD else "" +SQLALCHEMY_DATABASE_URI = f"{DB_DIALECT}://{DB_AUTH}{DB_HOST}:{DB_PORT}/{DB_NAME}{DB_PARAMS}" +SQLALCHEMY_TRACK_MODIFICATIONS = True + +# Optional functionality +# https://github.com/apache/superset/blob/142b2cc42543876c607c4a258dfac018da1f1d81/superset/config.py#L539 +### Role-based access control for dashboards +### Enables Alerts and Reports functionality +### Enable embedded Superset functionality +FEATURE_FLAGS = {'DASHBOARD_RBAC': True, + 'ALERT_REPORTS': True, + 'EMBEDDED_SUPERSET': True} + + +# After this : volatile config to try to get guest access tokens +GUEST_ROLE_NAME = "Gamma" +GUEST_TOKEN_JWT_AUDIENCE = "superset" +GUEST_TOKEN_JWT_SECRET = "superset-warp.api.cosmotech.com/@cosmotech_we_use_superset" +# Flask-WTF flag for CSRF +WTF_CSRF_ENABLED = False + +# Cross Origin Config +# default value +#ENABLE_CORS = True +CORS_OPTIONS = { + 'supports_credentials': True, + 'allow_headers': ['*'], + 'resources':['*'], + 'origins': ["https://superset-warp.api.cosmotech.com"] +} + +# Talisman Config +TALISMAN_ENABLED = True +TALISMAN_CONFIG = { + "content_security_policy": { + "frame-ancestors": ["https://superset-warp.api.cosmotech.com"] + }, + "force_https": False, + "force_https_permanent": False, + "frame_options": "ALLOWFROM", + "frame_options_allow_from": "*" +} \ No newline at end of file diff --git a/modules/chart_superset/main.tf b/modules/chart_superset/main.tf new file mode 100644 index 0000000..d6c6a9e --- /dev/null +++ b/modules/chart_superset/main.tf @@ -0,0 +1,77 @@ +locals { + # superset_secret_name = "superset" + superset_configmap_name = "superset-config-map" + + chart_values = { + NAMESPACE = var.namespace + CLUSTER_DOMAIN = var.cluster_domain + # SECRET_NAME = local.superset_secret_name + CONFIGMAP_NAME = local.superset_configmap_name + # SUPERSET_SECRET_KEY = random_password.superset_secret_key.result + } +} + + +# resource "random_password" "admin" { +# length = 40 +# special = false +# } + +# # resource "random_password" "redis" { +# # length = 40 +# # special = false +# # } + + +# resource "random_password" "superset_secret_key" { +# length = 40 +# special = false +# } + + +# resource "kubernetes_secret" "superset_config" { +# metadata { +# name = local.superset_secret_name +# namespace = var.namespace +# } + +# data = { +# superset-admin-user = "admin" +# superset-password = random_password.admin.result +# superset-secret-key = random_password.superset_secret_key.result +# # redis-password = random_password.redis.result +# } + +# type = "Opaque" +# } + + + +resource "kubernetes_config_map" "superset_config_map" { + metadata { + name = local.superset_configmap_name + namespace = var.namespace + } + + data = { + "superset_config.py" = templatefile("${path.module}/kube_objects/superset_config.py", local.chart_values) + } +} + + +resource "helm_release" "superset" { + name = "superset" + repository = var.helm_repo + chart = var.helm_chart + version = var.helm_chart_version + namespace = var.namespace + + values = [ + templatefile("${path.module}/values.yaml", local.chart_values) + ] + + depends_on = [ + # kubernetes_secret.superset_config, + kubernetes_config_map.superset_config_map + ] +} diff --git a/modules/chart_superset/values.yaml b/modules/chart_superset/values.yaml new file mode 100644 index 0000000..c07e1a2 --- /dev/null +++ b/modules/chart_superset/values.yaml @@ -0,0 +1,687 @@ +# auth: + # email: "" + # existingSecret: SECRET_NAME}" + # password: "" + # secretKey: "" + # username: admin +beat: + affinity: {} + args: [] + automountServiceAccountToken: false + command: [] + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + enabled: true + privileged: false + readOnlyRootFilesystem: false + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + customLivenessProbe: {} + customReadinessProbe: {} + customStartupProbe: {} + deploymentAnnotations: {} + enabled: false + extraEnvVars: [] + extraEnvVarsCM: "" + extraEnvVarsSecret: "" + extraVolumeMounts: [] + extraVolumes: [] + hostAliases: [] + initContainers: [] + lifecycleHooks: {} + livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + timeoutSeconds: 1 + networkPolicy: + allowExternal: true + allowExternalEgress: true + enabled: true + extraEgress: [] + extraIngress: [] + nodeAffinityPreset: + key: "" + type: "" + values: [] + nodeSelector: {} + pdb: + create: false + maxUnavailable: "" + minAvailable: 1 + podAffinityPreset: "" + podAnnotations: {} + podAntiAffinityPreset: soft + podLabels: {} + podSecurityContext: + enabled: true + fsGroup: 1001 + fsGroupChangePolicy: Always + supplementalGroups: [] + sysctls: [] + priorityClassName: "" + readinessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + resourcesPreset: small + schedulerName: "" + sidecars: [] + startupProbe: + enabled: false + failureThreshold: 60 + initialDelaySeconds: 15 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + terminationGracePeriodSeconds: "" + tolerations: + - effect: NoSchedule + key: vendor + operator: Equal + value: cosmotech + topologySpreadConstraints: [] + updateStrategy: + type: RollingUpdate +clusterDomain: cluster.local +commonAnnotations: {} +commonLabels: {} +config: "" +defaultInitContainers: + waitForDB: + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + enabled: true + privileged: false + readOnlyRootFilesystem: false + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + enabled: true + resources: {} + resourcesPreset: nano + waitForRedis: + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + enabled: true + privileged: false + readOnlyRootFilesystem: false + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + enabled: true + resources: {} + resourcesPreset: nano +diagnosticMode: + args: + - infinity + command: + - sleep + enabled: false +existingConfigmap: "${CONFIGMAP_NAME}" +extraDeploy: [] +flower: + affinity: {} + args: [] + auth: + enabled: true + existingSecret: "" + password: "" + username: user + automountServiceAccountToken: false + autoscaling: + hpa: + enabled: false + maxReplicas: "" + minReplicas: "" + targetCPU: "" + targetMemory: "" + vpa: + annotations: {} + controlledResources: [] + enabled: false + maxAllowed: {} + minAllowed: {} + updatePolicy: + updateMode: Auto + command: [] + containerPorts: + flower: 5555 + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + enabled: true + privileged: false + readOnlyRootFilesystem: false + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + customLivenessProbe: {} + customReadinessProbe: {} + customStartupProbe: {} + deploymentAnnotations: {} + enabled: false + extraContainerPorts: [] + extraEnvVars: + - name: SUPERSET_WEBSERVER_BASEURL + value: "https://${CLUSTER_DOMAIN}" + - name: SUPERSET_WEBSERVER_PATH_PREFIX + value: / + extraEnvVarsCM: "" + extraEnvVarsSecret: "" + extraVolumeMounts: [] + extraVolumes: [] + hostAliases: [] + initContainers: [] + lifecycleHooks: {} + livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 15 + successThreshold: 1 + timeoutSeconds: 1 + networkPolicy: + addExternalClientAccess: true + allowExternal: true + allowExternalEgress: true + enabled: true + extraEgress: [] + extraIngress: [] + ingressNSMatchLabels: {} + ingressNSPodMatchLabels: {} + ingressPodMatchLabels: {} + nodeAffinityPreset: + key: "" + type: "" + values: [] + nodeSelector: {} + pdb: + create: false + maxUnavailable: "" + minAvailable: 1 + podAffinityPreset: "" + podAnnotations: {} + podAntiAffinityPreset: soft + podLabels: {} + podSecurityContext: + enabled: true + fsGroup: 1001 + fsGroupChangePolicy: Always + supplementalGroups: [] + sysctls: [] + priorityClassName: "" + readinessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + timeoutSeconds: 1 + replicaCount: 1 + resources: {} + resourcesPreset: small + schedulerName: "" + service: + annotations: {} + clusterIP: "" + externalTrafficPolicy: Cluster + extraPorts: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + nodePorts: + flower: "" + ports: + flower: 5555 + sessionAffinity: None + sessionAffinityConfig: {} + type: LoadBalancer + sidecars: [] + startupProbe: + enabled: false + failureThreshold: 60 + initialDelaySeconds: 15 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + terminationGracePeriodSeconds: "" + tolerations: + - effect: NoSchedule + key: vendor + operator: Equal + value: cosmotech + topologySpreadConstraints: [] + updateStrategy: + type: RollingUpdate +fullnameOverride: "" +global: + compatibility: + openshift: + adaptSecurityContext: auto + imagePullSecrets: [] + imageRegistry: "" + storageClass: "" + security: + allowInsecureImages: true +image: + debug: true + digest: "" + pullPolicy: Always + pullSecrets: [] + registry: ghcr.io + repository: cosmo-tech/superset + tag: latest +ingress: + annotations: + kubernetes.io/ingress.class: nginx + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/hsts: "false" +# nginx.ingress.kubernetes.io/rewrite-target: /$2/ +# nginx.ingress.kubernetes.io/use-regex: "true" + apiVersion: "" + enabled: true + extraHosts: [] + extraPaths: [] + extraRules: [] + extraTls: + - hosts: + - "${CLUSTER_DOMAIN}" + secretName: letsencrypt-prod + path: / #superset(/|$)(.*) + pathType: Prefix #ImplementationSpecific + hostname: "${CLUSTER_DOMAIN}" + ingressClassName: nginx + secrets: [] + selfSigned: false + tls: [] +init: + args: [] + automountServiceAccountToken: false + backoffLimit: 10 + command: [] + containerSecurityContext: + allowPrivilegeEscalation: false + enabled: true + privileged: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + enabled: true + extraEnvVars: [] + extraEnvVarsCM: "" + extraEnvVarsSecret: "" + extraVolumes: [] + extraVolumeMounts: [] + hostAliases: [] + initContainers: [] + jobAnnotations: {} + networkPolicy: + allowExternal: true + allowExternalEgress: true + enabled: true + extraEgress: [] + extraIngress: [] + podAnnotations: {} + podLabels: {} + podSecurityContext: + enabled: true + fsGroup: 1001 + fsGroupChangePolicy: Always + supplementalGroups: [] + sysctls: [] + resources: {} + resourcesPreset: medium + sidecars: [] +kubeVersion: "" +loadExamples: false +nameOverride: "" +namespaceOverride: "" +postgresql: + image: + repository: bitnamilegacy/postgresql + architecture: standalone + auth: + database: bitnami_superset + enablePostgresUser: true + existingSecret: "" + password: "" + username: bn_superset + enabled: true + primary: + resources: {} + resourcesPreset: nano + service: + ports: + postgresql: 5432 +redis: + image: + repository: bitnamilegacy/redis + architecture: standalone + auth: + enabled: true + existingSecret: "" + password: "" + enabled: true + master: + resources: {} + resourcesPreset: nano + service: + ports: + redis: 6379 +serviceAccount: + annotations: {} + automountServiceAccountToken: true + create: true + name: "" +usePasswordFiles: true +web: + affinity: {} + args: [] + automountServiceAccountToken: false + autoscaling: + hpa: + enabled: false + maxReplicas: "" + minReplicas: "" + targetCPU: "" + targetMemory: "" + vpa: + annotations: {} + controlledResources: [] + enabled: false + maxAllowed: {} + minAllowed: {} + updatePolicy: + updateMode: Auto + command: [] + containerPorts: + http: 8080 + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + enabled: true + privileged: false + readOnlyRootFilesystem: false + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + customLivenessProbe: {} + customReadinessProbe: {} + customStartupProbe: {} + deploymentAnnotations: {} + extraContainerPorts: [] + extraEnvVars: [] + extraEnvVarsCM: "" + extraEnvVarsSecret: "" + extraVolumeMounts: [] + extraVolumes: [] + hostAliases: [] + initContainers: [] + lifecycleHooks: {} + livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + timeoutSeconds: 1 + networkPolicy: + addExternalClientAccess: true + allowExternal: true + allowExternalEgress: true + enabled: true + extraEgress: [] + extraIngress: [] + ingressNSMatchLabels: {} + ingressNSPodMatchLabels: {} + ingressPodMatchLabels: {} + nodeAffinityPreset: + key: "" + type: "" + values: [] + nodeSelector: + cosmotech.com/tier: services + pdb: + create: false + maxUnavailable: "" + minAvailable: 1 + podAffinityPreset: "" + podAnnotations: {} + podAntiAffinityPreset: soft + podLabels: {} + podSecurityContext: + enabled: true + fsGroup: 1001 + fsGroupChangePolicy: Always + supplementalGroups: [] + sysctls: [] + priorityClassName: "" + readinessProbe: + enabled: false + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + timeoutSeconds: 1 + replicaCount: 1 + resources: {} + resourcesPreset: small + schedulerName: "" + service: + annotations: {} + clusterIP: "" + externalTrafficPolicy: Cluster + extraPorts: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + nodePorts: + http: "" + ports: + http: 80 + sessionAffinity: None + sessionAffinityConfig: {} + type: LoadBalancer + sidecars: [] + startupProbe: + enabled: false + failureThreshold: 60 + initialDelaySeconds: 15 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + terminationGracePeriodSeconds: "" + tolerations: + - effect: NoSchedule + key: vendor + operator: Equal + value: cosmotech + topologySpreadConstraints: [] + updateStrategy: + type: RollingUpdate + waitForExamples: + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + enabled: true + privileged: false + readOnlyRootFilesystem: false + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + enabled: false + resources: {} + resourcesPreset: nano +worker: + affinity: {} + args: [] + automountServiceAccountToken: false + autoscaling: + hpa: + enabled: false + maxReplicas: "" + minReplicas: "" + targetCPU: "" + targetMemory: "" + vpa: + annotations: {} + controlledResources: [] + enabled: false + maxAllowed: {} + minAllowed: {} + updatePolicy: + updateMode: Auto + command: + - celery + - worker + - --concurrency=2 + - --loglevel=INFO + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + enabled: true + privileged: false + readOnlyRootFilesystem: false + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + customLivenessProbe: {} + customReadinessProbe: {} + customStartupProbe: {} + deploymentAnnotations: {} + extraEnvVars: + # - name: REDIS_PASSWORD + # valueFrom: + # secretKeyRef: + # key: redis-password + # name: SECRET_NAME + - name: CELERY_BROKER_URL + value: redis://default:$(REDIS_PASSWORD)@superset-redis-master:6379/0 + - name: CELERY_RESULT_BACKEND + value: redis://default:$(REDIS_PASSWORD)@superset-redis-master:6379/0 + extraEnvVarsCM: "" + extraEnvVarsSecret: "" + extraVolumeMounts: [] + extraVolumes: [] + hostAliases: [] + initContainers: [] + lifecycleHooks: {} + livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 60 + successThreshold: 1 + timeoutSeconds: 30 + networkPolicy: + allowExternal: true + allowExternalEgress: true + enabled: true + extraEgress: [] + extraIngress: [] + nodeAffinityPreset: + key: "" + type: "" + values: [] + nodeSelector: + cosmotech.com/tier: services + pdb: + create: false + maxUnavailable: "" + minAvailable: 1 + podAffinityPreset: "" + podAnnotations: {} + podAntiAffinityPreset: soft + podLabels: {} + podSecurityContext: + enabled: true + fsGroup: 1001 + fsGroupChangePolicy: Always + supplementalGroups: [] + sysctls: [] + priorityClassName: "" + readinessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 60 + successThreshold: 1 + timeoutSeconds: 30 + replicaCount: 1 + resources: + limits: + cpu: 4 + memory: 6Gi + requests: + cpu: 3 + memory: 4Gi + resourcesPreset: large + schedulerName: "" + sidecars: [] + startupProbe: + enabled: false + failureThreshold: 60 + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + timeoutSeconds: 30 + terminationGracePeriodSeconds: "" + tolerations: + - effect: NoSchedule + key: vendor + operator: Equal + value: cosmotech + topologySpreadConstraints: [] + updateStrategy: + type: RollingUpdate \ No newline at end of file diff --git a/modules/chart_superset/variables.tf b/modules/chart_superset/variables.tf new file mode 100644 index 0000000..fdc61e8 --- /dev/null +++ b/modules/chart_superset/variables.tf @@ -0,0 +1,20 @@ +variable "namespace" { + type = string +} + +variable "cluster_domain" { + type = string +} + +variable "helm_repo" { + type = string +} + +variable "helm_chart" { + type = string +} + +variable "helm_chart_version" { + type = string +} + From 43d9bb915909657e7a8c2c02e054c273472e04fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Reynard?= Date: Mon, 16 Feb 2026 19:07:55 +0100 Subject: [PATCH 2/2] Add Terraform module for Superset deployment with enhanced configuration --- modules/chart_superset/README.md | 97 +++++++++++ .../kube_objects/superset_config.py | 107 +++++++----- modules/chart_superset/main.tf | 158 ++++++++++++++---- modules/chart_superset/values.yaml | 117 ++++++++++--- 4 files changed, 383 insertions(+), 96 deletions(-) create mode 100644 modules/chart_superset/README.md diff --git a/modules/chart_superset/README.md b/modules/chart_superset/README.md new file mode 100644 index 0000000..cc5f382 --- /dev/null +++ b/modules/chart_superset/README.md @@ -0,0 +1,97 @@ +# Terraform module: `chart_superset` + +This module installs **Apache Superset** on a Kubernetes cluster using the **Helm** provider, and provisions the Kubernetes objects Superset needs to start (notably secrets and a config map). + +It is designed to be consumed from a “cluster bootstrap” Terraform stack (like the root of this repository), once your Kubernetes cluster is reachable by Terraform. + +--- + +## What this module does + +When applied, the module will: + +- Generate random credentials/keys used by Superset and its dependencies +- Create Kubernetes `Secret` objects for: + - Superset admin/secret key material + - Redis authentication + - PostgreSQL authentication + - A “guest token” secret used for embedded/dashboard access scenarios + - A “Superset secret key” secret used for signing the session cookie (Flask App Builder configuration) +- Create a Kubernetes `ConfigMap` containing a `superset_config.py` (templated for your cluster domain) +- Deploy the Superset Helm chart (from the repository/chart/version you provide), using a templated `values.yaml` + +--- + +## Requirements + +- A working Kubernetes cluster +- Terraform providers configured in the calling stack: + - `hashicorp/helm` + - `hashicorp/kubernetes` + - `hashicorp/random` +--- + +## Inputs + +| Name | Type | Required | Description | +|------|------|----------|-------------| +| `namespace` | `string` | yes | Kubernetes namespace where Superset will be installed. | +| `cluster_domain` | `string` | yes | DNS name used to build Superset URLs/ingress hostnames (passed into chart templating). | +| `helm_repo` | `string` | yes | Helm repository URL (e.g. `https://charts.bitnami.com/bitnami`). | +| `helm_chart` | `string` | yes | Helm chart name (e.g. `superset`). | +| `helm_chart_version` | `string` | yes | Helm chart version to install. | + +--- + +## Outputs + +This module does not declare explicit Terraform outputs. + +If you need the generated credentials, consider adding outputs in your own fork (be cautious: outputting secrets may expose them in state/CI logs). + +--- + +## Usage + +Please refer to the "How to" section in the root README. + +## Oauth providers configuration + +A list of oauth providers can be defined in a dedicated config map name `superset-oauth-providers`. +The `superset-oauth-providers` configmap should have a data entry named `oauth-providers` containing a JSON array with all desired oauth providers. +Please refer to Superset documentation for more information about oauth providers configuration. +https://superset.apache.org/docs/configuration/configuring-superset/#custom-oauth2-configuration + +Example: +```json +[ + { + "name": "oauth_provider1", + "icon": "fa-key", + "token_key": "access_token", + "remote_app": { + "client_id": "", + "client_secret": "", + "client_kwargs": {"scope": "openid profile email"}, + "server_metadata_url": "https:///.well-known/openid-configuration" + } + }, + { + "name": "oauth_provider2", + "icon": "fa-key", + "token_key": "access_token", + "remote_app": { + "client_id": "", + "client_secret": "", + "client_kwargs": {"scope": "openid profile email"}, + "server_metadata_url": "https:///.well-known/openid-configuration" + } + } +] +``` + +### Notes: + +If you want to add a new oauth provider, you need to: +- Add the new provider in the `superset-oauth-providers` configmap +- Restart the `superset-web` pod \ No newline at end of file diff --git a/modules/chart_superset/kube_objects/superset_config.py b/modules/chart_superset/kube_objects/superset_config.py index 1ef7cae..5947027 100644 --- a/modules/chart_superset/kube_objects/superset_config.py +++ b/modules/chart_superset/kube_objects/superset_config.py @@ -1,8 +1,10 @@ import os import logging +import json from logging import getLogger from flask_appbuilder.security.manager import AUTH_OAUTH from superset import SupersetSecurityManager +from flask_caching.backends.rediscache import RedisCache logger = logging.getLogger() log = getLogger(__name__) @@ -27,6 +29,7 @@ class KeycloakSecurity(SupersetSecurityManager): """ def oauth_user_info(self, provider, resp=None): + log.debug("Oauth2 provider: '{0}'.".format(provider)) log.debug("Keycloak response received : {0}".format(resp)) id_token = resp["id_token"] log.debug("ID Token: %s", id_token) @@ -52,8 +55,8 @@ def oauth_user_info(self, provider, resp=None): # End custom_sso_security_manager ## URLs config -ROOT_URL = 'https://superset-warp.api.cosmotech.com' -LOGOUT_REDIRECT_URL = 'https://superset-warp.api.cosmotech.com' +ROOT_URL = 'https://superset-${CLUSTER_DOMAIN}' +LOGOUT_REDIRECT_URL = 'https://superset-${CLUSTER_DOMAIN}' # Auth config AUTH_USER_REGISTRATION = True @@ -66,39 +69,7 @@ def oauth_user_info(self, provider, resp=None): } AUTH_ROLES_SYNC_AT_LOGIN = True -OAUTH_PROVIDERS = [ - { - "name": "sphinx", - "icon": "fa-key", - "token_key": "access_token", - "remote_app": { - "client_id": "cosmotech-superset-client", - "client_secret": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "client_kwargs": {"scope": "openid profile email"}, - "server_metadata_url": "https://warp.api.cosmotech.com/keycloak/realms/sphinx/.well-known/openid-configuration" - } - }, - { - "name": "eng-api-ci", - "icon": "fa-key", - "token_key": "access_token", - "remote_app": { - "client_id": "cosmotech-superset-client", - "client_secret": "xxxxxxxxxxxxxxxxxxxxxxxxx", - "client_kwargs": {"scope": "openid profile email"}, - "server_metadata_url": "https://warp.api.cosmotech.com/keycloak/realms/eng-api-ci/.well-known/openid-configuration" - } - } -] - -# Flask App Builder configuration -# Your App secret key will be used for securely signing the session cookie -# and encrypting sensitive information on the database -# Make sure you are changing this key for your deployment with a strong key. -# Alternatively you can set it with `SUPERSET_SECRET_KEY` environment variable. -# You MUST set this for production environments or the server will refuse -# to start and you will see an error in the logs accordingly. -# SECRET_KEY = 'xxxxxxxxxxxxxxxxxxxxxxx' +OAUTH_PROVIDERS = json.loads(env("SUPERSET_OAUTH_PROVIDERS", "[]")) # Database config ## @@ -113,6 +84,66 @@ def oauth_user_info(self, provider, resp=None): SQLALCHEMY_DATABASE_URI = f"{DB_DIALECT}://{DB_AUTH}{DB_HOST}:{DB_PORT}/{DB_NAME}{DB_PARAMS}" SQLALCHEMY_TRACK_MODIFICATIONS = True +## Redis settings +## +REDIS_HOST = env("REDIS_HOST", "redis") +REDIS_PORT = env("REDIS_PORT_NUMBER", "6379") +REDIS_CELERY_DB = env("REDIS_CELERY_DB", "0") +REDIS_DB = env("REDIS_DB", "1") +REDIS_PASSWORD = env("REDIS_PASSWORD") +REDIS_USER = env("REDIS_USER", "") +REDIS_TLS_ENABLED = env("REDIS_TLS_ENABLED", False) +REDIS_SSL_CERT_REQS = env("REDIS_SSL_CERT_REQS") +REDIS_URL_PARAMS = f"ssl_cert_reqs={REDIS_SSL_CERT_REQS}" if REDIS_SSL_CERT_REQS else "" +REDIS_AUTH = f"{REDIS_USER}:{REDIS_PASSWORD}@" if REDIS_PASSWORD else "" +REDIS_BASE_URL = f"redis://{REDIS_AUTH}{REDIS_HOST}:{REDIS_PORT}" +# Redis URLs +REDIS_CELERY_URL = f"{REDIS_BASE_URL}/{REDIS_CELERY_DB}{REDIS_URL_PARAMS}" +REDIS_CACHE_URL = f"{REDIS_BASE_URL}/{REDIS_DB}{REDIS_URL_PARAMS}" + +## Cache config +## +CACHE_CONFIG = { + "CACHE_TYPE": "RedisCache", + "CACHE_DEFAULT_TIMEOUT": 300, + "CACHE_KEY_PREFIX": "superset_", + "CACHE_REDIS_URL": REDIS_CACHE_URL, +} +DATA_CACHE_CONFIG = CACHE_CONFIG + +## Results backend +## +RESULTS_BACKEND = RedisCache( + host=REDIS_HOST, + password=REDIS_PASSWORD, + port=REDIS_PORT, + key_prefix='superset_results', + ssl=REDIS_TLS_ENABLED, + ssl_cert_reqs=REDIS_SSL_CERT_REQS, +) + +## Celery config +## +class CeleryConfig: + imports = ("superset.sql_lab", ) + broker_url = REDIS_CELERY_URL + result_backend = REDIS_CELERY_URL + +CELERY_CONFIG = CeleryConfig + +## Load user extended config +## +try: + import superset_config_docker + from superset_config_docker import * # noqa + + logger.info( + f"Loaded your configuration from " f"[{superset_config_docker.__file__}]" + ) +except ImportError: + logger.info("Using default settings") + + # Optional functionality # https://github.com/apache/superset/blob/142b2cc42543876c607c4a258dfac018da1f1d81/superset/config.py#L539 ### Role-based access control for dashboards @@ -126,7 +157,7 @@ def oauth_user_info(self, provider, resp=None): # After this : volatile config to try to get guest access tokens GUEST_ROLE_NAME = "Gamma" GUEST_TOKEN_JWT_AUDIENCE = "superset" -GUEST_TOKEN_JWT_SECRET = "superset-warp.api.cosmotech.com/@cosmotech_we_use_superset" +GUEST_TOKEN_JWT_SECRET = "${SUPERSET_GUEST_TOKEN}" # Flask-WTF flag for CSRF WTF_CSRF_ENABLED = False @@ -137,14 +168,14 @@ def oauth_user_info(self, provider, resp=None): 'supports_credentials': True, 'allow_headers': ['*'], 'resources':['*'], - 'origins': ["https://superset-warp.api.cosmotech.com"] + 'origins': ["https://superset-${CLUSTER_DOMAIN}"] } # Talisman Config TALISMAN_ENABLED = True TALISMAN_CONFIG = { "content_security_policy": { - "frame-ancestors": ["https://superset-warp.api.cosmotech.com"] + "frame-ancestors": ["https://superset-${CLUSTER_DOMAIN}"] }, "force_https": False, "force_https_permanent": False, diff --git a/modules/chart_superset/main.tf b/modules/chart_superset/main.tf index d6c6a9e..7af195d 100644 --- a/modules/chart_superset/main.tf +++ b/modules/chart_superset/main.tf @@ -1,53 +1,146 @@ locals { - # superset_secret_name = "superset" - superset_configmap_name = "superset-config-map" + superset_secret_name = "superset" + superset_redis_secret_name = "superset-redis" + superset_postgresql_secret_name = "superset-postgresql" + superset_guest_token_secret_name = "superset-guest-token" + superset_guest_token = random_password.superset_guest_token_secret.result + superset_secret_key_name = "superset-secret-key" + superset_configmap_name = "superset-config" + superset_oauth_providers_configmap_name = "superset-oauth-providers" chart_values = { NAMESPACE = var.namespace CLUSTER_DOMAIN = var.cluster_domain - # SECRET_NAME = local.superset_secret_name + SUPERSET_SECRET_NAME = local.superset_secret_name + SUPERSET_REDIS_SECRET_NAME = local.superset_redis_secret_name + SUPERSET_POSTGRESQL_SECRET_NAME = local.superset_postgresql_secret_name CONFIGMAP_NAME = local.superset_configmap_name - # SUPERSET_SECRET_KEY = random_password.superset_secret_key.result + OAUTH_PROVIDERS_CONFIGMAP_NAME = local.superset_oauth_providers_configmap_name + SUPERSET_GUEST_TOKEN = local.superset_guest_token + SUPERSET_SECRET_KEY_NAME = local.superset_secret_key_name } } -# resource "random_password" "admin" { -# length = 40 -# special = false -# } +## Secret Key for signing the session cookie (Flask App Builder configuration) +resource "random_password" "superset_secret_key_value" { + length = 40 + special = false +} + +resource "kubernetes_secret" "superset_secret_key_secret" { + metadata { + name = local.superset_secret_key_name + namespace = var.namespace + } + + data = { + secret-key = random_password.superset_secret_key_value.result + } + + type = "Opaque" +} + +## End of Secret Key for signing the session cookie (Flask App Builder configuration -# # resource "random_password" "redis" { -# # length = 40 -# # special = false -# # } +## Guest token for dashboard embedding +resource "random_password" "superset_guest_token_secret" { + length = 40 + special = false +} +resource "kubernetes_secret" "superset_guest_token" { + metadata { + name = local.superset_guest_token_secret_name + namespace = var.namespace + } -# resource "random_password" "superset_secret_key" { -# length = 40 -# special = false -# } + data = { + guest-token = local.superset_guest_token + } + type = "Opaque" +} -# resource "kubernetes_secret" "superset_config" { -# metadata { -# name = local.superset_secret_name -# namespace = var.namespace -# } +## End of Guest token -# data = { -# superset-admin-user = "admin" -# superset-password = random_password.admin.result -# superset-secret-key = random_password.superset_secret_key.result -# # redis-password = random_password.redis.result -# } +## Superset, Postgresql, Redis secrets -# type = "Opaque" -# } +## Superset + resource "random_password" "superset_password" { + length = 40 + special = false + } +resource "random_password" "superset_secret_key" { + length = 40 + special = false +} + resource "kubernetes_secret" "superset_secret" { + metadata { + name = local.superset_secret_name + namespace = var.namespace + } + + data = { + superset-password = random_password.superset_password.result + superset-secret-key = random_password.superset_secret_key.result + } + + type = "Opaque" + } +## End of Superset + +## Superset <-> Postgresql +resource "random_password" "superset_postgresql_password" { + length = 40 + special = false +} + +resource "random_password" "superset_user_postgresql_password" { + length = 40 + special = false +} -resource "kubernetes_config_map" "superset_config_map" { +resource "kubernetes_secret" "superset_postgresql" { + metadata { + name = local.superset_postgresql_secret_name + namespace = var.namespace + } + + data = { + password = random_password.superset_postgresql_password.result + postgresql-password = random_password.superset_user_postgresql_password.result + } + + type = "Opaque" +} +## End of Superset <-> Postgresql + +## Superset <-> Redis +resource "random_password" "superset_redis_password" { + length = 40 + special = false +} + +resource "kubernetes_secret" "superset_redis" { + metadata { + name = local.superset_redis_secret_name + namespace = var.namespace + } + + data = { + redis-password = random_password.superset_redis_password.result + } + + type = "Opaque" +} +## End of Superset <-> Redis +## End of Superset, Postgresql, Redis secrets + +## ConfigMap with superset_config.py +resource "kubernetes_config_map" "superset_config_map"{ metadata { name = local.superset_configmap_name namespace = var.namespace @@ -57,8 +150,9 @@ resource "kubernetes_config_map" "superset_config_map" { "superset_config.py" = templatefile("${path.module}/kube_objects/superset_config.py", local.chart_values) } } +## End of ConfigMap with superset_config.py - +## Superset Helm Chart resource "helm_release" "superset" { name = "superset" repository = var.helm_repo @@ -71,7 +165,7 @@ resource "helm_release" "superset" { ] depends_on = [ - # kubernetes_secret.superset_config, kubernetes_config_map.superset_config_map ] } +## End of Superset Helm Chart diff --git a/modules/chart_superset/values.yaml b/modules/chart_superset/values.yaml index c07e1a2..bb368ae 100644 --- a/modules/chart_superset/values.yaml +++ b/modules/chart_superset/values.yaml @@ -1,9 +1,6 @@ -# auth: - # email: "" - # existingSecret: SECRET_NAME}" - # password: "" - # secretKey: "" - # username: admin +auth: + existingSecret: "${SUPERSET_SECRET_NAME}" + username: admin beat: affinity: {} args: [] @@ -28,7 +25,18 @@ beat: customStartupProbe: {} deploymentAnnotations: {} enabled: false - extraEnvVars: [] + extraEnvVars: + - name: SUPERSET_OAUTH_PROVIDERS + valueFrom: + configMapKeyRef: + key: oauth-providers + name: ${OAUTH_PROVIDERS_CONFIGMAP_NAME} + optional: true + - name: SUPERSET_SECRET_KEY + valueFrom: + secretKeyRef: + key: secret-key + name: ${SUPERSET_SECRET_KEY_NAME} extraEnvVarsCM: "" extraEnvVarsSecret: "" extraVolumeMounts: [] @@ -53,7 +61,8 @@ beat: key: "" type: "" values: [] - nodeSelector: {} + nodeSelector: + "cosmotech.com/tier": "services" pdb: create: false maxUnavailable: "" @@ -150,8 +159,7 @@ flower: args: [] auth: enabled: true - existingSecret: "" - password: "" + existingSecret: "${SUPERSET_SECRET_NAME}" username: user automountServiceAccountToken: false autoscaling: @@ -197,6 +205,17 @@ flower: value: "https://${CLUSTER_DOMAIN}" - name: SUPERSET_WEBSERVER_PATH_PREFIX value: / + - name: SUPERSET_OAUTH_PROVIDERS + valueFrom: + configMapKeyRef: + key: oauth-providers + name: ${OAUTH_PROVIDERS_CONFIGMAP_NAME} + optional: true + - name: SUPERSET_SECRET_KEY + valueFrom: + secretKeyRef: + key: secret-key + name: ${SUPERSET_SECRET_KEY_NAME} extraEnvVarsCM: "" extraEnvVarsSecret: "" extraVolumeMounts: [] @@ -225,7 +244,8 @@ flower: key: "" type: "" values: [] - nodeSelector: {} + nodeSelector: + "cosmotech.com/tier": "services" pdb: create: false maxUnavailable: "" @@ -306,8 +326,6 @@ ingress: kubernetes.io/ingress.class: nginx cert-manager.io/cluster-issuer: letsencrypt-prod nginx.ingress.kubernetes.io/hsts: "false" -# nginx.ingress.kubernetes.io/rewrite-target: /$2/ -# nginx.ingress.kubernetes.io/use-regex: "true" apiVersion: "" enabled: true extraHosts: [] @@ -317,8 +335,8 @@ ingress: - hosts: - "${CLUSTER_DOMAIN}" secretName: letsencrypt-prod - path: / #superset(/|$)(.*) - pathType: Prefix #ImplementationSpecific + path: / + pathType: Prefix hostname: "${CLUSTER_DOMAIN}" ingressClassName: nginx secrets: [] @@ -344,7 +362,18 @@ init: seccompProfile: type: RuntimeDefault enabled: true - extraEnvVars: [] + extraEnvVars: + - name: SUPERSET_OAUTH_PROVIDERS + valueFrom: + configMapKeyRef: + key: oauth-providers + name: ${OAUTH_PROVIDERS_CONFIGMAP_NAME} + optional: true + - name: SUPERSET_SECRET_KEY + valueFrom: + secretKeyRef: + key: secret-key + name: ${SUPERSET_SECRET_KEY_NAME} extraEnvVarsCM: "" extraEnvVarsSecret: "" extraVolumes: [] @@ -380,11 +409,17 @@ postgresql: auth: database: bitnami_superset enablePostgresUser: true - existingSecret: "" - password: "" + existingSecret: "${SUPERSET_POSTGRESQL_SECRET_NAME}" username: bn_superset enabled: true primary: + nodeSelector: + "cosmotech.com/tier": "db" + tolerations: + - effect: NoSchedule + key: vendor + operator: Equal + value: cosmotech resources: {} resourcesPreset: nano service: @@ -396,10 +431,18 @@ redis: architecture: standalone auth: enabled: true - existingSecret: "" - password: "" + existingSecret: "${SUPERSET_REDIS_SECRET_NAME}" + username: default + #password: "" enabled: true master: + nodeSelector: + "cosmotech.com/tier": "db" + tolerations: + - effect: NoSchedule + key: vendor + operator: Equal + value: cosmotech resources: {} resourcesPreset: nano service: @@ -452,7 +495,18 @@ web: customStartupProbe: {} deploymentAnnotations: {} extraContainerPorts: [] - extraEnvVars: [] + extraEnvVars: + - name: SUPERSET_OAUTH_PROVIDERS + valueFrom: + configMapKeyRef: + key: oauth-providers + name: ${OAUTH_PROVIDERS_CONFIGMAP_NAME} + optional: true + - name: SUPERSET_SECRET_KEY + valueFrom: + secretKeyRef: + key: secret-key + name: ${SUPERSET_SECRET_KEY_NAME} extraEnvVarsCM: "" extraEnvVarsSecret: "" extraVolumeMounts: [] @@ -601,15 +655,26 @@ worker: customStartupProbe: {} deploymentAnnotations: {} extraEnvVars: - # - name: REDIS_PASSWORD - # valueFrom: - # secretKeyRef: - # key: redis-password - # name: SECRET_NAME + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + key: redis-password + name: ${SUPERSET_REDIS_SECRET_NAME} - name: CELERY_BROKER_URL value: redis://default:$(REDIS_PASSWORD)@superset-redis-master:6379/0 - name: CELERY_RESULT_BACKEND value: redis://default:$(REDIS_PASSWORD)@superset-redis-master:6379/0 + - name: SUPERSET_OAUTH_PROVIDERS + valueFrom: + configMapKeyRef: + key: oauth-providers + name: ${OAUTH_PROVIDERS_CONFIGMAP_NAME} + optional: true + - name: SUPERSET_SECRET_KEY + valueFrom: + secretKeyRef: + key: secret-key + name: ${SUPERSET_SECRET_KEY_NAME} extraEnvVarsCM: "" extraEnvVarsSecret: "" extraVolumeMounts: []