diff --git a/Babylon/commands/api/dataset.py b/Babylon/commands/api/dataset.py index a03641cc..789d6887 100644 --- a/Babylon/commands/api/dataset.py +++ b/Babylon/commands/api/dataset.py @@ -94,9 +94,7 @@ def list_datasets(config: dict, keycloak_token: str, organization_id: str, works @option("--oid", "organization_id", required=True, type=str, help="Organization ID") @option("--wid", "workspace_id", required=True, type=str, help="Workspace ID") @option("--did", "dataset_id", required=True, type=str, help="Dataset ID") -def delete( - config: dict, keycloak_token: str, organization_id: str, workspace_id: str, dataset_id: str -) -> CommandResponse: +def delete(config: dict, keycloak_token: str, organization_id: str, workspace_id: str, dataset_id: str) -> CommandResponse: """Delete a dataset by ID""" api_instance = get_dataset_api_instance(config, keycloak_token) try: @@ -121,9 +119,7 @@ def get(config: dict, keycloak_token: str, organization_id: str, workspace_id: s api_instance = get_dataset_api_instance(config, keycloak_token) try: logger.info(API_REQUEST_MESSAGE) - dataset = api_instance.get_dataset( - organization_id=organization_id, workspace_id=workspace_id, dataset_id=dataset_id - ) + dataset = api_instance.get_dataset(organization_id=organization_id, workspace_id=workspace_id, dataset_id=dataset_id) logger.info(f" [green]✔[/green] Dataset [bold cyan]{dataset.id}[/bold cyan] retrieved successfully") return CommandResponse.success(dataset.model_dump()) except Exception as e: @@ -191,9 +187,7 @@ def create_part( if not created: logger.error(" [bold red]✘ API returned no data.[/bold red]") return CommandResponse.fail() - logger.info( - f" [bold green]✔[/bold green] Dataset part [bold cyan]{created.id}[/bold cyan] successfully created" - ) + logger.info(f" [bold green]✔[/bold green] Dataset part [bold cyan]{created.id}[/bold cyan] successfully created") return CommandResponse.success(created.model_dump()) except Exception as e: logger.error(f" [bold red]✘[/bold red] Creation Failed Reason: {e}") @@ -452,9 +446,7 @@ def download_part( @option("--oid", "organization_id", required=True, type=str, help="Organization ID") @option("--wid", "workspace_id", required=True, type=str, help="Workspace ID") @option("--did", "dataset_id", required=True, type=str, help="Dataset ID") -def list_parts( - config: dict, keycloak_token: str, organization_id: str, workspace_id: str, dataset_id: str -) -> CommandResponse: +def list_parts(config: dict, keycloak_token: str, organization_id: str, workspace_id: str, dataset_id: str) -> CommandResponse: """List dataset parts""" api_instance = get_dataset_api_instance(config, keycloak_token) try: diff --git a/Babylon/commands/api/organization.py b/Babylon/commands/api/organization.py index d846da0b..199d2e08 100644 --- a/Babylon/commands/api/organization.py +++ b/Babylon/commands/api/organization.py @@ -50,9 +50,7 @@ def create(config: dict, keycloak_token: str, payload_file) -> CommandResponse: logger.error(" [bold red]✘[/bold red] API returned no data.") return CommandResponse.fail() - logger.info( - f" [bold green]✔[/bold green] Organization [bold cyan]{organization.id}[/bold cyan] successfully created" - ) + logger.info(f" [bold green]✔[/bold green] Organization [bold cyan]{organization.id}[/bold cyan] successfully created") return CommandResponse.success(organization.model_dump()) except Exception as e: logger.error(f" [bold red]✘[/bold red] Creation Failed Reason: {e}") @@ -72,9 +70,7 @@ def delete(config: dict, keycloak_token: str, organization_id: str) -> CommandRe # API Execution logger.info(API_REQUEST_MESSAGE) api_instance.delete_organization(organization_id) - logger.info( - f" [bold green]✔[/bold green] Organization [bold red]{organization_id}[/bold red] successfully deleted" - ) + logger.info(f" [bold green]✔[/bold green] Organization [bold red]{organization_id}[/bold red] successfully deleted") return CommandResponse.success() except Exception as e: logger.error(f" [bold red]✘[/bold red] Deletion Failed Reason: {e}") diff --git a/Babylon/commands/api/run.py b/Babylon/commands/api/run.py index 327165b5..62c17fe5 100644 --- a/Babylon/commands/api/run.py +++ b/Babylon/commands/api/run.py @@ -32,9 +32,7 @@ def runs(): @option("--wid", "workspace_id", required=True, type=str, help="Workspace ID") @option("--rid", "runner_id", required=True, type=str, help="Runner ID") @option("--rnid", "run_id", required=True, type=str, help="Run ID") -def get( - config: dict, keycloak_token: str, organization_id: str, workspace_id: str, runner_id: str, run_id: str -) -> CommandResponse: +def get(config: dict, keycloak_token: str, organization_id: str, workspace_id: str, runner_id: str, run_id: str) -> CommandResponse: """Get a run""" api_instance = get_run_api_instance(config, keycloak_token) try: @@ -59,9 +57,7 @@ def get( @option("--wid", "workspace_id", required=True, type=str, help="Workspace ID") @option("--rid", "runner_id", required=True, type=str, help="Runner ID") @option("--rnid", "run_id", required=True, type=str, help="Run ID") -def delete( - config: dict, keycloak_token: str, organization_id: str, workspace_id: str, runner_id: str, run_id: str -) -> CommandResponse: +def delete(config: dict, keycloak_token: str, organization_id: str, workspace_id: str, runner_id: str, run_id: str) -> CommandResponse: """Delete a run""" api_instance = get_run_api_instance(config, keycloak_token) try: @@ -86,9 +82,7 @@ def delete( @option("--oid", "organization_id", required=True, type=str, help="Organization ID") @option("--wid", "workspace_id", required=True, type=str, help="Workspace ID") @option("--rid", "runner_id", required=True, type=str, help="Runner ID") -def list_runs( - config: dict, keycloak_token: str, organization_id: str, workspace_id: str, runner_id: str -) -> CommandResponse: +def list_runs(config: dict, keycloak_token: str, organization_id: str, workspace_id: str, runner_id: str) -> CommandResponse: """List runs""" api_instance = get_run_api_instance(config, keycloak_token) try: diff --git a/Babylon/commands/api/runner.py b/Babylon/commands/api/runner.py index 7727785c..8e4f5f17 100644 --- a/Babylon/commands/api/runner.py +++ b/Babylon/commands/api/runner.py @@ -70,9 +70,7 @@ def create( @option("--oid", "organization_id", required=True, type=str, help="Organization ID") @option("--wid", "workspace_id", required=True, type=str, help="Workspace ID") @option("--rid", "runner_id", required=True, type=str, help="Runner ID") -def delete( - config: dict, keycloak_token: str, organization_id: str, workspace_id: str, runner_id: str -) -> CommandResponse: +def delete(config: dict, keycloak_token: str, organization_id: str, workspace_id: str, runner_id: str) -> CommandResponse: """Delete a runner by ID""" api_instance = get_runner_api_instance(config, keycloak_token) try: @@ -119,9 +117,7 @@ def get(config: dict, keycloak_token: str, organization_id: str, workspace_id: s api_instance = get_runner_api_instance(config, keycloak_token) try: logger.info(API_REQUEST_MESSAGE) - runner = api_instance.get_runner( - organization_id=organization_id, workspace_id=workspace_id, runner_id=runner_id - ) + runner = api_instance.get_runner(organization_id=organization_id, workspace_id=workspace_id, runner_id=runner_id) logger.info(f" [green]✔[/green] Runner [bold cyan]{runner.id}[/bold cyan] retrieved successfully") return CommandResponse.success(runner.model_dump()) except Exception as e: @@ -167,9 +163,7 @@ def update( @option("--oid", "organization_id", required=True, type=str, help="Organization ID") @option("--wid", "workspace_id", required=True, type=str, help="Workspace ID") @option("--rid", "runner_id", required=True, type=str, help="Runner ID") -def start( - config: dict, keycloak_token: str, organization_id: str, workspace_id: str, runner_id: str -) -> CommandResponse: +def start(config: dict, keycloak_token: str, organization_id: str, workspace_id: str, runner_id: str) -> CommandResponse: """Start a run""" api_instance = get_runner_api_instance(config, keycloak_token) try: diff --git a/Babylon/commands/api/workspace.py b/Babylon/commands/api/workspace.py index 61b47dab..9744e16b 100644 --- a/Babylon/commands/api/workspace.py +++ b/Babylon/commands/api/workspace.py @@ -51,9 +51,7 @@ def create(config: dict, keycloak_token: str, organization_id: str, solution_id: if not workspace: logger.error(" [bold red]✘[/bold red] API returned no data.") return CommandResponse.fail() - logger.info( - f" [bold green]✔[/bold green] Workspace [bold cyan]{workspace.id}[/bold cyan] successfully created" - ) + logger.info(f" [bold green]✔[/bold green] Workspace [bold cyan]{workspace.id}[/bold cyan] successfully created") return CommandResponse.success(workspace.model_dump()) except Exception as e: logger.error(f" [bold red]✘[/bold red] Creation Failed Reason: {e}") diff --git a/Babylon/commands/macro/apply.py b/Babylon/commands/macro/apply.py index 59393eb2..f27a74ed 100644 --- a/Babylon/commands/macro/apply.py +++ b/Babylon/commands/macro/apply.py @@ -36,7 +36,7 @@ def load_resources_from_files(files_to_deploy: list[PathlibPath]) -> tuple[list, return (organizations, solutions, workspaces, webapps) -def deploy_objects(objects: list, object_type: str): +def deploy_objects(objects: list, object_type: str, deploy_dir: PathlibPath): for o in objects: content = o.get("content") namespace = o.get("namespace") @@ -45,7 +45,7 @@ def deploy_objects(objects: list, object_type: str): elif object_type == "solution": deploy_solution(namespace=namespace, file_content=content) elif object_type == "workspace": - deploy_workspace(namespace=namespace, file_content=content) + deploy_workspace(namespace=namespace, file_content=content, deploy_dir=deploy_dir) elif object_type == "webapp": deploy_webapp(namespace=namespace, file_content=content) @@ -91,13 +91,13 @@ def apply( env.set_variable_files(variables_files) organizations, solutions, workspaces, webapps = load_resources_from_files(files_to_deploy) if organization: - deploy_objects(organizations, "organization") + deploy_objects(organizations, "organization", deploy_dir) if solution: - deploy_objects(solutions, "solution") + deploy_objects(solutions, "solution", deploy_dir) if workspace: - deploy_objects(workspaces, "workspace") + deploy_objects(workspaces, "workspace", deploy_dir) if webapp: - deploy_objects(webapps, "webapp") + deploy_objects(webapps, "webapp", deploy_dir) final_state = env.get_state_from_local() services = final_state.get("services", {}) api_data = services.get("api", {}) diff --git a/Babylon/commands/macro/deploy.py b/Babylon/commands/macro/deploy.py index ef156d5b..3c1ee06a 100644 --- a/Babylon/commands/macro/deploy.py +++ b/Babylon/commands/macro/deploy.py @@ -7,6 +7,7 @@ from cosmotech_api.models.solution_security import SolutionSecurity from cosmotech_api.models.workspace_access_control import WorkspaceAccessControl from cosmotech_api.models.workspace_security import WorkspaceSecurity +from kubernetes import client, config logger = getLogger(__name__) @@ -94,9 +95,7 @@ def update_default_security( getattr(api_instance, f"update_{object_type}_default_security")(object_id, desired_security.default) logger.info(f" [bold green]✔[/bold green] Updated [magenta]{object_type}[/magenta] default security") except Exception as e: - logger.error( - f" [bold red]✘[/bold red] Failed to update [magenta]{object_type}[/magenta] default security: {e}" - ) + logger.error(f" [bold red]✘[/bold red] Failed to update [magenta]{object_type}[/magenta] default security: {e}") def update_object_security( @@ -121,38 +120,21 @@ def update_object_security( if entry.id in to_add: try: getattr(api_instance, f"create_{object_type}_access_control")(*object_id, entry) - logger.info( - f" [bold green]✔[/bold green] Access control" - f" for id [magenta]{entry.id}[/magenta] added successfully" - ) + logger.info(f" [bold green]✔[/bold green] Access control for id [magenta]{entry.id}[/magenta] added successfully") except Exception as e: - logger.error( - f" [bold red]✘[/bold red] Failed to add access control for id [magenta]{entry.id}[/magenta]: {e}" - ) + logger.error(f" [bold red]✘[/bold red] Failed to add access control for id [magenta]{entry.id}[/magenta]: {e}") if entry.id in to_update: try: - getattr(api_instance, f"update_{object_type}_access_control")( - *object_id, entry.id, {"role": entry.role} - ) - logger.info( - f" [bold green]✔[/bold green] Access control" - f" for id [magenta]{entry.id}[/magenta] updated successfully" - ) + getattr(api_instance, f"update_{object_type}_access_control")(*object_id, entry.id, {"role": entry.role}) + logger.info(f" [bold green]✔[/bold green] Access control for id [magenta]{entry.id}[/magenta] updated successfully") except Exception as e: - logger.error( - f" [bold red]✘[/bold red] Failed to update access control" - f" for id [magenta]{entry.id}[/magenta]: {e}" - ) + logger.error(f" [bold red]✘[/bold red] Failed to update access control for id [magenta]{entry.id}[/magenta]: {e}") for entry_id in to_delete: try: getattr(api_instance, f"delete_{object_type}_access_control")(*object_id, entry_id) - logger.info( - f" [bold green]✔[/bold green] Access control for id [magenta]{entry_id}[/magenta] deleted successfully" - ) + logger.info(f" [bold green]✔[/bold green] Access control for id [magenta]{entry_id}[/magenta] deleted successfully") except Exception as e: - logger.error( - f" [bold red]✘[/bold red] Failed to delete access control for id [magenta]{entry_id}[/magenta]: {e}" - ) + logger.error(f" [bold red]✘[/bold red] Failed to delete access control for id [magenta]{entry_id}[/magenta]: {e}") def dict_to_tfvars(payload: dict) -> str: @@ -181,3 +163,26 @@ def dict_to_tfvars(payload: dict) -> str: else: lines.append(f'{key} = "{value}"') return "\n".join(lines) + + +def get_postgres_service_host(namespace: str) -> str: + """Discovers the PostgreSQL service name in a namespace to build its FQDN + + Note: This function assumes PostgreSQL is running within the same Kubernetes cluster. + External database clusters are not currently supported. + """ + try: + config.load_kube_config() + v1 = client.CoreV1Api() + services = v1.list_namespaced_service(namespace) + + for svc in services.items: + if "postgresql" in svc.metadata.name or svc.metadata.labels.get("app.kubernetes.io/name") == "postgresql": + logger.info(f" [dim]→ Found PostgreSQL service {svc.metadata.name}[/dim]") + return f"{svc.metadata.name}.{namespace}.svc.cluster.local" + + return f"postgresql.{namespace}.svc.cluster.local" + except Exception as e: + logger.warning(" [bold yellow]⚠[/bold yellow] Service discovery failed ! default will be used.") + logger.debug(f" Exception details: {e}", exc_info=True) + return f"postgresql.{namespace}.svc.cluster.local" diff --git a/Babylon/commands/macro/deploy_organization.py b/Babylon/commands/macro/deploy_organization.py index 7b9f4cc0..43589be9 100644 --- a/Babylon/commands/macro/deploy_organization.py +++ b/Babylon/commands/macro/deploy_organization.py @@ -49,9 +49,7 @@ def deploy_organization(namespace: str, file_content: str): state["services"]["api"]["organization_id"] = organization.id else: # Case: Update Existing Organization - logger.info( - f" [dim]→ Existing ID [bold cyan]{api_section['organization_id']}[/bold cyan] found. Updating...[/dim]" - ) + logger.info(f" [dim]→ Existing ID [bold cyan]{api_section['organization_id']}[/bold cyan] found. Updating...[/dim]") organization_update_request = OrganizationUpdateRequest.from_dict(payload) updated = api_instance.update_organization( organization_id=api_section["organization_id"], organization_update_request=organization_update_request @@ -63,9 +61,7 @@ def deploy_organization(namespace: str, file_content: str): if payload.get("security"): try: logger.info(" [dim]→ Syncing security policies...[/dim]") - current_security = api_instance.get_organization_security( - organization_id=api_section["organization_id"] - ) + current_security = api_instance.get_organization_security(organization_id=api_section["organization_id"]) update_object_security( "organization", current_security=current_security, @@ -76,10 +72,7 @@ def deploy_organization(namespace: str, file_content: str): except Exception as e: logger.error(f" [bold red]✘[/bold red] Security update failed: {e}") return CommandResponse.fail() - logger.info( - f" [bold green]✔[/bold green] Organization" - f" [bold magenta]{api_section['organization_id']}[/bold magenta] updated" - ) + logger.info(f" [bold green]✔[/bold green] Organization [bold magenta]{api_section['organization_id']}[/bold magenta] updated") # --- State Persistence --- # Ensure the local and remote states are synchronized after successful API calls env.store_state_in_local(state) diff --git a/Babylon/commands/macro/deploy_solution.py b/Babylon/commands/macro/deploy_solution.py index 15e0df19..bf004d55 100644 --- a/Babylon/commands/macro/deploy_solution.py +++ b/Babylon/commands/macro/deploy_solution.py @@ -51,9 +51,7 @@ def deploy_solution(namespace: str, file_content: str) -> bool: state["services"]["api"]["solution_id"] = solution.id else: # Case: Update Existing Solution - logger.info( - f" [dim]→ Existing ID [bold cyan]{api_section['solution_id']}[/bold cyan] found. Updating...[/dim]" - ) + logger.info(f" [dim]→ Existing ID [bold cyan]{api_section['solution_id']}[/bold cyan] found. Updating...[/dim]") solution_update_request = SolutionUpdateRequest.from_dict(payload) updated = api_instance.update_solution( organization_id=api_section["organization_id"], @@ -80,9 +78,7 @@ def deploy_solution(namespace: str, file_content: str) -> bool: except Exception as e: logger.error(f" [bold red]✘[/bold red] Security update failed: {e}") return CommandResponse.fail() - logger.info( - f" [bold green]✔[/bold green] Solution [bold magenta]{api_section['solution_id']}[/bold magenta] updated" - ) + logger.info(f" [bold green]✔[/bold green] Solution [bold magenta]{api_section['solution_id']}[/bold magenta] updated") # --- State Persistence --- # Ensure the local and remote states are synchronized after successful API calls env.store_state_in_local(state) diff --git a/Babylon/commands/macro/deploy_webapp.py b/Babylon/commands/macro/deploy_webapp.py index 650e6197..fd5674b2 100644 --- a/Babylon/commands/macro/deploy_webapp.py +++ b/Babylon/commands/macro/deploy_webapp.py @@ -48,9 +48,7 @@ def deploy_webapp(namespace: str, file_content: str): return logger.info(" [dim]→ Running Terraform deployment...[/dim]") try: - process = subprocess.Popen( - executable, cwd=tf_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1 - ) + process = subprocess.Popen(executable, cwd=tf_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) for line in process.stdout: clean_line = line.strip() if not clean_line: @@ -74,9 +72,7 @@ def deploy_webapp(namespace: str, file_content: str): services["webapp"]["webapp_name"] = f"webapp-{webapp_name}" services["webapp"]["webapp_url"] = webapp_url if return_code == 0: - logger.info( - f" [bold green]✔[/bold green] WebApp [bold white]{webapp_name}[/bold white] deployed successfully" - ) + logger.info(f" [bold green]✔[/bold green] WebApp [bold white]{webapp_name}[/bold white] deployed successfully") env.store_state_in_local(state) if env.remote: env.store_state_in_cloud(state) diff --git a/Babylon/commands/macro/deploy_workspace.py b/Babylon/commands/macro/deploy_workspace.py index 0e8dda43..ca0edd30 100644 --- a/Babylon/commands/macro/deploy_workspace.py +++ b/Babylon/commands/macro/deploy_workspace.py @@ -1,13 +1,20 @@ +import subprocess from json import dumps from logging import getLogger +from pathlib import Path as PathlibPath +from string import Template from click import echo, style from cosmotech_api.models.workspace_create_request import WorkspaceCreateRequest from cosmotech_api.models.workspace_security import WorkspaceSecurity from cosmotech_api.models.workspace_update_request import WorkspaceUpdateRequest +from kubernetes import client, utils +from kubernetes import config as kube_config +from kubernetes.utils import FailToCreateError +from yaml import safe_load from Babylon.commands.api.workspace import get_workspace_api_instance -from Babylon.commands.macro.deploy import update_object_security +from Babylon.commands.macro.deploy import get_postgres_service_host, update_object_security from Babylon.utils.credentials import get_keycloak_token from Babylon.utils.environment import Environment from Babylon.utils.response import CommandResponse @@ -16,7 +23,7 @@ env = Environment() -def deploy_workspace(namespace: str, file_content: str) -> bool: +def deploy_workspace(namespace: str, file_content: str, deploy_dir: PathlibPath) -> bool: echo(style(f"\n🚀 Deploying Workspace in namespace: {env.environ_id}", bold=True, fg="cyan")) # Retrieve the state @@ -28,7 +35,6 @@ def deploy_workspace(namespace: str, file_content: str) -> bool: keycloak_token, config = get_keycloak_token() payload: dict = content.get("spec").get("payload") api_section = state["services"]["api"] - # Determine if we are performing a Create or Update based on state api_section["workspace_id"] = payload.get("id") or api_section.get("workspace_id", "") spec = {} @@ -51,9 +57,7 @@ def deploy_workspace(namespace: str, file_content: str) -> bool: state["services"]["api"]["workspace_id"] = workspace.id else: # Case: Update Existing Workspace - logger.info( - f" [dim]→ Existing ID [bold cyan]{api_section['workspace_id']}[/bold cyan] found. Updating...[/dim]" - ) + logger.info(f" [dim]→ Existing ID [bold cyan]{api_section['workspace_id']}[/bold cyan] found. Updating...[/dim]") workspace_update_request = WorkspaceUpdateRequest.from_dict(payload) updated = api_instance.update_workspace( organization_id=api_section["organization_id"], @@ -80,9 +84,109 @@ def deploy_workspace(namespace: str, file_content: str) -> bool: except Exception as e: logger.error(f" [bold red]✘[/bold red] Security update failed: {e}") return CommandResponse.fail() - logger.info( - f" [bold green]✔[/bold green] Workspace [bold magenta]{api_section['workspace_id']}[/bold magenta] updated" - ) + logger.info(f" [bold green]✔[/bold green] Workspace [bold magenta]{api_section['workspace_id']}[/bold magenta] updated") + workspace_id = state["services"]["api"]["workspace_id"] + spec = content.get("spec") or {} + sidecars = spec.get("sidecars") or {} + postgres_section = sidecars.get("postgres") or {} + schema_config = postgres_section.get("schema") or {} + should_create_schema = schema_config.get("create", False) + if should_create_schema: + db_host = get_postgres_service_host(env.environ_id) + logger.info(f" [dim]→ Initializing PostgreSQL schema for workspace {workspace_id}...[/dim]") + pg_config = env.get_config_from_k8s_secret_by_tenant("postgresql-config", env.environ_id) + api_config = env.get_config_from_k8s_secret_by_tenant("postgresql-cosmotechapi", env.environ_id) + if pg_config and api_config: + schema_name = f"{workspace_id.replace('-', '_')}" + mapping = { + "namespace": env.environ_id, + "db_host": db_host, + "db_port": "5432", + "cosmotech_api_database": api_config.get("database-name"), + "cosmotech_api_admin_username": api_config.get("admin-username"), + "cosmotech_api_admin_password": api_config.get("admin-password"), + "cosmotech_api_writer_username": api_config.get("writer-username"), + "cosmotech_api_reader_username": api_config.get("reader-username"), + "workspace_schema": schema_name, + "job_name": workspace_id, + } + jobs = schema_config.get("jobs", []) + if not isinstance(deploy_dir, PathlibPath): + deploy_dir = PathlibPath(deploy_dir) + for job in jobs: + script_path = deploy_dir / job.get("path", "") / job.get("name", "") + if script_path.exists(): + kube_config.load_kube_config() + k8s_client = client.ApiClient() + k8s_job_name = f"postgresql-init-{workspace_id}" + with open(script_path, "r") as f: + raw_content = f.read() + templated_yaml = Template(raw_content).safe_substitute(mapping) + yaml_dict = safe_load(templated_yaml) + try: + utils.create_from_dict(k8s_client, yaml_dict, namespace=env.environ_id) + logger.info(f" [dim]→ Waiting for job [cyan]{k8s_job_name}[/cyan] to complete...[/dim]") + wait_process = subprocess.run( + [ + "kubectl", + "wait", + "--for=condition=complete", + "job", + k8s_job_name, + f"--namespace={env.environ_id}", + "--timeout=50s", + ], + capture_output=True, + text=True, + ) + if wait_process.returncode != 0: + logger.error( + f" [bold red]✘[/bold red] Job {k8s_job_name} did not complete successfully" + f" see babylon logs for details" + ) + logger.debug(f" [bold red]✘[/bold red] Job wait output {wait_process.stdout} {wait_process.stderr}") + else: + # Job completed, now check the logs for error + logger.info(" [dim]→ Checking job logs for errors...[/dim]") + logs_process = subprocess.run( + ["kubectl", "logs", f"job/{k8s_job_name}", "-n", env.environ_id], + capture_output=True, + text=True, + ) + if logs_process.returncode == 0: + job_logs = logs_process.stdout if logs_process.stdout else logs_process.stderr + if "ERROR" in job_logs or "error" in job_logs: + logger.error(" [bold red]✘[/bold red] Schema creation failed inside the container") + logger.debug(f" [bold red]✘[/bold red] Job logs : {job_logs}") + elif "already exists" in job_logs: + logger.info( + f" [yellow]⚠[/yellow] [dim]Schema [magenta]{schema_name}[/magenta]" + f" already exists (skipping creation)[/dim]" + ) + else: + logger.info( + f" [green]✔[/green] Schema creation [magenta]{schema_name}[/magenta] completed successfully" + ) + state["services"]["postgres"]["schema_name"] = schema_name + else: + logger.error(f" [bold red]✘[/bold red] Failed to retrieve logs for job {k8s_job_name}") + logger.debug( + f" [bold red]✘[/bold red] Logs retrieval output {logs_process.stdout} {logs_process.stderr}" + ) + + except FailToCreateError as e: + for inner_exception in e.api_exceptions: + if inner_exception.status == 409: + logger.warning(f" [yellow]⚠[/yellow] [dim]Job [cyan]{k8s_job_name}[/cyan] already exists.[/dim]") + else: + logger.error( + f" [bold red]✘[/bold red] K8s Error ({inner_exception.status}): {inner_exception.reason}" + ) + logger.debug(f" Detail: {inner_exception.body}") + except Exception as e: + logger.error(" [bold red]✘[/bold red] Unexpected error please check babylon logs file for details") + logger.debug(f" [bold red]✘[/bold red] {e}") + # --- State Persistence --- # Ensure the local and remote states are synchronized after successful API calls env.store_state_in_local(state) diff --git a/Babylon/commands/macro/destroy.py b/Babylon/commands/macro/destroy.py index 9ddd77f7..fe065a7f 100644 --- a/Babylon/commands/macro/destroy.py +++ b/Babylon/commands/macro/destroy.py @@ -1,14 +1,18 @@ import subprocess from logging import getLogger from pathlib import Path +from string import Template from typing import Callable from click import command, echo, option, style +from kubernetes import client, utils +from kubernetes import config as kube_config +from yaml import safe_load from Babylon.commands.api.organization import get_organization_api_instance from Babylon.commands.api.solution import get_solution_api_instance from Babylon.commands.api.workspace import get_workspace_api_instance -from Babylon.commands.macro.deploy import resolve_inclusion_exclusion +from Babylon.commands.macro.deploy import get_postgres_service_host, resolve_inclusion_exclusion from Babylon.utils.credentials import get_keycloak_token from Babylon.utils.decorators import injectcontext, retrieve_state from Babylon.utils.environment import Environment @@ -18,8 +22,98 @@ env = Environment() +def _destroy_schema(schema_name: str, state: dict) -> bool: + """ + Destroy PostgreSQL schema for a workspace. + """ + if not schema_name: + logger.warning(" [yellow]⚠[/yellow] [dim]No schema found ! skipping deletion[/dim]") + return + workspace_id_tmp = f"{schema_name.replace('_', '-')}" + db_host = get_postgres_service_host(env.environ_id) + logger.info(f" [dim]→ Destroying postgreSQL schema for workspace {workspace_id_tmp}...[/dim]") + + pg_config = env.get_config_from_k8s_secret_by_tenant("postgresql-config", env.environ_id) + api_config = env.get_config_from_k8s_secret_by_tenant("postgresql-cosmotechapi", env.environ_id) + + if not pg_config or not api_config: + logger.error(" [bold red]✘[/bold red] Failed to retrieve postgreSQL configuration from secrets") + return + + mapping = { + "namespace": env.environ_id, + "db_host": db_host, + "db_port": "5432", + "cosmotech_api_database": api_config.get("database-name"), + "cosmotech_api_admin_username": api_config.get("admin-username"), + "cosmotech_api_admin_password": api_config.get("admin-password"), + "cosmotech_api_writer_username": api_config.get("writer-username"), + "cosmotech_api_reader_username": api_config.get("reader-username"), + "workspace_schema": schema_name, + "job_name": workspace_id_tmp, + } + destroy_jobs = env.original_template_path / "yaml" / "k8s_job_destroy.yaml" + k8s_job_name = f"postgresql-destroy-{workspace_id_tmp}" + kube_config.load_kube_config() + k8s_client = client.ApiClient() + with open(destroy_jobs, "r") as f: + raw_content = f.read() + + templated_yaml = Template(raw_content).safe_substitute(mapping) + yaml_dict = safe_load(templated_yaml) + logger.info(" [dim]→ Applying kubernetes destroy job...[/dim]") + try: + utils.create_from_dict(k8s_client, yaml_dict, namespace=env.environ_id) + logger.info(f" [dim]→ Waiting for job [cyan]{k8s_job_name}[/cyan] to complete...[/dim]") + wait_process = subprocess.run( + [ + "kubectl", + "wait", + "--for=condition=complete", + "job", + k8s_job_name, + f"--namespace={env.environ_id}", + "--timeout=300s", + ], + capture_output=True, + text=True, + ) + if wait_process.returncode == 0: + # Job completed, now check the logs for error + logger.info(" [dim]→ Checking job logs for errors...[/dim]") + logs_process = subprocess.run( + ["kubectl", "logs", f"job/{k8s_job_name}", "-n", env.environ_id], + capture_output=True, + text=True, + ) + if logs_process.returncode == 0: + job_logs = logs_process.stdout if logs_process.stdout else logs_process.stderr + if "ERROR" in job_logs or "error" in job_logs: + logger.error(" [bold red]✘[/bold red] Schema destruction failed inside the container") + logger.debug(f" [bold red]✘[/bold red] Job logs : {job_logs}") + elif "does not exist" in job_logs: + logger.info( + f" [yellow]⚠[/yellow] [dim]Schema [magenta]{schema_name}[/magenta] does not exist (nothing to clean)[/dim]" + ) + state["services"]["postgres"]["schema_name"] = "" + else: + logger.info(f" [green]✔[/green] Schema destruction [magenta]{schema_name}[/magenta] completed successfully") + state["services"]["postgres"]["schema_name"] = "" + else: + logger.error(f" [bold red]✘[/bold red] Failed to retrieve logs for job {k8s_job_name}") + logger.debug(f" [bold red]✘[/bold red] Logs retrieval output {logs_process.stdout} {logs_process.stderr}") + + else: + logger.error(f" [bold red]✘[/bold red] Job {k8s_job_name} did not complete successfully see babylon logs for details") + logger.debug(f" [bold red]✘[/bold red] Job wait output {wait_process.stdout} {wait_process.stderr}") + + except Exception as e: + logger.error(" [bold red]✘[/bold red] Unexpected error please check babylon logs file for details") + logger.debug(f" [bold red]✘[/bold red] {e}") + + def _destroy_webapp(state: dict): - """Run the Terraform destroy process for WebApp resources""" + """Terraform Destroy webapp""" logger.info(" [dim]→ Running Terraform destroy for WebApp resources...[/dim]") webapp_state = state.get("services", {}).get("webapp", {}) webapp_neme = webapp_state.get("webapp_name") @@ -89,9 +183,7 @@ def _delete_resource( except Exception as e: error_msg = str(e) if "404" in error_msg or "Not Found" in error_msg: - logger.info( - f" [bold yellow]⚠[/bold yellow] {resource_name} [magenta]{resource_id}[/magenta] already deleted (404)" - ) + logger.info(f" [bold yellow]⚠[/bold yellow] {resource_name} [magenta]{resource_id}[/magenta] already deleted (404)") state["services"]["api"][state_key] = "" else: logger.error(f" [bold red]✘[/bold red] Error deleting {resource_name.lower()} {resource_id} reason: {e}") @@ -101,12 +193,8 @@ def _delete_resource( @injectcontext() @retrieve_state @option("--include", "include", multiple=True, type=str, help="Specify the resources to destroy.") -@option("--exclude", "exclude", multiple=True, type=str, help="Specify the resources to exclude from destroction.") -def destroy( - state: dict, - include: tuple[str], - exclude: tuple[str], -): +@option("--exclude", "exclude", multiple=True, type=str, help="Specify the resources to exclude from destruction.") +def destroy(state: dict, include: tuple[str], exclude: tuple[str]): """Macro Destroy""" organization, solution, workspace, webapp = resolve_inclusion_exclusion(include, exclude) # Header for the destructive operation @@ -115,12 +203,15 @@ def destroy( # We need the Org ID for most sub-resource deletions api_state = state["services"]["api"] + schema_state = state["services"]["postgres"] org_id = api_state["organization_id"] if solution: api = get_solution_api_instance(config=config, keycloak_token=keycloak_token) _delete_resource(api.delete_solution, "Solution", org_id, api_state["solution_id"], state, "solution_id") + if workspace: + _destroy_schema(schema_state["schema_name"], state) api = get_workspace_api_instance(config=config, keycloak_token=keycloak_token) _delete_resource(api.delete_workspace, "Workspace", org_id, api_state["workspace_id"], state, "workspace_id") diff --git a/Babylon/commands/macro/init.py b/Babylon/commands/macro/init.py index 11d94580..62a29de4 100644 --- a/Babylon/commands/macro/init.py +++ b/Babylon/commands/macro/init.py @@ -40,8 +40,6 @@ def init(project_folder: str, variables_file: str): "Organization.yaml", "Solution.yaml", "Workspace.yaml", - "Dataset.yaml", - "Runner.yaml", "Webapp.yaml", ] try: @@ -55,10 +53,12 @@ def init(project_folder: str, variables_file: str): copy(deploy_file, destination) logger.info(f" [green]✔[/green] Generated [white]{file}[/white]") - customers_src = env.original_template_path / "yaml" / "dataset" / "customers.csv" - customers_dst = Path(getcwd()) / "customers.csv" - copy(customers_src, customers_dst) - logger.info(" [green]✔[/green] Generated [white]customers.csv[/white]") + postgres_jobs_path = project_path / "postgres" / "jobs" + postgres_jobs_path.mkdir(parents=True, exist_ok=True) + k8s_template = env.original_template_path / "yaml" / "k8s_job.yaml" + if k8s_template.exists(): + copy(k8s_template, postgres_jobs_path / "k8s_job.yaml") + logger.info(" [green]✔[/green] Generated [white]postgres/jobs/k8s_job.yaml[/white]") variables_template = env.original_template_path / "yaml" / "variables.yaml" copy(variables_template, variables_path) @@ -71,4 +71,5 @@ def init(project_folder: str, variables_file: str): echo(style(f" 1. Edit your variables in {variables_file}", fg="cyan")) echo(style(" 2. Run your first deployment command", fg="cyan")) except Exception as e: - logger.error(f" [bold red]✘[/bold red] An error occurred while scaffolding: {e}") + logger.error(" [bold red]✘[/bold red] An error occurred while scaffolding see babylon logs for details") + logger.debug(f" [bold red]✘[/bold red] Error details: {e}", exc_info=True) diff --git a/Babylon/commands/namespace/use.py b/Babylon/commands/namespace/use.py index 9709216d..d94b0417 100644 --- a/Babylon/commands/namespace/use.py +++ b/Babylon/commands/namespace/use.py @@ -16,7 +16,6 @@ def use() -> CommandResponse: """Switch to a specific Babylon namespace or create a new one""" env.store_namespace_in_local() logger.info( - f" [green]✔[/green] Switched to context [green]{env.context_id}[/green]," - f" tenant [green]{env.environ_id}[/green] successfully", + f" [green]✔[/green] Switched to context [green]{env.context_id}[/green], tenant [green]{env.environ_id}[/green] successfully", ) return CommandResponse.success() diff --git a/Babylon/commands/plugin/add.py b/Babylon/commands/plugin/add.py index d109d75a..8c145df8 100644 --- a/Babylon/commands/plugin/add.py +++ b/Babylon/commands/plugin/add.py @@ -22,7 +22,6 @@ def add(plugin_path: pathlib.Path) -> CommandResponse: logger.info(f"Plugin {plugin_name} was added to config") return CommandResponse.success() logger.error( - "Plugin was not added to the config, make sure the folder is a correct plugin " - "or that no plugin with the same name exists" + "Plugin was not added to the config, make sure the folder is a correct plugin or that no plugin with the same name exists" ) return CommandResponse.fail() diff --git a/Babylon/commands/powerbi/dataset/services/powerbi_params_svc.py b/Babylon/commands/powerbi/dataset/services/powerbi_params_svc.py index edd38dc9..76874958 100644 --- a/Babylon/commands/powerbi/dataset/services/powerbi_params_svc.py +++ b/Babylon/commands/powerbi/dataset/services/powerbi_params_svc.py @@ -24,9 +24,7 @@ def update(self, workspace_id: str, params: list[tuple[str, str]], dataset_id: s workspace_id = workspace_id or self.state["powerbi"]["workspace"]["id"] # Preparing parameter data details = {"updateDetails": [{"name": param.get("id"), "newValue": param.get("value")} for param in params]} - update_url = ( - f"https://api.powerbi.com/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/Default.UpdateParameters" - ) + update_url = f"https://api.powerbi.com/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/Default.UpdateParameters" response = oauth_request(update_url, self.powerbi_token, json=details, type="POST") if response is None: logger.info(f"[powerbi] failled to update dataset with id: {dataset_id}") diff --git a/Babylon/commands/powerbi/report/service/powerbi_report_api_svc.py b/Babylon/commands/powerbi/report/service/powerbi_report_api_svc.py index 8c9a0ce7..ddbad850 100644 --- a/Babylon/commands/powerbi/report/service/powerbi_report_api_svc.py +++ b/Babylon/commands/powerbi/report/service/powerbi_report_api_svc.py @@ -101,8 +101,7 @@ def upload( } name_conflict = "CreateOrOverwrite" if override else "Abort" route = ( - f"https://api.powerbi.com/v1.0/myorg/groups/{workspace_id}" - f"/imports?datasetDisplayName={name}&nameConflict={name_conflict}" + f"https://api.powerbi.com/v1.0/myorg/groups/{workspace_id}/imports?datasetDisplayName={name}&nameConflict={name_conflict}" ) session = requests.Session() with open(pbix_filename, "rb") as _f: diff --git a/Babylon/commands/powerbi/workspace/services/powerbi_workspace_api_svc.py b/Babylon/commands/powerbi/workspace/services/powerbi_workspace_api_svc.py index eebe1787..9ffaeb7e 100644 --- a/Babylon/commands/powerbi/workspace/services/powerbi_workspace_api_svc.py +++ b/Babylon/commands/powerbi/workspace/services/powerbi_workspace_api_svc.py @@ -46,9 +46,7 @@ def get_all(self): url_groups = "https://api.powerbi.com/v1.0/myorg/groups" response = oauth_request(url=url_groups, access_token=self.powerbi_token) if response is None: - logger.warning( - "[powerbi] Either workspace name list is empty or you are not allowed to access the PowerBI service" - ) + logger.warning("[powerbi] Either workspace name list is empty or you are not allowed to access the PowerBI service") return None output_data = response.json().get("value") return output_data diff --git a/Babylon/main.py b/Babylon/main.py index d4cd4364..3d974cef 100644 --- a/Babylon/main.py +++ b/Babylon/main.py @@ -55,19 +55,17 @@ def setup_logging(log_path: pathlibPath = pathlibPath.cwd()) -> None: date_format = "%Y-%m-%d %H:%M:%S" file_formatter = CleanFormatter(fmt=file_format, datefmt=date_format) - log_file_handler = logging.FileHandler(log_path / "babylon_info.log", encoding="utf-8") - log_file_handler.setLevel(logging.INFO) - log_file_handler.setFormatter(file_formatter) + file_handler = logging.FileHandler(log_path / "babylon.log", encoding="utf-8") + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(file_formatter) - error_file_handler = logging.FileHandler(log_path / "babylon_error.log", encoding="utf-8") - error_file_handler.setLevel(logging.WARNING) - error_file_handler.setFormatter(file_formatter) logging.basicConfig( + level=logging.DEBUG, format="%(message)s", handlers=[ - log_file_handler, - error_file_handler, + file_handler, RichHandler( + level=logging.INFO, show_time=False, rich_tracebacks=True, tracebacks_suppress=[click], diff --git a/Babylon/templates/working_dir/.templates/yaml/Dataset.yaml b/Babylon/templates/working_dir/.templates/yaml/Dataset.yaml deleted file mode 100644 index 9f7342e2..00000000 --- a/Babylon/templates/working_dir/.templates/yaml/Dataset.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: Dataset -spec: - sidecars: - payload: - id: "" - name: "{{dataset_name}}" - description: "{{dataset_description}}" - tags: - - brewery - parts: - - name: part1 - sourceName: customers.csv - security: {{security}} \ No newline at end of file diff --git a/Babylon/templates/working_dir/.templates/yaml/Runner.yaml b/Babylon/templates/working_dir/.templates/yaml/Runner.yaml deleted file mode 100644 index b7b18942..00000000 --- a/Babylon/templates/working_dir/.templates/yaml/Runner.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: Runner -namespace: - remote: false -spec: - sidecars: - payload: - id: "" - name: "{{runner_name}}" - runTemplateId: "{{run_template_id}}" - solutionId: "{{services['api.solution_id']}}" - solutionName: "{{solution_name}}" - runTemplateName: "{{runTemplate_name}}" - ownerName: "{{owner_name}}" - security: {{security}} \ No newline at end of file diff --git a/Babylon/templates/working_dir/.templates/yaml/Solution.yaml b/Babylon/templates/working_dir/.templates/yaml/Solution.yaml index e6a9c795..a94a2a7c 100644 --- a/Babylon/templates/working_dir/.templates/yaml/Solution.yaml +++ b/Babylon/templates/working_dir/.templates/yaml/Solution.yaml @@ -18,8 +18,8 @@ spec: tags: - brewery runTemplates: - - id: "{{run_template_id}}" - name: "{{runTemplate_name}}" + - id: standalone + name: Standard simulation csmSimulation: AzureWebApp/AzureWebApp_Simulation run: true preRun: true diff --git a/Babylon/templates/working_dir/.templates/yaml/Workspace.yaml b/Babylon/templates/working_dir/.templates/yaml/Workspace.yaml index 554ac2b0..dca0dad4 100644 --- a/Babylon/templates/working_dir/.templates/yaml/Workspace.yaml +++ b/Babylon/templates/working_dir/.templates/yaml/Workspace.yaml @@ -3,6 +3,12 @@ namespace: remote: false spec: sidecars: + postgres: + schema: + create: false + jobs: + - name: k8s_job.yaml + path: postgres/jobs payload: key: "{{workspace_key}}" name: "{{workspace_name}}" diff --git a/Babylon/templates/working_dir/.templates/yaml/k8s_job.yaml b/Babylon/templates/working_dir/.templates/yaml/k8s_job.yaml new file mode 100644 index 00000000..be94c890 --- /dev/null +++ b/Babylon/templates/working_dir/.templates/yaml/k8s_job.yaml @@ -0,0 +1,37 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: postgresql-init-${job_name} + namespace: ${namespace} +spec: + ttlSecondsAfterFinished: 600 + template: + spec: + containers: + - name: postgresql-init + image: postgres:17-trixie + command: ["/bin/sh", "-c"] + args: + - | + set -e + echo "INFO: Starting schema initialization for ${workspace_schema}" + + SCHEMA_EXISTS=$(psql -h ${db_host} -p ${db_port} -U ${cosmotech_api_admin_username} -d ${cosmotech_api_database} \ + -tAc "SELECT 1 FROM information_schema.schemata WHERE schema_name = '${workspace_schema}'") + + if [ "$SCHEMA_EXISTS" = "1" ]; then + echo "INFO: Schema ${workspace_schema} already exists. Skipping creation." + else + psql -h ${db_host} -p ${db_port} -U ${cosmotech_api_admin_username} -d ${cosmotech_api_database} -c "CREATE SCHEMA ${workspace_schema} AUTHORIZATION ${cosmotech_api_writer_username};" + echo "SUCCESS: Schema created successfully" + fi + + psql -h ${db_host} -p ${db_port} -U ${cosmotech_api_admin_username} -d ${cosmotech_api_database} -c "GRANT USAGE ON SCHEMA ${workspace_schema} TO ${cosmotech_api_reader_username};" + echo "SUCCESS: Permissions granted" + env: + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: postgresql-cosmotechapi + key: admin-password + restartPolicy: Never \ No newline at end of file diff --git a/Babylon/templates/working_dir/.templates/yaml/k8s_job_destroy.yaml b/Babylon/templates/working_dir/.templates/yaml/k8s_job_destroy.yaml new file mode 100644 index 00000000..3ba47fd2 --- /dev/null +++ b/Babylon/templates/working_dir/.templates/yaml/k8s_job_destroy.yaml @@ -0,0 +1,39 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: postgresql-destroy-${job_name} + namespace: ${namespace} +spec: + ttlSecondsAfterFinished: 180 + backoffLimit: 1 + template: + spec: + containers: + - name: postgresql-cleanup + image: postgres:17-trixie + command: ["/bin/sh", "-c"] + args: + - | + set -e + echo "INFO: Starting cleanup for schema ${workspace_schema}" + SCHEMA_EXISTS=$(psql -h ${db_host} -p ${db_port} -U ${cosmotech_api_admin_username} -d ${cosmotech_api_database} -tAc "SELECT 1 FROM information_schema.schemata WHERE schema_name = '${workspace_schema}'") + + if [ "$SCHEMA_EXISTS" != "1" ]; then + echo "INFO: Schema ${workspace_schema} does not exist. Nothing to clean." + exit 0 + fi + psql -h ${db_host} -p ${db_port} -U ${cosmotech_api_admin_username} -d ${cosmotech_api_database} \ + -c "REVOKE ALL PRIVILEGES ON SCHEMA ${workspace_schema} FROM ${cosmotech_api_reader_username}, ${cosmotech_api_writer_username};" \ + && echo "SUCCESS: Permissions revoked successfully" || echo "ERROR: Failed to revoke permissions" + + psql -h ${db_host} -p ${db_port} -U ${cosmotech_api_admin_username} -d ${cosmotech_api_database} \ + -c "DROP SCHEMA IF EXISTS ${workspace_schema} CASCADE;" \ + && echo "SUCCESS: Schema dropped successfully" || echo "ERROR: Failed to drop schema" + + env: + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: postgresql-cosmotechapi + key: admin-password + restartPolicy: Never \ No newline at end of file diff --git a/Babylon/templates/working_dir/.templates/yaml/variables.yaml b/Babylon/templates/working_dir/.templates/yaml/variables.yaml index 8dde224c..24b100fb 100644 --- a/Babylon/templates/working_dir/.templates/yaml/variables.yaml +++ b/Babylon/templates/working_dir/.templates/yaml/variables.yaml @@ -14,14 +14,6 @@ simulator_version: latest workspace_name: Babylon v5 workspace workspace_key: brewerytestingwork workspace_description: Testing workspace for the brewery web application -# Dataset -dataset_name: minimal -dataset_description: Babylon v5 Dataset -# Runner -run_template_id: standalone -runner_name: Babylon v5 Runner -runTemplate_name: Standard simulation -owner_name: toto # Webapp cloud_provider: azure cluster_name: warpvy52ww diff --git a/Babylon/utils/credentials.py b/Babylon/utils/credentials.py index de0705ac..776b8d2a 100644 --- a/Babylon/utils/credentials.py +++ b/Babylon/utils/credentials.py @@ -91,9 +91,7 @@ def get_keycloak_credentials() -> tuple[dict, dict]: } if not all(credentials.values()): missing = [k for k, v in credentials.items() if not v] - raise AttributeError( - f" [bold red]✘[/bold red] Missing required Keycloak credentials: {', '.join(missing)}" - ) + raise AttributeError(f" [bold red]✘[/bold red] Missing required Keycloak credentials: {', '.join(missing)}") return credentials, config diff --git a/Babylon/utils/environment.py b/Babylon/utils/environment.py index 80df274b..5b005564 100644 --- a/Babylon/utils/environment.py +++ b/Babylon/utils/environment.py @@ -139,7 +139,7 @@ def set_blob_client(self): logger.error(f" [bold red]✘[/bold red] Failed to initialize BlobServiceClient: {e}") sys.exit(1) - def get_config_from_k8s_secret_by_tenant(self, tenant: str): + def get_config_from_k8s_secret_by_tenant(self, secret_name: str, tenant: str): response_parsed = {} try: config.load_kube_config() @@ -152,12 +152,10 @@ def get_config_from_k8s_secret_by_tenant(self, tenant: str): sys.exit(1) try: v1 = client.CoreV1Api() - secret = v1.read_namespaced_secret(name="keycloak-babylon", namespace=tenant) + secret = v1.read_namespaced_secret(name=secret_name, namespace=tenant) except ApiException: logger.error("\n [bold red]✘[/bold red] Resource Not Found") - logger.error( - f" Secret [green]keycloak-babylon[/green] could not be found in namespace [green]{tenant}[/green]" - ) + logger.error(f" Secret [green]{secret_name}[/green] could not be found in namespace [green]{tenant}[/green]") logger.info("\n [bold white]💡 Troubleshooting:[/bold white]") logger.info(" • Please ensure your kubeconfig is valid") logger.info(" • Check that your context is correctly set [cyan]kubectl config current-context[/cyan]") @@ -174,7 +172,7 @@ def get_config_from_k8s_secret_by_tenant(self, tenant: str): decoded_value = b64decode(value).decode("utf-8") response_parsed[key] = decoded_value else: - logger.warning(f" [yellow]⚠[/yellow] Secret 'keycloak-babylon' in namespace '{tenant}' has no data") + logger.warning(f" [yellow]⚠[/yellow] Secret {secret_name} in namespace '{tenant}' has no data") return response_parsed def store_state_in_local(self, state: dict): @@ -223,6 +221,9 @@ def get_state_from_local(self): "webapp_name": "", "webapp_url": "", }, + "postgres": { + "schema_name": "", + }, }, } state_data = load(state_file.open("r"), Loader=SafeLoader) @@ -248,6 +249,9 @@ def get_state_from_cloud(self) -> dict: "webapp_name": "", "webapp_url": "", }, + "postgres": { + "schema_name": "", + }, }, } data = load(state_blob.download_blob().readall(), Loader=SafeLoader) @@ -299,7 +303,7 @@ def retrieve_config(self): } # Log missing env vars logger.info(" [dim]→ Loading configuration from Kubernetes secret... [/dim]") - return self.get_config_from_k8s_secret_by_tenant(self.environ_id) + return self.get_config_from_k8s_secret_by_tenant("keycloak-babylon", self.environ_id) def retrieve_state_func(self): if self.remote: diff --git a/Babylon/utils/interactive.py b/Babylon/utils/interactive.py index 148f2b84..8d27c79e 100644 --- a/Babylon/utils/interactive.py +++ b/Babylon/utils/interactive.py @@ -81,9 +81,7 @@ def confirm_deletion(entity_type: str, entity_id: str) -> bool: :param entity_id: Entity ID :return: Should execution continue ? """ - prompt_text = click.style( - f"You are trying to delete {entity_type} {entity_id} \nDo you want to continue?", fg="red", bold=True - ) + prompt_text = click.style(f"You are trying to delete {entity_type} {entity_id} \nDo you want to continue?", fg="red", bold=True) if not click.confirm(prompt_text): logger.info(f"{entity_type} deletion aborted.") return False diff --git a/Babylon/utils/request.py b/Babylon/utils/request.py index 42cc3452..a6eccb4b 100644 --- a/Babylon/utils/request.py +++ b/Babylon/utils/request.py @@ -41,9 +41,7 @@ def oauth_request( logger.error("[api] Unauthorized (401): token missing or expired. Refresh token or check client credentials.") return None if response.status_code == 403: - logger.error( - "[api] Forbidden (403): token valid but lacks required roles/scopes. Check Keycloak client permissions." - ) + logger.error("[api] Forbidden (403): token valid but lacks required roles/scopes. Check Keycloak client permissions.") return None if not response.ok: logger.warning(f"[api] Request failed ({response.status_code}): {response.text}") diff --git a/Babylon/utils/response.py b/Babylon/utils/response.py index 5185408e..ff59ae9e 100644 --- a/Babylon/utils/response.py +++ b/Babylon/utils/response.py @@ -88,9 +88,7 @@ def print_table(self): return is_api_info = self._is_api_info_type(items[0]) - table = Table( - show_header=True, header_style="bold white", box=None, padding=(0, 2), show_edge=False, expand=False - ) + table = Table(show_header=True, header_style="bold white", box=None, padding=(0, 2), show_edge=False, expand=False) if is_api_info: table.add_column("PROPERTY") table.add_column("VALUE") diff --git a/pyproject.toml b/pyproject.toml index 09f5a026..30898ffb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ babylon = "Babylon.main:main" path = "Babylon/version.py" [tool.ruff] -line-length = 120 +line-length = 135 [tool.ruff.lint] select = ["E", "F", "I"] diff --git a/tests/integration/test_api_endpoints.sh b/tests/integration/test_api_endpoints.sh index c9a8c50f..a229fa13 100755 --- a/tests/integration/test_api_endpoints.sh +++ b/tests/integration/test_api_endpoints.sh @@ -87,4 +87,4 @@ babylon api solutions delete --oid $O --sid $S babylon api organizations delete --oid $O rm -rf ./output -rm babylon_error.log babylon_info.log \ No newline at end of file +rm babylon.log \ No newline at end of file diff --git a/tests/unit/test_macro.py b/tests/unit/test_macro.py index 19f27254..8714b1ce 100644 --- a/tests/unit/test_macro.py +++ b/tests/unit/test_macro.py @@ -82,6 +82,7 @@ def test_resolve_inclusion_exclusion_include_duplicates(): False, ) + def test_resolve_inclusion_exclusion_invalid_exclude(): with pytest.raises(Abort): resolve_inclusion_exclusion(include=(), exclude=("invalid",))