Pass the ID in the query if it has invalid chars.

cwms/catalog/blobs.py

@@ -1,9 +1,9 @@
 import base64
-from typing import Optional
+from typing import Any, Optional
 
 import cwms.api as api
 from cwms.cwms_types import JSON, Data
-from cwms.utils.checks import is_base64
+from cwms.utils.checks import has_invalid_chars, is_base64
 
 STORE_DICT = """data = {
     "office-id": "SWT",
@@ -14,6 +14,8 @@
 }
 """
 
+IGNORED_ID = "ignored"
+
 
 def get_blob(blob_id: str, office_id: str) -> str:
     """Get a single BLOB (Binary Large Object).
@@ -29,8 +31,13 @@ def get_blob(blob_id: str, office_id: str) -> str:
         str: the value returned based on the content-type it was stored with as a string
     """
 
-    endpoint = f"blobs/{blob_id}"
-    params = {"office": office_id}
+    params: dict[str, Any] = {}
+    if has_invalid_chars(blob_id):
+        endpoint = f"blobs/{IGNORED_ID}"
+        params["blob-id"] = blob_id
+    else:
+        endpoint = f"blobs/{blob_id}"
+    params["office"] = office_id
     response = api.get(endpoint, params, api_version=1)
     return str(response)
 
@@ -107,8 +114,13 @@ def delete_blob(blob_id: str, office_id: str) -> None:
         None
     """
 
-    endpoint = f"blobs/{blob_id}"
-    params = {"office": office_id}
+    params: dict[str, Any] = {}
+    if has_invalid_chars(blob_id):
+        endpoint = f"blobs/{IGNORED_ID}"
+        params["blob-id"] = blob_id
+    else:
+        endpoint = f"blobs/{blob_id}"
+    params["office"] = office_id
     return api.delete(endpoint, params, api_version=1)
 
 
@@ -143,6 +155,11 @@ def update_blob(data: JSON, fail_if_not_exists: Optional[bool] = True) -> None:
 
     blob_id = data.get("id", "").upper()
 
-    endpoint = f"blobs/{blob_id}"
-    params = {"fail-if-not-exists": fail_if_not_exists}
+    params: dict[str, Any] = {}
+    if has_invalid_chars(blob_id):
+        endpoint = f"blobs/{IGNORED_ID}"
+        params["blob-id"] = blob_id
+    else:
+        endpoint = f"blobs/{blob_id}"
+    params["fail-if-not-exists"] = fail_if_not_exists
     return api.patch(endpoint, data, params, api_version=1)

cwms/catalog/clobs.py

@@ -1,10 +1,21 @@
-from typing import Optional
+from typing import Any, Optional
 
 import cwms.api as api
 from cwms.cwms_types import JSON, Data
+from cwms.utils.checks import has_invalid_chars
 
+STORE_DICT = """data = {
+    "office-id": "SWT",
+    "id": "CLOB_ID",
+    "description": "Your description here",
+    "value": "STRING of content"
+}
+"""
 
-def get_clob(clob_id: str, office_id: str, clob_id_query: Optional[str] = None) -> Data:
+IGNORED_ID = "ignored"
+
+
+def get_clob(clob_id: str, office_id: str) -> Data:
     """Get a single clob.
 
     Parameters
@@ -13,28 +24,20 @@ def get_clob(clob_id: str, office_id: str, clob_id_query: Optional[str] = None)
                 Specifies the id of the clob
             office_id: string
                 Specifies the office of the clob.
-            clob_id_query: string
-                If this query parameter is provided the id path parameter is ignored and the
-                value of the query parameter is used. Note: this query parameter is necessary
-                for id's that contain '/' or other special characters. Because of abuse even
-                properly escaped '/' in url paths are blocked. When using this query parameter
-                a valid path parameter must still be provided for the request to be properly
-                routed. If your clob id contains '/' you can't specify the clob-id query
-                parameter and also specify the id path parameter because firewall and/or server
-                rules will deny the request even though you are specifying this override. "ignored"
-                is suggested.
 
 
         Returns
         -------
             cwms data type.  data.json will return the JSON output and data.df will return a dataframe
     """
 
-    endpoint = f"clobs/{clob_id}"
-    params = {
-        "office": office_id,
-        "clob-id-query": clob_id_query,
-    }
+    params: dict[str, Any] = {}
+    if has_invalid_chars(clob_id):
+        endpoint = f"clobs/{IGNORED_ID}"
+        params["clob-id"] = clob_id
+    else:
+        endpoint = f"clobs/{clob_id}"
+    params["office"] = office_id
     response = api.get(endpoint, params)
     return Data(response)
 
@@ -90,13 +93,20 @@ def delete_clob(clob_id: str, office_id: str) -> None:
             None
     """
 
-    endpoint = f"clobs/{clob_id}"
-    params = {"office": office_id}
+    params: dict[str, Any] = {}
+    if has_invalid_chars(clob_id):
+        endpoint = f"clobs/{IGNORED_ID}"
+        params["clob-id"] = clob_id
+    else:
+        endpoint = f"clobs/{clob_id}"
+    params["office"] = office_id
 
     return api.delete(endpoint, params=params, api_version=1)
 
 
-def update_clob(data: JSON, clob_id: str, ignore_nulls: Optional[bool] = True) -> None:
+def update_clob(
+    data: JSON, clob_id: Optional[str] = None, ignore_nulls: Optional[bool] = True
+) -> None:
     """Updates clob
 
     Parameters
@@ -110,7 +120,7 @@ def update_clob(data: JSON, clob_id: str, ignore_nulls: Optional[bool] = True) -
                         "value": "string"
                     }
             clob_id: string
-                Specifies the id of the clob to be deleted
+                Specifies the id of the clob to be deleted. Unused if "id" is present in JSON data.
             ignore_nulls: Boolean
                 If true, null and empty fields in the provided clob will be ignored and the existing value of those fields left in place. Default: true
 
@@ -122,8 +132,19 @@ def update_clob(data: JSON, clob_id: str, ignore_nulls: Optional[bool] = True) -
     if not isinstance(data, dict):
         raise ValueError("Cannot store a Clob without a JSON data dictionary")
 
-    endpoint = f"clobs/{clob_id}"
-    params = {"ignore-nulls": ignore_nulls}
+    if "id" in data:
+        clob_id = data.get("id", "").upper()
+
+    if clob_id is None:
+        raise ValueError(f"Cannot update a Clob without an 'id' field:\n{STORE_DICT}")
+
+    params: dict[str, Any] = {}
+    if has_invalid_chars(clob_id):
+        endpoint = f"clobs/{IGNORED_ID}"
+        params["clob-id"] = clob_id
+    else:
+        endpoint = f"clobs/{clob_id}"
+    params["ignore-nulls"] = ignore_nulls
 
     return api.patch(endpoint, data, params, api_version=1)
 

cwms/utils/checks.py

@@ -8,3 +8,15 @@ def is_base64(s: str) -> bool:
         return base64.b64encode(decoded).decode("utf-8") == s
     except (ValueError, TypeError):
         return False
+
+
+def has_invalid_chars(id: str) -> bool:
+    """
+    Checks if ID contains any invalid web path characters.
+    """
+    INVALID_PATH_CHARS = ["/", "\\", "&", "?", "="]
+
+    for char in INVALID_PATH_CHARS:
+        if char in id:
+            return True
+    return False

docker-compose.yml

@@ -3,22 +3,27 @@ volumes:
   auth_data:
 services:
   db:
-    image: ghcr.io/hydrologicengineeringcenter/cwms-database/cwms/database-ready-ora-23.5:latest-dev
+    image: ghcr.io/hydrologicengineeringcenter/cwms-database/cwms/database-ready-ora-23.5:develop-nightly
     environment:
       #- ORACLE_DATABASE=FREEPDB1
       - ORACLE_PASSWORD=badSYSpassword
       - CWMS_PASSWORD=simplecwmspasswD1
       - OFFICE_ID=HQ
       - OFFICE_EROC=s0
-    ports: ["1526:1521"]
+    ports:
+      - "1526:1521"
     healthcheck:
-      test: ["CMD", "tnsping", "FREEPDB1"]
+      test:
+        [
+          "CMD-SHELL",
+          "sqlplus -s -L sys/badSYSpassword@localhost:1521/FREEPDB1 as sysdba <<< 'exit;'",
+        ]
       interval: 30s
       timeout: 50s
       retries: 50
       start_period: 40m
   db_webuser_permissions:
-    image: ghcr.io/hydrologicengineeringcenter/cwms-database/cwms/schema_installer:latest-dev
+    image: ${CWMS_SCHEMA_INSTALLER_IMAGE:-registry-public.hecdev.net/cwms/schema_installer:latest-dev}
     restart: "no"
     environment:
       - DB_HOST_PORT=db:1521
@@ -31,16 +36,12 @@ services:
       - INSTALLONCE=1
       - QUIET=1
     command: >
-      sh -xc "sqlplus CWMS_20/$$CWMS_PASSWORD@$$DB_HOST_PORT$$DB_NAME @/setup_sql/users
-      $$OFFICE_EROC"
-    volumes: [./compose_files/sql:/setup_sql:ro]
+      sh -xc "sqlplus CWMS_20/$$CWMS_PASSWORD@$$DB_HOST_PORT$$DB_NAME @/setup_sql/users $$OFFICE_EROC"
+    volumes:
+      - ./compose_files/sql:/setup_sql:ro
     depends_on:
       db:
         condition: service_healthy
-      auth:
-        condition: service_healthy
-      traefik:
-        condition: service_healthy
 
   data-api:
     depends_on:
@@ -58,6 +59,7 @@ services:
       - ./compose_files/pki/certs:/conf/
       - ./compose_files/tomcat/logging.properties:/usr/local/tomcat/conf/logging.properties:ro
     environment:
+      - JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005
       - CDA_JDBC_DRIVER=oracle.jdbc.driver.OracleDriver
       - CDA_JDBC_URL=jdbc:oracle:thin:@db/FREEPDB1
       - CDA_JDBC_USERNAME=s0webtest
@@ -73,9 +75,12 @@ services:
       - cwms.dataapi.access.openid.altAuthUrl=http://localhost:${APP_PORT:-8082}
       - cwms.dataapi.access.openid.useAltWellKnown=true
       - cwms.dataapi.access.openid.issuer=http://localhost:${APP_PORT:-8082}/auth/realms/cwms
-    expose: [7000]
+    expose:
+      - 7000
+      - 5005
     healthcheck:
-      test: ["CMD", "/usr/bin/curl", "-I", "localhost:7000/cwms-data/offices/HEC"]
+      test:
+        ["CMD", "/usr/bin/curl", "-I", "localhost:7000/cwms-data/offices/HEC"]
       interval: 5s
       timeout: 1s
       retries: 100
@@ -84,15 +89,13 @@ services:
       - "traefik.enable=true"
       - "traefik.http.routers.data-api.rule=PathPrefix(`/cwms-data`)"
       - "traefik.http.routers.data-api.entryPoints=web"
+      - "traefik.http.services.data-api.loadbalancer.server.port=7000"
 
   auth:
     image: quay.io/keycloak/keycloak:19.0.1
-    command: ["start-dev", "--import-realm"]
+    command: ["start-dev", "--features-disabled=admin2", "--import-realm"]
     healthcheck:
-      test:
-        - "CMD-SHELL"
-        - "/usr/bin/curl -If localhost:${APP_PORT:-8082}/auth/health/ready || exit\
-          \ 1"
+      test: "/usr/bin/curl -If localhost:${APP_PORT:-8082}/auth/health/ready || exit 1"
       interval: 5s
       timeout: 1s
       retries: 100
@@ -108,8 +111,6 @@ services:
       - KC_PROXY=none
       - KC_HTTP_ENABLED=true
       - KC_HTTP_RELATIVE_PATH=/auth
-      - KC_HOSTNAME=localhost
-      - KC_DB=dev-file
     volumes:
       - ./compose_files/keycloak/realm.json:/opt/keycloak/data/import/realm.json:ro
     labels:
@@ -123,10 +124,13 @@ services:
 
   # Proxy for HTTPS for OpenID
   traefik:
-    image: traefik:v3.6.2
-    ports: ["${APP_PORT:-8082}:80"]
-    expose: ["8080"]
-    volumes: ["/var/run/docker.sock:/var/run/docker.sock:ro"]
+    image: "traefik:v3.6.2"
+    ports:
+      - "${APP_PORT:-8082}:80"
+    expose:
+      - "8081"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
     healthcheck:
       test: traefik healthcheck --ping
     command: