From 213d25a10cfbb44f77ff4058d21a8991727e3e92 Mon Sep 17 00:00:00 2001 From: Yousef Abdulwahab Date: Wed, 1 Jan 2025 13:39:00 +0300 Subject: [PATCH 1/5] feat: add clickhouse suppurt --- .../type_helpers.sql | 69 +++++++++++++++++++ .../upload_exposures.sql | 4 ++ .../upload_invocations.sql | 3 + .../upload_model_executions.sql | 3 + .../upload_models.sql | 3 + .../upload_seed_executions.sql | 3 + .../upload_seeds.sql | 3 + .../upload_snapshot_executions.sql | 3 + .../upload_snapshots.sql | 3 + .../upload_sources.sql | 3 + .../upload_test_executions.sql | 4 +- .../upload_tests.sql | 3 + .../insert_into_metadata_table.sql | 4 ++ models/sources/exposures.sql | 12 +++- models/sources/models.sql | 12 +++- models/sources/snapshots.sql | 6 +- models/sources/tests.sql | 13 +++- 17 files changed, 143 insertions(+), 8 deletions(-) diff --git a/macros/database_specific_helpers/type_helpers.sql b/macros/database_specific_helpers/type_helpers.sql index 4064ad46..22e119f1 100644 --- a/macros/database_specific_helpers/type_helpers.sql +++ b/macros/database_specific_helpers/type_helpers.sql @@ -8,6 +8,10 @@ {{ return(api.Column.translate_type("boolean")) }} {% endmacro %} +{% macro clickhouse__type_boolean() %} + Nullable(Boolean) +{% endmacro %} + {#- JSON -#} {% macro type_json() %} @@ -26,6 +30,10 @@ json {% endmacro %} +{% macro clickhouse__type_json() %} + Nullable(String) +{% endmacro %} + {#- ARRAY -#} {% macro type_array() %} @@ -43,3 +51,64 @@ {% macro bigquery__type_array() %} array {% endmacro %} + +{% macro clickhouse__type_array() %} + Array(String) +{% endmacro %} + +{#- STRING -#} + +{% macro type_string() %} + {{ return(adapter.dispatch('type_string', 'dbt_artifacts')()) }} +{% endmacro %} + +{% macro default__type_string() %} + {{ return(api.Column.translate_type("string")) }} +{% endmacro %} + +{% macro clickhouse__type_string() %} + Nullable(String) +{% endmacro %} + +{#- TIMESTAMP -#} + +{% macro type_timestamp() %} + {{ return(adapter.dispatch('type_timestamp', 'dbt_artifacts')()) }} +{% endmacro %} + +{% macro default__type_timestamp() %} + {{ return(api.Column.translate_type("timestamp")) }} +{% endmacro %} + +{% macro clickhouse__type_timestamp() %} + Nullable(DateTime) +{% endmacro %} + +{#- INT -#} + +{% macro type_int() %} + {{ return(adapter.dispatch('type_int', 'dbt_artifacts')()) }} +{% endmacro %} + +{% macro default__type_int() %} + {{ return(api.Column.translate_type("integer")) }} +{% endmacro %} + +{% macro clickhouse__type_int() %} + Nullable(Int32) +{% endmacro %} + +{#- FLOAT -#} + +{% macro type_float() %} + {{ return(adapter.dispatch('type_float', 'dbt_artifacts')()) }} +{% endmacro %} + +{% macro default__type_float() %} + {{ return(api.Column.translate_type("float")) }} +{% endmacro %} + +{% macro clickhouse__type_float() %} + Nullable(Float32) +{% endmacro %} + diff --git a/macros/upload_individual_datasets/upload_exposures.sql b/macros/upload_individual_datasets/upload_exposures.sql index e097be7e..43bcaf44 100644 --- a/macros/upload_individual_datasets/upload_exposures.sql +++ b/macros/upload_individual_datasets/upload_exposures.sql @@ -154,3 +154,7 @@ {% endif %} {% endmacro -%} +{% macro clickhouse__get_exposures_dml_sql(exposures) -%} +{{ return(dbt_artifacts.postgres__get_exposures_dml_sql(exposures)) }} +{%- endmacro %} + diff --git a/macros/upload_individual_datasets/upload_invocations.sql b/macros/upload_individual_datasets/upload_invocations.sql index 775861ac..5d1d0d4d 100644 --- a/macros/upload_individual_datasets/upload_invocations.sql +++ b/macros/upload_individual_datasets/upload_invocations.sql @@ -296,3 +296,6 @@ {% endmacro -%} +{% macro clickhouse__get_invocations_dml_sql() -%} +{{ return(dbt_artifacts.postgres__get_invocations_dml_sql()) }} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index 1d5fa83b..5f559810 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -246,3 +246,6 @@ {% endif %} {% endmacro -%} +{% macro clickhouse__get_model_executions_dml_sql(models) -%} +{{ return(dbt_artifacts.postgres__get_model_executions_dml_sql(models)) }} +{% endmacro -%} \ No newline at end of file diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql index 2c27daec..dd34b588 100644 --- a/macros/upload_individual_datasets/upload_models.sql +++ b/macros/upload_individual_datasets/upload_models.sql @@ -168,3 +168,6 @@ {% endif %} {% endmacro -%} +{% macro clickhouse__get_models_dml_sql(models) -%} +{{ return(dbt_artifacts.postgres__get_models_dml_sql(models)) }} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql index c19a9a2f..754969fe 100644 --- a/macros/upload_individual_datasets/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -260,3 +260,6 @@ {% endif %} {% endmacro -%} +{% macro clickhouse__get_seed_executions_dml_sql(seeds) -%} +{{ return(dbt_artifacts.postgres__get_seed_executions_dml_sql(seeds)) }} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_seeds.sql b/macros/upload_individual_datasets/upload_seeds.sql index 2501a9e9..396aeb97 100644 --- a/macros/upload_individual_datasets/upload_seeds.sql +++ b/macros/upload_individual_datasets/upload_seeds.sql @@ -143,3 +143,6 @@ {% endif %} {% endmacro -%} +{% macro clickhouse__get_seeds_dml_sql(seeds) -%} +{{ return(dbt_artifacts.postgres__get_seeds_dml_sql(seeds)) }} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_snapshot_executions.sql b/macros/upload_individual_datasets/upload_snapshot_executions.sql index b0af313b..2eb389f6 100644 --- a/macros/upload_individual_datasets/upload_snapshot_executions.sql +++ b/macros/upload_individual_datasets/upload_snapshot_executions.sql @@ -260,3 +260,6 @@ {% endif %} {% endmacro -%} +{% macro clickhouse__get_snapshot_executions_dml_sql(snapshots) -%} +{{ return(dbt_artifacts.postgres__get_snapshot_executions_dml_sql(snapshots)) }} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_snapshots.sql b/macros/upload_individual_datasets/upload_snapshots.sql index e540dc0b..3fd7365f 100644 --- a/macros/upload_individual_datasets/upload_snapshots.sql +++ b/macros/upload_individual_datasets/upload_snapshots.sql @@ -157,3 +157,6 @@ {% endif %} {% endmacro -%} +{% macro clickhouse__get_snapshots_dml_sql(snapshots) -%} +{{ return(dbt_artifacts.postgres__get_snapshots_dml_sql(snapshots)) }} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_sources.sql b/macros/upload_individual_datasets/upload_sources.sql index 3d899755..6adaf117 100644 --- a/macros/upload_individual_datasets/upload_sources.sql +++ b/macros/upload_individual_datasets/upload_sources.sql @@ -107,6 +107,9 @@ {% endif %} {%- endmacro %} +{% macro clickhouse__get_sources_dml_sql(sources) -%} +{{ return(dbt_artifacts.postgres__get_sources_dml_sql(sources)) }} +{%- endmacro %} {% macro sqlserver__get_sources_dml_sql(sources) -%} diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index 3af31d73..6e345246 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -241,4 +241,6 @@ {% endif %} {% endmacro -%} - +{% macro clickhouse__get_test_executions_dml_sql(tests) -%} +{{ return(dbt_artifacts.postgres__get_test_executions_dml_sql(tests)) }} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_tests.sql b/macros/upload_individual_datasets/upload_tests.sql index 18d7171c..91bb7c27 100644 --- a/macros/upload_individual_datasets/upload_tests.sql +++ b/macros/upload_individual_datasets/upload_tests.sql @@ -95,6 +95,9 @@ {% endif %} {%- endmacro %} +{% macro clickhouse__get_tests_dml_sql(tests) -%} +{{ return(dbt_artifacts.postgres__get_tests_dml_sql(tests)) }} +{%- endmacro %} {% macro sqlserver__get_tests_dml_sql(tests) -%} diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index 8d4afa40..97c854fc 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -70,6 +70,10 @@ {%- endmacro %} +{% macro clickhouse__insert_into_metadata_table(relation, fields, content) -%} +{{ return(dbt_artifacts.postgres__insert_into_metadata_table(relation, fields, content)) }} +{%- endmacro %} + {% macro default__insert_into_metadata_table(relation, fields, content) -%} {%- endmacro %} diff --git a/models/sources/exposures.sql b/models/sources/exposures.sql index 6d593bcf..7c87f47a 100644 --- a/models/sources/exposures.sql +++ b/models/sources/exposures.sql @@ -18,8 +18,16 @@ select , cast(null as {{ type_string() }}) as description , cast(null as {{ type_string() }}) as url , cast(null as {{ type_string() }}) as package_name - , cast(null as {{ type_array() }}) as depends_on_nodes - , cast(null as {{ type_array() }}) as tags + {% if target.type == "clickhouse" %} + , cast([] as {{ type_array() }}) as depends_on_nodes + {% else %} + , cast(null as {{ type_array() }}) as depends_on_nodes + {% endif %} + {% if target.type == "clickhouse" %} + , cast([] as {{ type_array() }}) as tags + {% else %} + , cast(null as {{ type_array() }}) as tags + {% endif %} , cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 diff --git a/models/sources/models.sql b/models/sources/models.sql index 7ac8ee84..4032d976 100644 --- a/models/sources/models.sql +++ b/models/sources/models.sql @@ -12,12 +12,20 @@ select {% else %} schema {% endif %}, cast(null as {{ type_string() }}) as name, - cast(null as {{ type_array() }}) as depends_on_nodes, + {% if target.type == "clickhouse" %} + cast([] as {{ type_array() }}) as depends_on_nodes, + {% else %} + cast(null as {{ type_array() }}) as depends_on_nodes, + {% endif %} cast(null as {{ type_string() }}) as package_name, cast(null as {{ type_string() }}) as path, cast(null as {{ type_string() }}) as checksum, cast(null as {{ type_string() }}) as materialization, - cast(null as {{ type_array() }}) as tags, + {% if target.type == "clickhouse" %} + cast([] as {{ type_array() }}) as tags, + {% else %} + cast(null as {{ type_array() }}) as tags, + {% endif %} cast(null as {{ type_json() }}) as meta, cast(null as {{ type_string() }}) as alias, cast(null as {{ type_json() }}) as all_results diff --git a/models/sources/snapshots.sql b/models/sources/snapshots.sql index 77061731..0d53bd1a 100644 --- a/models/sources/snapshots.sql +++ b/models/sources/snapshots.sql @@ -12,7 +12,11 @@ select {% else %} schema {% endif %}, cast(null as {{ type_string() }}) as name, - cast(null as {{ type_array() }}) as depends_on_nodes, + {% if target.type == "clickhouse" %} + cast([] as {{ type_array() }}) as depends_on_nodes, + {% else %} + cast(null as {{ type_array() }}) as depends_on_nodes, + {% endif %} cast(null as {{ type_string() }}) as package_name, cast(null as {{ type_string() }}) as path, cast(null as {{ type_string() }}) as checksum, diff --git a/models/sources/tests.sql b/models/sources/tests.sql index d46ed1dc..58f783d7 100644 --- a/models/sources/tests.sql +++ b/models/sources/tests.sql @@ -11,10 +11,19 @@ select , cast(null as {{ type_string() }}) as node_id , cast(null as {{ type_timestamp() }}) as run_started_at , cast(null as {{ type_string() }}) as name - , cast(null as {{ type_array() }}) as depends_on_nodes + {% if target.type == "clickhouse" %} + , cast([] as {{ type_array() }}) as depends_on_nodes + {% else %} + , cast(null as {{ type_array() }}) as depends_on_nodes + {% endif %} , cast(null as {{ type_string() }}) as package_name , cast(null as {{ type_string() }}) as test_path - , cast(null as {{ type_array() }}) as tags + {% if target.type == "clickhouse" %} + , cast([] as {{ type_array() }}) as tags + {% else %} + , cast(null as {{ type_array() }}) as tags + {% endif %} , cast(null as {{ type_json() }}) as all_results from dummy_cte where 1 = 0 + From 03add114a2ed5bc3dbb7ed2c8912f5f175ee58c3 Mon Sep 17 00:00:00 2001 From: Yousef Abdulwahab Date: Sat, 18 Jan 2025 22:56:47 +0300 Subject: [PATCH 2/5] feat: add clickhouse tox-tests and env vars --- integration_test_project/example-env.sh | 3 +++ integration_test_project/models/sources.yml | 2 +- integration_test_project/profiles.yml | 8 ++++++++ tox.ini | 11 +++++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/integration_test_project/example-env.sh b/integration_test_project/example-env.sh index 47cb0d67..f10396de 100755 --- a/integration_test_project/example-env.sh +++ b/integration_test_project/example-env.sh @@ -16,6 +16,9 @@ export DBT_ENV_SECRET_DATABRICKS_TOKEN= export DBT_ENV_SECRET_GCP_PROJECT= export DBT_ENV_SPARK_DRIVER_PATH= # /Library/simba/spark/lib/libsparkodbc_sbu.dylib on a Mac export DBT_ENV_SPARK_ENDPOINT= # The endpoint ID from the Databricks HTTP path +export DBT_ENV_SECRET_CLICKHOUSE_HOST= +export DBT_ENV_SECRET_CLICKHOUSE_USER= +export DBT_ENV_SECRET_CLICKHOUSE_PASSWORD= # dbt environment variables, change these export DBT_VERSION="1_5_0" diff --git a/integration_test_project/models/sources.yml b/integration_test_project/models/sources.yml index ac8a0f9f..0c90a32a 100644 --- a/integration_test_project/models/sources.yml +++ b/integration_test_project/models/sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: dummy_source - database: "{% if target.type not in ('spark', 'databricks') %}{{ var('dbt_artifacts_database', target.database) }}{% endif %}" + database: "{% if target.type not in ('spark', 'databricks', 'clickhouse') %}{{ var('dbt_artifacts_database', target.database) }}{% endif %}" schema: "{{ target.schema }}" freshness: error_after: {count: 24, period: hour} diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index fee862ad..03d096e7 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -64,3 +64,11 @@ dbt_artifacts: Encrypt: False user: dbt password: "123" + clickhouse: + type: clickhouse + host: "{{ env_var('DBT_ENV_SECRET_CLICKHOUSE_HOST') }}" + port: 8123 + user: "{{ env_var('DBT_ENV_SECRET_CLICKHOUSE_USER') }}" + password: "{{ env_var('DBT_ENV_SECRET_CLICKHOUSE_PASSWORD') }}" + schema: dbt_artifacts_test_commit_{{ env_var('DBT_VERSION', '') }}_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }} + threads: 8 diff --git a/tox.ini b/tox.ini index d865318c..039e718b 100644 --- a/tox.ini +++ b/tox.ini @@ -84,6 +84,9 @@ passenv = DBT_ENV_SECRET_GCP_PROJECT DBT_ENV_SPARK_DRIVER_PATH DBT_ENV_SPARK_ENDPOINT + DBT_ENV_SECRET_CLICKHOUSE_HOST + DBT_ENV_SECRET_CLICKHOUSE_USER + DBT_ENV_SECRET_CLICKHOUSE_PASSWORD GOOGLE_APPLICATION_CREDENTIALS DBT_CLOUD_PROJECT_ID DBT_CLOUD_JOB_ID @@ -396,3 +399,11 @@ commands = dbt clean dbt deps dbt build --target sqlserver + +[testenv:integration_clickhouse] +changedir = integration_test_project +deps = dbt-clickhouse~=1.8.0 +commands = + dbt clean + dbt deps + dbt build --target clickhouse From 5190e13809a6026ee04681cc733cd743fe87327c Mon Sep 17 00:00:00 2001 From: Yousef Abdulwahab Date: Sat, 18 Jan 2025 22:57:11 +0300 Subject: [PATCH 3/5] fix: array data type errors --- macros/upload_results/insert_into_metadata_table.sql | 10 +++++++++- models/sources/exposures.sql | 4 ++-- models/sources/models.sql | 4 ++-- models/sources/snapshots.sql | 2 +- models/sources/tests.sql | 4 ++-- models/staging/stg_dbt__model_executions.sql | 4 +++- models/staging/stg_dbt__seed_executions.sql | 4 +++- models/staging/stg_dbt__snapshot_executions.sql | 4 +++- models/staging/stg_dbt__test_executions.sql | 5 ++++- 9 files changed, 29 insertions(+), 12 deletions(-) diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index 97c854fc..d1b3cc00 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -71,7 +71,15 @@ {%- endmacro %} {% macro clickhouse__insert_into_metadata_table(relation, fields, content) -%} -{{ return(dbt_artifacts.postgres__insert_into_metadata_table(relation, fields, content)) }} + + {% set insert_into_table_query %} + insert into {{ relation }} {{ fields }} + values + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} + {%- endmacro %} {% macro default__insert_into_metadata_table(relation, fields, content) -%} diff --git a/models/sources/exposures.sql b/models/sources/exposures.sql index 7c87f47a..04031add 100644 --- a/models/sources/exposures.sql +++ b/models/sources/exposures.sql @@ -19,12 +19,12 @@ select , cast(null as {{ type_string() }}) as url , cast(null as {{ type_string() }}) as package_name {% if target.type == "clickhouse" %} - , cast([] as {{ type_array() }}) as depends_on_nodes + , cast(null as {{ type_string() }}) as depends_on_nodes {% else %} , cast(null as {{ type_array() }}) as depends_on_nodes {% endif %} {% if target.type == "clickhouse" %} - , cast([] as {{ type_array() }}) as tags + , cast(null as {{ type_string() }}) as tags {% else %} , cast(null as {{ type_array() }}) as tags {% endif %} diff --git a/models/sources/models.sql b/models/sources/models.sql index 4032d976..26db9ca3 100644 --- a/models/sources/models.sql +++ b/models/sources/models.sql @@ -13,7 +13,7 @@ select {% endif %}, cast(null as {{ type_string() }}) as name, {% if target.type == "clickhouse" %} - cast([] as {{ type_array() }}) as depends_on_nodes, + cast([] as {{ type_string() }}) as depends_on_nodes, {% else %} cast(null as {{ type_array() }}) as depends_on_nodes, {% endif %} @@ -22,7 +22,7 @@ select cast(null as {{ type_string() }}) as checksum, cast(null as {{ type_string() }}) as materialization, {% if target.type == "clickhouse" %} - cast([] as {{ type_array() }}) as tags, + cast([] as {{ type_string() }}) as tags, {% else %} cast(null as {{ type_array() }}) as tags, {% endif %} diff --git a/models/sources/snapshots.sql b/models/sources/snapshots.sql index 0d53bd1a..e67d5770 100644 --- a/models/sources/snapshots.sql +++ b/models/sources/snapshots.sql @@ -13,7 +13,7 @@ select {% endif %}, cast(null as {{ type_string() }}) as name, {% if target.type == "clickhouse" %} - cast([] as {{ type_array() }}) as depends_on_nodes, + cast([] as {{ type_string() }}) as depends_on_nodes, {% else %} cast(null as {{ type_array() }}) as depends_on_nodes, {% endif %} diff --git a/models/sources/tests.sql b/models/sources/tests.sql index 58f783d7..41dfd043 100644 --- a/models/sources/tests.sql +++ b/models/sources/tests.sql @@ -12,14 +12,14 @@ select , cast(null as {{ type_timestamp() }}) as run_started_at , cast(null as {{ type_string() }}) as name {% if target.type == "clickhouse" %} - , cast([] as {{ type_array() }}) as depends_on_nodes + , cast([] as {{ type_string() }}) as depends_on_nodes {% else %} , cast(null as {{ type_array() }}) as depends_on_nodes {% endif %} , cast(null as {{ type_string() }}) as package_name , cast(null as {{ type_string() }}) as test_path {% if target.type == "clickhouse" %} - , cast([] as {{ type_array() }}) as tags + , cast([] as {{ type_string() }}) as tags {% else %} , cast(null as {{ type_array() }}) as tags {% endif %} diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index ad2c4f50..46574845 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -9,7 +9,9 @@ with node_id, run_started_at, was_full_refresh, - {{ split_part("thread_id", "'-'", 2) }} as thread_id, + {% if target.type == "clickhouse" %} {{ split_part("coalesce(thread_id, '')", "'-'", 2) }} as thread_id, + {% else %} {{ split_part("thread_id", "'-'", 2) }} as thread_id, + {% endif %} status, compile_started_at, query_completed_at, diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql index 90f52870..dac8edb5 100644 --- a/models/staging/stg_dbt__seed_executions.sql +++ b/models/staging/stg_dbt__seed_executions.sql @@ -9,7 +9,9 @@ with node_id, run_started_at, was_full_refresh, - {{ split_part("thread_id", "'-'", 2) }} as thread_id, + {% if target.type == "clickhouse" %} {{ split_part("coalesce(thread_id, '')", "'-'", 2) }} as thread_id, + {% else %} {{ split_part("thread_id", "'-'", 2) }} as thread_id, + {% endif %} status, compile_started_at, query_completed_at, diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index 8794b99b..fba8c056 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -9,7 +9,9 @@ with node_id, run_started_at, was_full_refresh, - {{ split_part("thread_id", "'-'", 2) }} as thread_id, + {% if target.type == "clickhouse" %} {{ split_part("coalesce(thread_id, '')", "'-'", 2) }} as thread_id, + {% else %} {{ split_part("thread_id", "'-'", 2) }} as thread_id, + {% endif %} status, compile_started_at, query_completed_at, diff --git a/models/staging/stg_dbt__test_executions.sql b/models/staging/stg_dbt__test_executions.sql index 9fdb3a5f..11eb3718 100644 --- a/models/staging/stg_dbt__test_executions.sql +++ b/models/staging/stg_dbt__test_executions.sql @@ -14,7 +14,10 @@ with , node_id , run_started_at , was_full_refresh - , {{ split_part('thread_id', "'-'", 2) }} as thread_id + , + {% if target.type == "clickhouse" %} {{ split_part("coalesce(thread_id, '')", "'-'", 2) }} as thread_id + {% else %} {{ split_part("thread_id", "'-'", 2) }} as thread_id + {% endif %} , status , compile_started_at , query_completed_at From d1c504f39d714fd732fa7ee95ca057fc76033731 Mon Sep 17 00:00:00 2001 From: Yousef Abdulwahab Date: Sat, 18 Jan 2025 23:12:24 +0300 Subject: [PATCH 4/5] chore: add clickhouse to the list of supported packages --- CONTRIBUTING.md | 1 + README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35653061..3bbc62c1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,6 +78,7 @@ Tox will take care of installing the dependencies for each environment, so you d tox -e integration_snowflake # For the Snowflake tests tox -e integration_databricks # For the Databricks tests tox -e integration_bigquery # For the BigQuery tests + tox -e integration_clickhouse # For the Clickhouse tests ``` The Spark tests require installing the [ODBC driver](https://www.databricks.com/spark/odbc-drivers-download). On a Mac, diff --git a/README.md b/README.md index fa3f6ef0..f1d0bc23 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ The package currently supports - Google BigQuery :white_check_mark: - Postgres :white_check_mark: - SQL Server :white_check_mark: +- Clickhouse :white_check_mark: Models included: From db2640c2a193aae93a9b5941760808112f0a2385 Mon Sep 17 00:00:00 2001 From: Yousef Abdulwahab Date: Sat, 18 Jan 2025 23:25:12 +0300 Subject: [PATCH 5/5] chore: chagne [] to null for arrays data type columns --- models/sources/models.sql | 4 ++-- models/sources/snapshots.sql | 2 +- models/sources/tests.sql | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/sources/models.sql b/models/sources/models.sql index 26db9ca3..a58d4a43 100644 --- a/models/sources/models.sql +++ b/models/sources/models.sql @@ -13,7 +13,7 @@ select {% endif %}, cast(null as {{ type_string() }}) as name, {% if target.type == "clickhouse" %} - cast([] as {{ type_string() }}) as depends_on_nodes, + cast(null as {{ type_string() }}) as depends_on_nodes, {% else %} cast(null as {{ type_array() }}) as depends_on_nodes, {% endif %} @@ -22,7 +22,7 @@ select cast(null as {{ type_string() }}) as checksum, cast(null as {{ type_string() }}) as materialization, {% if target.type == "clickhouse" %} - cast([] as {{ type_string() }}) as tags, + cast(null as {{ type_string() }}) as tags, {% else %} cast(null as {{ type_array() }}) as tags, {% endif %} diff --git a/models/sources/snapshots.sql b/models/sources/snapshots.sql index e67d5770..0d9d2c03 100644 --- a/models/sources/snapshots.sql +++ b/models/sources/snapshots.sql @@ -13,7 +13,7 @@ select {% endif %}, cast(null as {{ type_string() }}) as name, {% if target.type == "clickhouse" %} - cast([] as {{ type_string() }}) as depends_on_nodes, + cast(null as {{ type_string() }}) as depends_on_nodes, {% else %} cast(null as {{ type_array() }}) as depends_on_nodes, {% endif %} diff --git a/models/sources/tests.sql b/models/sources/tests.sql index 41dfd043..72bf74ea 100644 --- a/models/sources/tests.sql +++ b/models/sources/tests.sql @@ -12,14 +12,14 @@ select , cast(null as {{ type_timestamp() }}) as run_started_at , cast(null as {{ type_string() }}) as name {% if target.type == "clickhouse" %} - , cast([] as {{ type_string() }}) as depends_on_nodes + , cast(null as {{ type_string() }}) as depends_on_nodes {% else %} , cast(null as {{ type_array() }}) as depends_on_nodes {% endif %} , cast(null as {{ type_string() }}) as package_name , cast(null as {{ type_string() }}) as test_path {% if target.type == "clickhouse" %} - , cast([] as {{ type_string() }}) as tags + , cast(null as {{ type_string() }}) as tags {% else %} , cast(null as {{ type_array() }}) as tags {% endif %}