From b00b274ededd57f72e8c9df8eaea338b42cc0f75 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Wed, 6 Mar 2024 12:34:38 -0800 Subject: [PATCH 01/14] add the following to `_live` schema - udf_streamline - udf_function_selector --- macros/core/_live.yaml.sql | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/macros/core/_live.yaml.sql b/macros/core/_live.yaml.sql index 5a698a80..946a444c 100644 --- a/macros/core/_live.yaml.sql +++ b/macros/core/_live.yaml.sql @@ -15,4 +15,44 @@ NOT NULL RETURNS NULL ON NULL INPUT sql: udf_api +- name: {{ schema }}.udf_streamline + signature: + - [method, STRING] + - [url, STRING] + - [headers, OBJECT] + - [DATA, VARIANT] + - [user_id, STRING] + - [SECRET, STRING] + return_type: VARIANT + func_type: EXTERNAL + api_integration: '{{ var("API_INTEGRATION") }}' + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + sql: udf_bulk_rest_api +- name: {{ schema }}.udf_function_selector + description: | + This function is used to select the appropriate function to call based on the user_id + signature: + - [method, STRING] + - [url, STRING] + - [headers, OBJECT] + - [DATA, VARIANT] + - [user_id, STRING] + - [SECRET, STRING] + return_type: VARIANT + func_type: SECURE + api_integration: '{{ var("API_INTEGRATION") }}' + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + sql: | + SELECT + CASE + WHEN user_id ilike 'AWS_%' + THEN {{ schema }}.udf_streamline(method, url, headers, DATA, user_id, SECRET) + ELSE {{ schema }}.udf_api(method, url, headers, DATA, user_id, SECRET) + END + + {% endmacro %} \ No newline at end of file From e220fff87f421c3b737a4b9db3fecf00b3729c07 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Wed, 6 Mar 2024 18:04:07 -0800 Subject: [PATCH 02/14] Update macro names in _live.yaml.sql --- macros/core/_live.yaml.sql | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/macros/core/_live.yaml.sql b/macros/core/_live.yaml.sql index 946a444c..da2e9a57 100644 --- a/macros/core/_live.yaml.sql +++ b/macros/core/_live.yaml.sql @@ -1,6 +1,6 @@ {% macro config_core__live(schema="_live") %} -- name: {{ schema }}.udf_api +- name: {{ schema }}._udf_api signature: - [method, STRING] - [url, STRING] @@ -15,22 +15,29 @@ NOT NULL RETURNS NULL ON NULL INPUT sql: udf_api -- name: {{ schema }}.udf_streamline + +- name: {{ schema }}.udf_rest_api_args_only signature: - [method, STRING] - [url, STRING] - [headers, OBJECT] - [DATA, VARIANT] - - [user_id, STRING] - [SECRET, STRING] - return_type: VARIANT - func_type: EXTERNAL - api_integration: '{{ var("API_INTEGRATION") }}' + return_type: OBJECT + func_type: SECRET options: | NOT NULL RETURNS NULL ON NULL INPUT - sql: udf_bulk_rest_api -- name: {{ schema }}.udf_function_selector + sql: | + { + 'method': method, + 'url': url, + 'headers': headers, + 'data': data, + 'secret_name': SECRET + } + +- name: {{ schema }}.udf_api description: | This function is used to select the appropriate function to call based on the user_id signature: @@ -42,7 +49,6 @@ - [SECRET, STRING] return_type: VARIANT func_type: SECURE - api_integration: '{{ var("API_INTEGRATION") }}' options: | NOT NULL RETURNS NULL ON NULL INPUT @@ -50,7 +56,7 @@ SELECT CASE WHEN user_id ilike 'AWS_%' - THEN {{ schema }}.udf_streamline(method, url, headers, DATA, user_id, SECRET) + THEN {{ schema }}.udf_rest_api_args_only(method, url, headers, DATA, SECRET) ELSE {{ schema }}.udf_api(method, url, headers, DATA, user_id, SECRET) END From 39e8baab0d19f57959e3bdd655757241631cd7f3 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Thu, 7 Mar 2024 10:01:25 -0800 Subject: [PATCH 03/14] Update SECRET to SECRET_NAME in _live.yaml.sql --- macros/core/_live.yaml.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/core/_live.yaml.sql b/macros/core/_live.yaml.sql index da2e9a57..968812c3 100644 --- a/macros/core/_live.yaml.sql +++ b/macros/core/_live.yaml.sql @@ -22,7 +22,7 @@ - [url, STRING] - [headers, OBJECT] - [DATA, VARIANT] - - [SECRET, STRING] + - [SECRET_NAME, STRING] return_type: OBJECT func_type: SECRET options: | @@ -34,7 +34,7 @@ 'url': url, 'headers': headers, 'data': data, - 'secret_name': SECRET + 'secret_name': SECRET_NAME } - name: {{ schema }}.udf_api From 1ef6a998c79fe9409ff821a65a2b2ec8375c7a43 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Thu, 7 Mar 2024 11:10:33 -0800 Subject: [PATCH 04/14] Update func_type from SECRET to SECURE --- macros/core/_live.yaml.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/core/_live.yaml.sql b/macros/core/_live.yaml.sql index 968812c3..6543f046 100644 --- a/macros/core/_live.yaml.sql +++ b/macros/core/_live.yaml.sql @@ -24,7 +24,7 @@ - [DATA, VARIANT] - [SECRET_NAME, STRING] return_type: OBJECT - func_type: SECRET + func_type: SECURE options: | NOT NULL RETURNS NULL ON NULL INPUT From 77ccc09dc603ecf944b15186f657a71d4cc4f966 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:02:59 -0800 Subject: [PATCH 05/14] Add UDF(object) for API calls --- macros/core/live.yaml.sql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/macros/core/live.yaml.sql b/macros/core/live.yaml.sql index e5988f00..fda4dbce 100644 --- a/macros/core/live.yaml.sql +++ b/macros/core/live.yaml.sql @@ -120,6 +120,24 @@ secret_name ) +- name: {{ schema }}.udf_api + signature: + - [args, OBJECT] + return_type: VARIANT + options: | + NOT NULL + IMMUTABLE + sql: | + SELECT + _live.UDF_API( + COALESCE(args:method, 'GET'), + args:url, + COALESCE(args:headers, {}), + COALESCE(args:data, {}), + _utils.UDF_WHOAMI(), + COALESCE(args:secret_name, '') + ) + - name: {{ schema }}.udf_rpc signature: - [blockchain, STRING] From 08e31f8c1d5aec40c6d8662af865289cf17e13f6 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Fri, 8 Mar 2024 13:34:26 -0800 Subject: [PATCH 06/14] Update function name in _live.yaml.sql --- macros/core/_live.yaml.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/core/_live.yaml.sql b/macros/core/_live.yaml.sql index 6543f046..373968cf 100644 --- a/macros/core/_live.yaml.sql +++ b/macros/core/_live.yaml.sql @@ -57,7 +57,7 @@ CASE WHEN user_id ilike 'AWS_%' THEN {{ schema }}.udf_rest_api_args_only(method, url, headers, DATA, SECRET) - ELSE {{ schema }}.udf_api(method, url, headers, DATA, user_id, SECRET) + ELSE {{ schema }}._udf_api(method, url, headers, DATA, user_id, SECRET) END From a98106eb4dba07ad099e3c2aeac0bc439c47c05e Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Fri, 8 Mar 2024 13:39:43 -0800 Subject: [PATCH 07/14] Fix type conversion issue in udf_rest_api_args_only --- macros/core/_live.yaml.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/core/_live.yaml.sql b/macros/core/_live.yaml.sql index 373968cf..f8525727 100644 --- a/macros/core/_live.yaml.sql +++ b/macros/core/_live.yaml.sql @@ -56,7 +56,7 @@ SELECT CASE WHEN user_id ilike 'AWS_%' - THEN {{ schema }}.udf_rest_api_args_only(method, url, headers, DATA, SECRET) + THEN {{ schema }}.udf_rest_api_args_only(method, url, headers, DATA, SECRET)::VARIANT ELSE {{ schema }}._udf_api(method, url, headers, DATA, user_id, SECRET) END From 5a014f4aad8e227bd292d8006572e0b2fff1baac Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Fri, 8 Mar 2024 13:49:34 -0800 Subject: [PATCH 08/14] Update dbt_project.yml profile setting --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index 2946a8fe..d3783446 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -6,7 +6,7 @@ version: "1.0.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. -profile: "livequery" +profile: livequery # These configurations specify where dbt should look for different types of files. # The `source-paths` config, for example, states that models in this project can be From dbb5a52b357c4ec66febad3af0fd747f81de52b4 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Mon, 11 Mar 2024 12:50:49 -0700 Subject: [PATCH 09/14] Add _live.yml model for core deployment --- models/deploy/core/_live.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 models/deploy/core/_live.yml diff --git a/models/deploy/core/_live.yml b/models/deploy/core/_live.yml new file mode 100644 index 00000000..2a5113ac --- /dev/null +++ b/models/deploy/core/_live.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: _live + columns: + - name: udf_rest_api_args_only + tests: + - test_udf: + name: test___live_udf_rest_api_args_only + args: > + 'GET', + 'https://api.example.com', + {'Content-Type': 'application/json'}, + {'hello': 'world'}, + 'my_secret' + assertions: + - | + result = { + 'method': 'GET', + 'url': 'https://api.example.com', + 'headers': {'Content-Type': 'application/json'}, + 'data': {'hello': 'world'}, + 'secret_name': 'my_secret' + } + - test_udf: + name: test___live_udf_rest_api_args_only_null + args: null, null, null, null, null + assertions: + - result = {} \ No newline at end of file From 353c5d006bf54f318aae27ea9b89dab0cf59b391 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Mon, 11 Mar 2024 12:51:07 -0700 Subject: [PATCH 10/14] Add test cases for UDF API in _live.yml --- models/deploy/core/_live.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/models/deploy/core/_live.yml b/models/deploy/core/_live.yml index 2a5113ac..d363dd81 100644 --- a/models/deploy/core/_live.yml +++ b/models/deploy/core/_live.yml @@ -25,4 +25,23 @@ models: name: test___live_udf_rest_api_args_only_null args: null, null, null, null, null assertions: - - result = {} \ No newline at end of file + - result = {} + - test_udf: + name: test___live_udf_api + args: > + 'GET', + 'https://api.example.com', + {'Content-Type': 'application/json'}, + {'hello': 'world'}, + 'non_streamline_user' + 'my_secret' + assertions: + - | + result = { + 'method': 'GET', + 'url': 'https://httpbin.org/get', + 'headers': {'Content-Type': 'application/json'}, + 'data': {'hello': 'world'}, + 'user': 'non_streamline_user', + 'secret_name': 'my_secret' + } From 7f6216324d75de9f794ec7f5ecc254f43e7be7cb Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Tue, 12 Mar 2024 10:53:11 -0700 Subject: [PATCH 11/14] Change IMMUTABLE to VOLATILE in UDF_API return_type --- macros/core/live.yaml.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/core/live.yaml.sql b/macros/core/live.yaml.sql index fda4dbce..e6945cbc 100644 --- a/macros/core/live.yaml.sql +++ b/macros/core/live.yaml.sql @@ -126,7 +126,7 @@ return_type: VARIANT options: | NOT NULL - IMMUTABLE + VOLATILE sql: | SELECT _live.UDF_API( From de2ade547b8e87be0902605d42152ed97f53e05d Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Tue, 12 Mar 2024 11:03:14 -0700 Subject: [PATCH 12/14] Refactor test_udf in _live.yml --- models/deploy/core/_live.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/models/deploy/core/_live.yml b/models/deploy/core/_live.yml index d363dd81..bae72e41 100644 --- a/models/deploy/core/_live.yml +++ b/models/deploy/core/_live.yml @@ -27,21 +27,21 @@ models: assertions: - result = {} - test_udf: - name: test___live_udf_api - args: > - 'GET', - 'https://api.example.com', - {'Content-Type': 'application/json'}, - {'hello': 'world'}, - 'non_streamline_user' - 'my_secret' - assertions: - - | - result = { - 'method': 'GET', - 'url': 'https://httpbin.org/get', - 'headers': {'Content-Type': 'application/json'}, - 'data': {'hello': 'world'}, - 'user': 'non_streamline_user', - 'secret_name': 'my_secret' - } + name: test___live_udf_api + args: > + 'GET', + 'https://api.example.com', + {'Content-Type': 'application/json'}, + {'hello': 'world'}, + 'non_streamline_user' + 'my_secret' + assertions: + - | + result = { + 'method': 'GET', + 'url': 'https://httpbin.org/get', + 'headers': {'Content-Type': 'application/json'}, + 'data': {'hello': 'world'}, + 'user': 'non_streamline_user', + 'secret_name': 'my_secret' + } From 32ff24fe127bf88cd1917765ec8042f4533c3056 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:14:11 -0700 Subject: [PATCH 13/14] Refactor macro names and add deployment model for _live schema --- macros/core/_live.yaml.sql | 10 +++++----- models/deploy/core/{_live.yml => _live._yml} | 0 2 files changed, 5 insertions(+), 5 deletions(-) rename models/deploy/core/{_live.yml => _live._yml} (100%) diff --git a/macros/core/_live.yaml.sql b/macros/core/_live.yaml.sql index f8525727..04e4ae01 100644 --- a/macros/core/_live.yaml.sql +++ b/macros/core/_live.yaml.sql @@ -1,6 +1,6 @@ {% macro config_core__live(schema="_live") %} -- name: {{ schema }}._udf_api +- name: {{ schema }}.udf_api signature: - [method, STRING] - [url, STRING] @@ -16,7 +16,7 @@ RETURNS NULL ON NULL INPUT sql: udf_api -- name: {{ schema }}.udf_rest_api_args_only +{# - name: {{ schema }}.udf_rest_api_args_only signature: - [method, STRING] - [url, STRING] @@ -35,9 +35,9 @@ 'headers': headers, 'data': data, 'secret_name': SECRET_NAME - } + } #} -- name: {{ schema }}.udf_api +{# - name: {{ schema }}.udf_api description: | This function is used to select the appropriate function to call based on the user_id signature: @@ -58,7 +58,7 @@ WHEN user_id ilike 'AWS_%' THEN {{ schema }}.udf_rest_api_args_only(method, url, headers, DATA, SECRET)::VARIANT ELSE {{ schema }}._udf_api(method, url, headers, DATA, user_id, SECRET) - END + END #} {% endmacro %} \ No newline at end of file diff --git a/models/deploy/core/_live.yml b/models/deploy/core/_live._yml similarity index 100% rename from models/deploy/core/_live.yml rename to models/deploy/core/_live._yml From efbd76076d07138a037d7868d828ba85d709d7cc Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Mon, 8 Apr 2024 14:08:52 -0400 Subject: [PATCH 14/14] wip --- macros/core/_live.yaml.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/core/_live.yaml.sql b/macros/core/_live.yaml.sql index 04e4ae01..680bd646 100644 --- a/macros/core/_live.yaml.sql +++ b/macros/core/_live.yaml.sql @@ -16,7 +16,7 @@ RETURNS NULL ON NULL INPUT sql: udf_api -{# - name: {{ schema }}.udf_rest_api_args_only +- name: {{ schema }}.udf_rest_api_args_only signature: - [method, STRING] - [url, STRING] @@ -35,7 +35,7 @@ 'headers': headers, 'data': data, 'secret_name': SECRET_NAME - } #} + } {# - name: {{ schema }}.udf_api description: |