From 7a19beb06dee40c99da0bf087f0e9f450584b8e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Chm=C3=BArny?= Date: Sun, 17 Nov 2024 13:17:23 +0100 Subject: [PATCH 01/12] Add httpx.Response to http_client methods type hints --- forloop_modules/utils/http_client.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/forloop_modules/utils/http_client.py b/forloop_modules/utils/http_client.py index 5b6ebc8..79df4d3 100644 --- a/forloop_modules/utils/http_client.py +++ b/forloop_modules/utils/http_client.py @@ -19,7 +19,14 @@ def __init__(self, *args, **httpx_kwargs): self._sse_parser = SSEParser(httpx_client=self) self.sse_stream = self._sse_parser.stream - def _request(self, url: str, method: str, *, headers: Optional[dict[str, str]] = None, **kwargs): + def _request( + self, + url: str, + method: str, + *, + headers: Optional[dict[str, str]] = None, + **kwargs, + ) -> httpx.Response: headers = headers or {} global_headers = { # "Authorization": f"Bearer {aet.get_access_token()}", @@ -27,20 +34,20 @@ def _request(self, url: str, method: str, *, headers: Optional[dict[str, str]] = } global_headers.update(headers) response = super().request(method, url, headers=global_headers, **kwargs) - response.ok = response.is_success # Cross compatibility with requests + response.ok = response.is_success # Cross compatibility with requests # response.raise_for_status() return response - def get(self, url: str, **kwargs): + def get(self, url: str, **kwargs) -> httpx.Response: return self._request(url, "GET", **kwargs) - def post(self, url: str, **kwargs): + def post(self, url: str, **kwargs) -> httpx.Response: return self._request(url, "POST", **kwargs) - def put(self, url: str, **kwargs): + def put(self, url: str, **kwargs) -> httpx.Response: return self._request(url, "PUT", **kwargs) - def delete(self, url: str, **kwargs): + def delete(self, url: str, **kwargs) -> httpx.Response: return self._request(url, "DELETE", **kwargs) From beef2f25e0e4b119b76527945b2c6fe19d79f04b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Chm=C3=BArny?= Date: Sun, 17 Nov 2024 13:18:02 +0100 Subject: [PATCH 02/12] Format and update imports in ncrb --- .../queries/node_context_requests_backend.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/forloop_modules/queries/node_context_requests_backend.py b/forloop_modules/queries/node_context_requests_backend.py index e8dd06f..98da06a 100644 --- a/forloop_modules/queries/node_context_requests_backend.py +++ b/forloop_modules/queries/node_context_requests_backend.py @@ -2,18 +2,30 @@ """In this file all functions should have response-like return value""" import json -from inspect import Parameter, Signature -from pydantic import BaseModel -from typing import Optional, Any, Generator import sys +from inspect import Parameter, Signature from pathlib import Path -import forloop_modules.flog as flog -import json +from typing import Any, Generator, Optional -from forloop_modules.utils.http_client import HttpClient -from forloop_modules.globals.active_entity_tracker import aet +from httpx import Response -from forloop_modules.queries.db_model_templates import APIProject, APITrigger, APIDataset, APIDbTable, APIScript, APIFile, APIDatabase, APIPipeline, APIEdge, APIVariable, APIPopup, APIInitialVariable +import forloop_modules.flog as flog +from forloop_modules.globals.active_entity_tracker import aet +from forloop_modules.queries.db_model_templates import ( + APIDatabase, + APIDataset, + APIDbTable, + APIEdge, + APIFile, + APIInitialVariable, + APIPipeline, + APIPopup, + APIProject, + APIScript, + APITrigger, + APIVariable, +) +from forloop_modules.utils.http_client import HttpClient if sys.platform == "darwin": # MAC OS config_path = 'config/server_config_remote.ini' From 51cd34202d70cbf43efd0b87dfd06908c71b150e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Chm=C3=BArny?= Date: Sun, 17 Nov 2024 13:18:34 +0100 Subject: [PATCH 03/12] Format 'resources' variable in ncrb --- forloop_modules/queries/node_context_requests_backend.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/forloop_modules/queries/node_context_requests_backend.py b/forloop_modules/queries/node_context_requests_backend.py index 98da06a..e73439d 100644 --- a/forloop_modules/queries/node_context_requests_backend.py +++ b/forloop_modules/queries/node_context_requests_backend.py @@ -40,7 +40,7 @@ BASE_API = f'{SERVER}:{PORT}/api/v1' http_client = HttpClient() -resources = { +RESOURCES = { "databases": ["get_all", "get", "new", "delete", "update"], "dbtables": ["get_all", "get", "new", "delete", "update"], "files": ["get_all", "get", "new", "delete", "update"], @@ -49,9 +49,9 @@ "edges": ["get_all", "get", "new", "delete", "update"], "variables": ["get_all", "get", "new", "delete"], "popups": ["get_all", "get", "delete"], - "nodes": ["get_all", "get", "delete"],# "get", "new", "delete", "update"], + "nodes": ["get_all", "get", "delete"], "pipelines": ["get_all", "get", "new", "delete", "update"], - "initial_variables": ["get_all", "get", "new", "delete"] + "initial_variables": ["get_all", "get", "new", "delete"], } @@ -177,7 +177,7 @@ def update(uid, *args, model_attribute_names_without_uid=model_attribute_names_w return update -for resource_name, actions in resources.items(): +for resource_name, actions in RESOURCES.items(): resource_name_singular = resource_name[:-1] for action in actions: function_name = None From 988c7b9fe14c97fd8707328e200eb94faec21cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Chm=C3=BArny?= Date: Sun, 17 Nov 2024 13:19:02 +0100 Subject: [PATCH 04/12] Format DB_API_BODY_TEMPLATE in ncrb --- .../queries/node_context_requests_backend.py | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/forloop_modules/queries/node_context_requests_backend.py b/forloop_modules/queries/node_context_requests_backend.py index e73439d..b83409d 100644 --- a/forloop_modules/queries/node_context_requests_backend.py +++ b/forloop_modules/queries/node_context_requests_backend.py @@ -54,21 +54,20 @@ "initial_variables": ["get_all", "get", "new", "delete"], } - - -db_api_body_template ={"projects":APIProject, - "triggers":APITrigger, - "datasets":APIDataset, - "dbtables":APIDbTable, - "scripts":APIScript, - "files":APIFile, - "databases":APIDatabase, - "pipelines":APIPipeline, - "edges": APIEdge, - "variables": APIVariable, - "popups": APIPopup, - "initial_variables": APIInitialVariable - } +DB_API_BODY_TEMPLATE = { + "projects": APIProject, + "triggers": APITrigger, + "datasets": APIDataset, + "dbtables": APIDbTable, + "scripts": APIScript, + "files": APIFile, + "databases": APIDatabase, + "pipelines": APIPipeline, + "edges": APIEdge, + "variables": APIVariable, + "popups": APIPopup, + "initial_variables": APIInitialVariable, +} #! Auxiliary function - sets project_uid and pipeline_uid into factory functions' payloads def set_stored_project_uid_and_pipeline_uid_to_factory_payload(payload: dict): @@ -188,14 +187,14 @@ def update(uid, *args, model_attribute_names_without_uid=model_attribute_names_w fn = get_factory(resource_name_singular) function_name = f'{action}_{resource_name_singular}_by_uid' elif action == 'new': - model = db_api_body_template[resource_name] + model = DB_API_BODY_TEMPLATE[resource_name] fn = new_factory(resource_name, model) function_name = f'{action}_{resource_name_singular}' elif action == 'delete': fn = delete_factory(resource_name_singular) function_name = f'{action}_{resource_name_singular}_by_uid' elif action == 'update': - model = db_api_body_template[resource_name] + model = DB_API_BODY_TEMPLATE[resource_name] fn = update_factory(resource_name_singular, model) function_name = f'{action}_{resource_name_singular}_by_uid' else: From 28061ff78f9f71012e1de76b29112fcd75b8f1bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Chm=C3=BArny?= Date: Sun, 17 Nov 2024 13:21:21 +0100 Subject: [PATCH 05/12] Improve set_stored_project_uid_and_pipeline_uid_to_factory_payload docs and comments --- forloop_modules/queries/node_context_requests_backend.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/forloop_modules/queries/node_context_requests_backend.py b/forloop_modules/queries/node_context_requests_backend.py index b83409d..90488ad 100644 --- a/forloop_modules/queries/node_context_requests_backend.py +++ b/forloop_modules/queries/node_context_requests_backend.py @@ -69,14 +69,16 @@ "initial_variables": APIInitialVariable, } -#! Auxiliary function - sets project_uid and pipeline_uid into factory functions' payloads + def set_stored_project_uid_and_pipeline_uid_to_factory_payload(payload: dict): - """Sets project_uid and pipeline_uid values in payload from those stored in auth handler. + """ + Sets project_uid and pipeline_uid values in payload from those stored in auth handler. Args: payload (dict): API request payload. """ - #! This approach might not be safe if API validation becomes strict (not all calls take both project_uid and pipeline_uid as parameters) + # This approach might not be safe if API validation becomes strict (not all calls take both + # project_uid and pipeline_uid as parameters) # TODO: Think about a better implementation (due to the point above) payload['project_uid'] = aet.project_uid payload['pipeline_uid'] = aet.active_pipeline_uid From 151e4a6895e26bb611ce115f0ff55d8c9312510e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Chm=C3=BArny?= Date: Sun, 17 Nov 2024 13:23:02 +0100 Subject: [PATCH 06/12] Add docstrings and type-hints to ncrb factory functions --- .../queries/node_context_requests_backend.py | 161 ++++++++++-------- 1 file changed, 94 insertions(+), 67 deletions(-) diff --git a/forloop_modules/queries/node_context_requests_backend.py b/forloop_modules/queries/node_context_requests_backend.py index 90488ad..218bb0b 100644 --- a/forloop_modules/queries/node_context_requests_backend.py +++ b/forloop_modules/queries/node_context_requests_backend.py @@ -84,96 +84,138 @@ def set_stored_project_uid_and_pipeline_uid_to_factory_payload(payload: dict): payload['pipeline_uid'] = aet.active_pipeline_uid -def get_all_factory(resource_name): - resource_url = f'{SERVER}:{str(PORT)}/api/v1/{resource_name}' - def get_all(): +def remove_none_values_from_payload(payload: dict) -> dict: + payload = {key: value for key, value in payload.items() if value is not None} + + return payload + + +def get_all_factory(resource_name: str): + """ + Factory creating a "GET all " request function. + + Args: + resource_name (str): Name of resources to GET (e.g. nodes, edges, scripts etc.) + + Returns: + (() -> Response): "GET all " request calling function + """ + resource_url = f"{SERVER}:{str(PORT)}/api/v1/{resource_name}" + + def get_all() -> Response: response = http_client.get(resource_url) - #flog.info(f'GET all response: {response.text}') return response - get_all.__name__ = f'get_all_{resource_name}' + + get_all.__name__ = f"get_all_{resource_name}" return get_all -def get_factory(resource_name): - def get(resource_uid): - resource_url = f'{BASE_API}/{resource_name}/{resource_uid}' + +def get_factory(resource_name: str): + """ + Factory creating a "GET " request function. + + Args: + resource_name (str): Name of a resource to GET (e.g. node, edge, script etc.) + + Returns: + ((resource_uid: str) -> Response): "GET " request calling function + """ + def get(resource_uid: str): + resource_url = f"{BASE_API}/{resource_name}/{resource_uid}" response = http_client.get(resource_url) - #flog.info(f'GET response: {response.text}') return response - get.__name__ = f'get_{resource_name}_by_uid' + + get.__name__ = f"get_{resource_name}_by_uid" return get -def delete_factory(resource_name): - def delete(resource_uid): - resource_url = f'{BASE_API}/{resource_name}/{resource_uid}' + +def delete_factory(resource_name: str): + """ + Factory creating a "DELETE " request function. + + Args: + resource_name (str): Name of a resource DELETE (e.g. node, edge, script etc.) + + Returns: + ((resource_uid: str) -> Response): "DELETE " request calling function + """ + def delete(resource_uid: str): + resource_url = f"{BASE_API}/{resource_name}/{resource_uid}" response = http_client.delete(resource_url) - #flog.info(f'DELETE response: {response.text}') return response - delete.__name__ = f'delete_{resource_name}_by_uid' - return delete + delete.__name__ = f"delete_{resource_name}_by_uid" + return delete -def new_factory(resource_name, model): - #list of pydantic attributes +def new_factory(resource_name: str, model): + # list of pydantic attributes model_attribute_names = list(vars(model()).keys()) - #remove the uid attribute – see VariableModel and APIVariable - model_attribute_names_without_uid = [v for v in model_attribute_names if v != 'uid'] - #the new function (eg new_database) will expect same args as what the attributes are (without uid) - params = [Parameter(name, Parameter.POSITIONAL_OR_KEYWORD) for name in model_attribute_names_without_uid] - - def new(*args,model_attribute_names_without_uid=model_attribute_names_without_uid, **kwargs): + # remove the uid attribute – see VariableModel and APIVariable + model_attribute_names_without_uid = [v for v in model_attribute_names if v != "uid"] + # the new function (eg new_database) will expect same args as what the attributes are (without uid) + params = [ + Parameter(name, Parameter.POSITIONAL_OR_KEYWORD) + for name in model_attribute_names_without_uid + ] + + def new( + *args, + model_attribute_names_without_uid=model_attribute_names_without_uid, + **kwargs, + ): payload = { - param: arg - for param, arg in zip(model_attribute_names_without_uid, args) + param: arg for param, arg in zip(model_attribute_names_without_uid, args) } payload.update(kwargs) set_stored_project_uid_and_pipeline_uid_to_factory_payload(payload) if issubclass(model, APIVariable): payload["pipeline_job_uid"] = aet.active_pipeline_job_uid - #flog.info(f'New {resource_name} payload: {payload}') - resource_url = f'{BASE_API}/{resource_name}' - + resource_url = f"{BASE_API}/{resource_name}" response = http_client.post(resource_url, json=payload) - #flog.info(f'New {resource_name} response: {response.text}') - if not response.ok: - flog.error(response.json()) - # response.raise_for_status() + return response new.__signature__ = Signature(params) - new.__name__ = f'new_{resource_name}' + new.__name__ = f"new_{resource_name}" return new -def update_factory(resource_name, model): - #list of pydantic class attributes - model_attribute_names = list(vars(model()).keys()) - #remove the uid attribute – see VariableModel and APIVariable - model_attribute_names_without_uid = [v for v in model_attribute_names if v != 'uid'] - params = [Parameter(name, Parameter.POSITIONAL_OR_KEYWORD) for name in model_attribute_names_without_uid] - def update(uid, *args, model_attribute_names_without_uid=model_attribute_names_without_uid, **kwargs): +def update_factory(resource_name: str, model): + # list of pydantic class attributes + model_attribute_names = list(vars(model()).keys()) + # remove the uid attribute – see VariableModel and APIVariable + model_attribute_names_without_uid = [v for v in model_attribute_names if v != "uid"] + params = [ + Parameter(name, Parameter.POSITIONAL_OR_KEYWORD) + for name in model_attribute_names_without_uid + ] + + def update( + uid, + *args, + model_attribute_names_without_uid=model_attribute_names_without_uid, + **kwargs, + ): payload = { - param: arg - for param, arg in zip(model_attribute_names_without_uid, args) + param: arg for param, arg in zip(model_attribute_names_without_uid, args) } payload.update(kwargs) set_stored_project_uid_and_pipeline_uid_to_factory_payload(payload) if issubclass(model, APIVariable): payload["pipeline_job_uid"] = aet.active_pipeline_job_uid - #flog.info(f'Update {resource_name} payload: {payload}') - resource_url = f'{BASE_API}/{resource_name}/{uid}' - + resource_url = f"{BASE_API}/{resource_name}/{uid}" response = http_client.put(resource_url, json=payload) - #flog.info(f'Update {resource_name} response: {response.text}') - if not response.ok: - flog.error(response.json()) + return response - update.__signature__ = Signature([Parameter('uid', Parameter.POSITIONAL_OR_KEYWORD)] + params) - update.__name__ = f'update_{resource_name}_by_uid' + update.__signature__ = Signature( + [Parameter("uid", Parameter.POSITIONAL_OR_KEYWORD)] + params + ) + update.__name__ = f"update_{resource_name}_by_uid" return update @@ -205,25 +247,10 @@ def update(uid, *args, model_attribute_names_without_uid=model_attribute_names_w globals()[function_name] = fn - - -# def get_project_key() -> Optional[str]: -# project_key = auth.user_email -# if project_key is not None: -# project_key = project_key.replace("@", "at") - -# return project_key - - def get_project_uid() -> Optional[str]: project_uid = aet.project_uid - - return project_uid -def remove_none_values_from_payload(payload:dict) -> dict: - payload = {key:value for key, value in payload.items() if value is not None} - - return payload + return project_uid ############### Nodes ################# From 3f2a37053a6fe63be5699e3037ee739684a53f99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Chm=C3=BArny?= Date: Sun, 17 Nov 2024 13:23:24 +0100 Subject: [PATCH 07/12] Remove old legacy code from ncrb --- .../queries/node_context_requests_backend.py | 380 +----------------- 1 file changed, 3 insertions(+), 377 deletions(-) diff --git a/forloop_modules/queries/node_context_requests_backend.py b/forloop_modules/queries/node_context_requests_backend.py index 218bb0b..2d699a1 100644 --- a/forloop_modules/queries/node_context_requests_backend.py +++ b/forloop_modules/queries/node_context_requests_backend.py @@ -255,6 +255,7 @@ def get_project_uid() -> Optional[str]: ############### Nodes ################# + def new_node(pos, typ, params_dict: Optional[dict] = None, fields: Optional[list] = None, visible: Optional[bool] = True): project_uid = aet.project_uid pipeline_uid = aet.active_pipeline_uid @@ -335,14 +336,6 @@ def delete_node_by_uid(node_uid): return response -# def delete_node_by_uid(node_uid): -# url = f'{BASE_API}/node/{node_uid}' -# -# response = http_client.delete(url) -# flog.info(f'DELETE Node response: {response.text}') -# -# return response - def delete_all_nodes(): pipeline_uid = aet.active_pipeline_uid @@ -459,39 +452,8 @@ def node_button_click_wrapper(args): - ############### Edges ################# -"""In this file all functions should have response-like return value""" - -#Todo -# def new_edge(from_node_uid, to_node_uid, visible:bool=True): -# project_uid = aet.project_uid -# -# payload = { -# "from_node_uid": from_node_uid, -# "to_node_uid": to_node_uid, -# "project_uid": project_uid, -# "visible": visible -# } -# -# flog.info(f'New Edge payload: {payload}') -# -# url = f'{BASE_API}/edges' -# -# response = http_client.post(url, json=payload) -# flog.info(f'New Edge response: {response.text}') -# -# return response - - -# def get_all_edges(): -# url = f'{BASE_API}/edges' -# -# response = http_client.get(url) -# flog.info(f'GET all Edges response: {response.text}') -# -# return response def get_edges_by_node_uid(node_uid:str): url = f'{BASE_API}/nodes/{node_uid}/edges' @@ -556,30 +518,6 @@ def get_last_active_dataframe_node_uid(): ########### Variables ############### -"""In this file all functions should have response-like return value""" - - -# def new_variable(name, value, type=None, size=None): -# # TODO: solve DFs, dicts, lists, objects etc -# -# project_uid = get_project_uid() -# -# payload = { -# "name": name, -# "value": value, -# "type": type, -# "size": size, -# "project_uid": project_uid -# } -# -# flog.info(f'New Variable payload: {payload}') -# -# url = f'{BASE_API}/variables' -# -# response = http_client.post(url, json=payload) -# flog.info(f'New Variable response: {response.text}') -# -# return response def get_variable(uid: str) -> Optional[dict]: @@ -604,16 +542,6 @@ def get_variable_by_name(variable_name: str): return response - -# def get_all_variables(): -# url = f'{BASE_API}/variables' -# -# response = http_client.get(url) -# flog.info(f'GET all Variables response: {response.text}') -# -# return response - - def delete_variable_by_uid(variable_uid): @@ -660,25 +588,6 @@ def update_variable_by_uid(variable_uid: str, name: str, value: Any, is_result: result=json.loads(response.content.decode('utf-8')) return(response) - -# def update_variable_by_uid(variable_uid,name,value,type=None,size=None): -# # TODO: don't we miss project_uid here? - -# payload = { -# "name": name, -# "value": value, -# "type": type, -# "size": size -# } - -# flog.info(f'Updated Variable payload: {payload}') -# url = f'{BASE_API}/variable/{variable_uid}' - -# response = http_client.put(url, json=payload) -# flog.info(f'Updated Variable response: {response.text}') - -# return response - def get_job_variables(): job_uid = aet.active_pipeline_job_uid @@ -752,72 +661,7 @@ def update_initial_variable_by_uid( return response -############# Dbtables ############### -# -# def new_dbtable(name, pos, columns, is_rolled, database_uid, project_uid): -# # payload = '{"name":"' + str(name) + '","pos":'+ json.dumps(pos) + ',"columns":' + json.dumps(columns) + ',"is_rolled":' + json.dumps( -# # is_rolled) + ',"database_uid":"' + database_uid + '","project_uid":"' + project_uid + '"}' # TODO: solve DFs, dicts, lists, objects etc. -# # # -# # -# payload = { -# "name": str(name), -# "pos": pos, -# "columns": columns, -# "is_rolled": is_rolled, -# "database_uid": database_uid, -# "project_uid": project_uid -# } -# flog.info(payload) -# response = http_client.post(SERVER + ":" + str(PORT) + "/api/v1/dbtables", json=payload) -# return (response) - - -# def get_all_dbtables(): -# response=http_client.get(SERVER+":"+str(PORT)+"/api/v1/dbtables") -# return(response) - -# def get_dbtable_by_uid(dbtable_uid): -# response=http_client.get(SERVER+":"+str(PORT)+"/api/v1/dbtable/"+str(dbtable_uid)) -# return(response) -# -# def update_dbtable_by_uid(dbtable_uid, name, pos, columns, is_rolled, database_uid, project_uid): -# # payload = '{"name":"' + str(name) + '","pos":' + json.dumps(pos) + ',"columns":' + json.dumps( -# # columns) + ',"is_rolled":' + json.dumps( -# # is_rolled) + ',"database_uid":"' + database_uid + '","project_uid":"' + project_uid + '"}' # TODO: solve DFs, dicts, lists, objects etc. -# payload = { -# "name": str(name), -# "pos": pos, -# "columns": columns, -# "is_rolled": is_rolled, -# "database_uid": database_uid, -# "project_uid": project_uid -# } -# flog.info(payload) -# response = http_client.put(SERVER + ":" + str(PORT) + "/api/v1/dbtable/"+str(dbtable_uid), json=payload) -# return (response) - -##### files ##### - - -########### files ############### -"""In this file all functions should have response-like return value""" - - -# def new_file(file_name: str, data=None): -# project_uid = "0" -# -# # TODO: not tested with data -# payload={ -# "file_name": file_name, -# "data": json.dumps(data), -# "project_uid": project_uid -# } -# flog.info(f'New File payload: {payload}') -# print("REQUEST", SERVER, PORT, payload) -# response=http_client.post(SERVER+":"+str(PORT)+"/api/v1/files",json=payload) -# flog.info(f'New File response: {response.text}') -# -# return(response) +########### Files ############### def upload_urls_from_file(path:str): file = {'file': open(path, 'rb')} @@ -825,21 +669,6 @@ def upload_urls_from_file(path:str): flog.info(f"file upload response: {response.text}") return response - - -# def get_file_by_name(file_name): -# response=http_client.get(SERVER+":"+str(PORT)+"/api/v1/get_file_by_name/"+str(file_name)) -# return(response) - - -# def get_all_files(): -# response=http_client.get(SERVER+":"+str(PORT)+"/api/v1/files") -# return(response) - - -# def delete_file_by_uid(file_uid): -# response=http_client.delete(SERVER+":"+str(PORT)+"/api/v1/file/"+str(file_uid)) -# return(response) def delete_all_files(): @@ -850,69 +679,11 @@ def delete_all_files(): payload='{"project_uid":"'+project_uid+'"}' flog.info(payload) response=http_client.delete(SERVER+":"+str(PORT)+"/api/v1/files",data=payload) - return(response) - - - - - -# def update_file_by_uid(file_uid,name,value,type=None,size=None): -# payload='{"name":'+json.dumps(name)+',"value":'+json.dumps(value)+',"type":'+json.dumps(type)+',"size":'+json.dumps(size)+'}' # -# flog.info(payload) -# response=http_client.put(SERVER+":"+str(PORT)+"/api/v1/file/"+str(file_uid),data=payload) -# result=json.loads(response.content.decode('utf-8')) -# return(response) - - - - -# def delete_dbtable_by_uid(dbtable_uid): -# response = http_client.delete(SERVER + ":" + str(PORT) + "/api/v1/dbtable/" + str(dbtable_uid)) -# return (response) - - - + return(response) ############## Pipelines ############## -"""In this file all functions should have response-like return value""" -# def get_all_pipelines(): -# url = f'{BASE_API}/pipelines' -# -# response = http_client.get(url) -# flog.info(f'GET all Pipelines response: {response.text}') -# -# return response - - -# def delete_pipeline_by_uid(pipeline_uid): -# url = f'{BASE_API}/pipeline/{pipeline_uid}' -# -# response = http_client.delete(url) -# flog.info(f'DELETE Pipeline response: {response.text}') -# -# return response -# -# -# def new_pipeline(pos, typ, params_dict): -# project_uid = get_project_uid() -# -# payload = { -# "pos": pos, -# "typ": typ, -# "params": params_dict, -# "project_uid": project_uid -# } -# -# flog.info(f'New Pipeline payload: {payload}') -# -# url = f'{BASE_API}/pipelines' -# -# response = http_client.post(url, json=payload) -# flog.info(f'New Pipeline response: {response.text}') -# -# return response def pipeline_refresh_building_blocks(pipeline_uid:str): url = f'{BASE_API}/pipeline_refresh_building_blocks/{pipeline_uid}' @@ -1023,78 +794,8 @@ def update_popup_by_uid(popup_uid, pos=None, typ=None, params_dict=None): return response -# def get_all_popups(): -# url = f'{BASE_API}/popups' -# -# response = http_client.get(url) -# flog.info(f'GET all Popups response: {response.text}') -# -# return response - - -# def get_popup_by_uid(popup_uid): -# url = f'{BASE_API}/popup/{popup_uid}' -# -# response = http_client.get(url) -# flog.info(f'GET Popup response: {response.text}') -# -# return response - - -# def delete_popup_by_uid(popup_uid): -# url = f'{BASE_API}/popup/{str(popup_uid)}' -# response = http_client.delete(url) -# flog.info(f'DELETE Popup response: {response.text}') -# -# return response - - ############### Scripts ############### -# def new_script(script_name, text="", project_uid="0"): -# -# payload = f'{{"script_name":"{script_name}", "text":{json.dumps(text)}, "project_uid":{project_uid}}}' -# -# flog.info(f'New Script payload: {payload}') -# -# url = f'{SERVER}:{PORT}/api/v1/scripts' -# -# response = http_client.post(url, data=payload) -# flog.info(f'New Script response: {response.text}') -# -# return response -# -# -# def update_script_by_uid(script_uid, script_name, text, project_uid="0"): -# -# payload = f'{{"script_name":"{script_name}", "text":{json.dumps(text)}, "project_uid":{project_uid}}}' -# -# flog.info(f'Update Script by Uid Payload: {payload}') -# -# response = http_client.put(SERVER + ":" + str(PORT) + "/api/v1/script/" + str(script_uid), data=payload) -# result = json.loads(response.content.decode('utf-8')) -# -# flog.info(f'Update Script by Uid Result: {result}') -# -# return (response) - - - -# def get_all_scripts(): -# response=http_client.get(SERVER+":"+str(PORT)+"/api/v1/scripts") -# print("Get all scripts response: ", response) -# return(response) - - - -# def get_script_by_uid(script_uid): -# response=http_client.get(SERVER+":"+str(PORT)+"/api/v1/script/"+str(script_uid)) -# return(response) -# -# -# def delete_script_by_uid(script_uid): -# response=http_client.delete(SERVER+":"+str(PORT)+"/api/v1/script/"+str(script_uid)) -# return(response) def update_last_active_script(script_uid: Optional[str] = None): payload = { @@ -1161,66 +862,6 @@ def get_all_databases_by_project_uid(): return project_databases -# def get_all_databases(): -# response=http_client.get(SERVER+":"+str(PORT)+"/api/v1/databases") -# return(response) - -# def delete_database_by_uid(database_uid): -# response=http_client.delete(SERVER+":"+str(PORT)+"/api/v1/databases/"+str(database_uid)) -# flog.info(f'DELETE Database response: {response.text}') -# return(response) - -# def new_database(database_name, server, port, database, username, password, dialect, project_uid="0"): -# payload = { -# "database_name": database_name, -# "server": server, -# "port": port, -# "database": database, -# "username": username, -# "password": password, -# "dialect": dialect, -# } -# flog.info(f'New Dataset payload: {payload}') -# -# url = f'{SERVER}:{PORT}/api/v1/databases' -# -# response = http_client.post(url, json=payload) -# flog.info(f'New Database response: {response.text}') -# -# return response - -# def update_database_by_uid(database_uid, database_name, server, port, database, username, password, dialect, project_uid="0"): -# payload = { -# "database_name": database_name, -# "server": server, -# "port": port, -# "database": database, -# "username": username, -# "password": password, -# "dialect": dialect, -# } -# flog.info(f'New Dataset payload: {payload}') -# -# url = f'{SERVER}:{PORT}/api/v1/databases/{database_uid}' -# -# response = http_client.put(url, json=payload) -# flog.info(f'New Database response: {response.text}') -# -# return response - - -# def delete_script_by_uid(script_uid): -# response=http_client.delete(SERVER+":"+str(PORT)+"/api/v1/script/"+str(script_uid)) -# return response - -# def get_all_datasets(): -# response=http_client.get(SERVER+":"+str(PORT)+"/api/v1/datasets") -# return(response) - -# def delete_dataset_by_uid(dataset_uid): -# response=http_client.delete(SERVER+":"+str(PORT)+"/api/v1/dataset/"+str(dataset_uid)) -# return(response) - def store_df_to_google_sheet(dataset_uid:str, sheet_name:str, email:str): payload = { @@ -1268,21 +909,6 @@ def initialize_last_or_new_pipeline(project_uid): return response -# def new_dataset(dataset_name, data="", project_uid="0"): -# payload = { -# "dataset_name": dataset_name, -# "data": data, -# "project_uid": project_uid -# } -# flog.info(f'New Dataset payload: {payload}') -# -# url = f'{SERVER}:{PORT}/api/v1/datasets' -# -# response = http_client.post(url, json=payload) -# flog.info(f'New Script response: {response.text}') -# -# return response - def get_next_node_predictions(initial_node_uid: Optional[str] = None, is_used_for_autopilot: bool = False): payload = { "node_uid": initial_node_uid, From eaae0e69294821b1cdab1744a45aa03e0843d0dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Chm=C3=BArny?= Date: Sun, 17 Nov 2024 13:40:35 +0100 Subject: [PATCH 08/12] Add type-hints to ncrb functions --- .../queries/node_context_requests_backend.py | 149 ++++++++++-------- 1 file changed, 80 insertions(+), 69 deletions(-) diff --git a/forloop_modules/queries/node_context_requests_backend.py b/forloop_modules/queries/node_context_requests_backend.py index 2d699a1..3a91c04 100644 --- a/forloop_modules/queries/node_context_requests_backend.py +++ b/forloop_modules/queries/node_context_requests_backend.py @@ -256,7 +256,7 @@ def get_project_uid() -> Optional[str]: ############### Nodes ################# -def new_node(pos, typ, params_dict: Optional[dict] = None, fields: Optional[list] = None, visible: Optional[bool] = True): +def new_node(pos: list[int, int], typ: str, params_dict: Optional[dict] = None, fields: Optional[list] = None, visible: Optional[bool] = True) -> Response: project_uid = aet.project_uid pipeline_uid = aet.active_pipeline_uid @@ -269,7 +269,9 @@ def new_node(pos, typ, params_dict: Optional[dict] = None, fields: Optional[list "project_uid": project_uid, "visible": visible } - payload = {k:v for k,v in payload.items() if v is not None} # sending None breaks node reflection, no need to send empty field on node generation + + # sending None breaks node reflection, no need to send empty field on node generation + payload = {k:v for k,v in payload.items() if v is not None} flog.info(f'New Node payload: {payload}') @@ -282,7 +284,7 @@ def new_node(pos, typ, params_dict: Optional[dict] = None, fields: Optional[list return response -def direct_execute_node(node_uid): +def direct_execute_node(node_uid: str) -> Response: payload = dict() @@ -297,7 +299,7 @@ def direct_execute_node(node_uid): -def export_node_code(node_uid): +def export_node_code(node_uid: str) -> Response: payload = dict() @@ -313,7 +315,7 @@ def export_node_code(node_uid): -def get_node_by_uid(node_uid): +def get_node_by_uid(node_uid: str) -> Response: url = f'{BASE_API}/nodes/{node_uid}' response = http_client.get(url) @@ -322,14 +324,14 @@ def get_node_by_uid(node_uid): return response -def get_all_nodes(): +def get_all_nodes() -> Response: url = f'{BASE_API}/nodes' response = http_client.get(url) flog.debug(f'GET all Nodes response: {response.text}') return response -def delete_node_by_uid(node_uid): +def delete_node_by_uid(node_uid: str) -> Response: url = f'{BASE_API}/nodes/{node_uid}' response = http_client.delete(url) flog.info(f'DELETE Node response: {response.text}') @@ -337,7 +339,7 @@ def delete_node_by_uid(node_uid): return response -def delete_all_nodes(): +def delete_all_nodes() -> Response: pipeline_uid = aet.active_pipeline_uid url = f'{BASE_API}/pipelines/{pipeline_uid}/nodes' response = http_client.delete(url) @@ -345,7 +347,7 @@ def delete_all_nodes(): return response -def move_node_by_uid(node_uid, new_pos): +def move_node_by_uid(node_uid: str, new_pos: list[int, int]) -> Response: payload = { "new_pos": new_pos, } @@ -357,7 +359,7 @@ def move_node_by_uid(node_uid, new_pos): flog.info(f'Move Node response: {response.text}') return response -def node_breakpoint_status(node_uid, breakpoint_status): +def node_breakpoint_status(node_uid: str, breakpoint_status) -> Response: payload = { "uid": node_uid, "status": breakpoint_status, @@ -369,7 +371,7 @@ def node_breakpoint_status(node_uid, breakpoint_status): flog.info(f'Node breakpoint status: {response.text}') return response -def node_disabled_status(node_uid, disabled_status): +def node_disabled_status(node_uid: str, disabled_status) -> Response: payload = { "uid": node_uid, "status": disabled_status, @@ -381,13 +383,19 @@ def node_disabled_status(node_uid, disabled_status): flog.info(f'Node disabled status: {response.text}') return response -def update_node_params_by_node_detail_form(node_detail_form): +def update_node_params_by_node_detail_form(node_detail_form) -> Response: node_params = node_detail_form.node_params.params_dict_repr() node_uid = node_detail_form.node_uid update_node_by_uid(node_uid, params=node_params) -def update_node_by_uid(node_uid, pos=None, typ=None, params=None, fields=None): +def update_node_by_uid( + node_uid: str, + pos: Optional[list[int, int]] = None, + typ: Optional[str] = None, + params: Optional[dict] = None, + fields: Optional[list[dict]] = None, +) -> Response: # TODO: if inputs are None, it should not put them in payload # TODO: rename params to params_dict as in other functions @@ -424,7 +432,7 @@ def update_node_by_uid(node_uid, pos=None, typ=None, params=None, fields=None): ####### SPECIAL NODE - ITEM DETAIL FORM NODE BUTTON FUNCTIONS ######### -def node_button_click(node_uid, button_name): +def node_button_click(node_uid: str, button_name: str) -> Response: payload = { "button_name": button_name @@ -440,29 +448,26 @@ def node_button_click(node_uid, button_name): return response -# unwraps button function_args into multiple parameters (cant call node_button_click with function_args directly as it is stored in a list, but the function expects 2 parameters) -def node_button_click_wrapper(args): +# unwraps button function_args into multiple parameters (cant call node_button_click with +# function_args directly as it is stored in a list, but the function expects 2 parameters) +def node_button_click_wrapper(args) -> Response: assert len(args) == 2 - node_button_click(args[0], args[1]) - - - - - + response = node_button_click(args[0], args[1]) + return response ############### Edges ################# -def get_edges_by_node_uid(node_uid:str): +def get_edges_by_node_uid(node_uid: str) -> Response: url = f'{BASE_API}/nodes/{node_uid}/edges' response = http_client.get(url) flog.debug(f'GET Edges by Node Uid response: {response.text}') return response -def get_edge_by_connected_node_uids(from_node_uid:str, to_node_uid:str): +def get_edge_by_connected_node_uids(from_node_uid: str, to_node_uid: str) -> Response: url = f'{BASE_API}/edges/?from_node_uid={from_node_uid}&to_node_uid={to_node_uid}' response = http_client.get(url) flog.info(f'GET Edge by Connected Node Uids response: {response.text}') @@ -471,7 +476,7 @@ def get_edge_by_connected_node_uids(from_node_uid:str, to_node_uid:str): -def delete_edge_by_uid(uid): +def delete_edge_by_uid(uid: str) -> Response: url = f'{BASE_API}/edges/{uid}' response = http_client.delete(url) @@ -480,7 +485,7 @@ def delete_edge_by_uid(uid): return response -def delete_all_edges(): +def delete_all_edges() -> Response: pipeline_uid = aet.active_pipeline_uid url = f'{BASE_API}/pipelines/{pipeline_uid}/edges' response = http_client.delete(url) @@ -490,7 +495,7 @@ def delete_all_edges(): ### LAST ACTIVE DF -def update_last_active_dataframe_node_uid(last_active_dataframe_node_uid: Optional[str]): +def update_last_active_dataframe_node_uid(last_active_dataframe_node_uid: Optional[str]) -> Response: payload = { "project_uid": aet.project_uid, "last_active_dataframe_node_uid": last_active_dataframe_node_uid @@ -506,7 +511,7 @@ def update_last_active_dataframe_node_uid(last_active_dataframe_node_uid: Option return response -def get_last_active_dataframe_node_uid(): +def get_last_active_dataframe_node_uid() -> Response: url = f'{BASE_API}/last_active_dataframe_node_uid?project_uid={aet.project_uid}' response = http_client.get(url) flog.debug(f'GET Last active DF node_uid response: {response.text}') @@ -532,7 +537,7 @@ def get_variable(uid: str) -> Optional[dict]: return response.json() -def get_variable_by_name(variable_name: str): +def get_variable_by_name(variable_name: str) -> Response: pipeline_job_uid = aet.active_pipeline_job_uid url = f'{BASE_API}/variables?name={variable_name}&pipeline_job_uid={pipeline_job_uid}' @@ -544,7 +549,7 @@ def get_variable_by_name(variable_name: str): -def delete_variable_by_uid(variable_uid): +def delete_variable_by_uid(variable_uid: str) -> Response: url = f'{BASE_API}/variables/{variable_uid}' response = http_client.delete(url) @@ -554,7 +559,7 @@ def delete_variable_by_uid(variable_uid): return response -def delete_all_variables(): +def delete_all_variables() -> Response: pipeline_uid = aet.active_pipeline_uid url = f'{BASE_API}/pipelines/{pipeline_uid}/variables' response = http_client.delete(url) @@ -562,7 +567,7 @@ def delete_all_variables(): return response -def update_variable_by_uid(variable_uid: str, name: str, value: Any, is_result: bool = None, type = None, size: Optional[int] = None): +def update_variable_by_uid(variable_uid: str, name: str, value: Any, is_result: bool = None, type = None, size: Optional[int] = None) -> Response: project_uid = aet.project_uid pipeline_uid = aet.active_pipeline_uid pipeline_job_uid = aet.active_pipeline_job_uid @@ -585,25 +590,25 @@ def update_variable_by_uid(variable_uid: str, name: str, value: Any, is_result: payload["is_result"] = is_result response=http_client.put(f"{BASE_API}/variables/{variable_uid}",json=payload) - result=json.loads(response.content.decode('utf-8')) + return(response) -def get_job_variables(): +def get_job_variables() -> Response: job_uid = aet.active_pipeline_job_uid url = f'{BASE_API}/jobs/{job_uid}/variables' response = http_client.get(url) response.raise_for_status() return response -def cancel_pipeline_job(): +def cancel_pipeline_job() -> Response: job_uid = aet.active_pipeline_job_uid url = f'{BASE_API}/jobs/{job_uid}/cancel' response = http_client.post(url) response.raise_for_status() return response -def cancel_prototype_job(uid: str): +def cancel_prototype_job(uid: str) -> Response: url = f'{BASE_API}/prototype_jobs/{uid}/cancel' response = http_client.post(url=url) @@ -618,7 +623,7 @@ def consume_execution_stream(job_uid: str) -> Generator[dict, None, None]: ##### INITIAL VARIABLES ##### -def get_initial_variable_by_name(uid: str): +def get_initial_variable_by_name(uid: str) -> Response: pipeline_uid = aet.active_pipeline_uid url = f'{BASE_API}/initial_variables?name={uid}&pipeline_uid={pipeline_uid}' @@ -627,13 +632,13 @@ def get_initial_variable_by_name(uid: str): return response -def delete_initial_variable_by_uid(uid): +def delete_initial_variable_by_uid(uid: str) -> Response: response = http_client.delete(f'{BASE_API}/initial_variables/{uid}') response.raise_for_status() return response -def delete_all_initial_variables(): +def delete_all_initial_variables() -> Response: pipeline_uid = aet.active_pipeline_uid response = http_client.delete(f'{BASE_API}/pipelines/{pipeline_uid}/initial_variables') response.raise_for_status() @@ -642,7 +647,7 @@ def delete_all_initial_variables(): def update_initial_variable_by_uid( variable_uid: str, name: str, value: Any, is_result: bool, type=None, size: Optional[int] = None -): +) -> Response: project_uid = aet.project_uid pipeline_uid = aet.active_pipeline_uid pipeline_job_uid = aet.active_pipeline_job_uid @@ -663,7 +668,7 @@ def update_initial_variable_by_uid( ########### Files ############### -def upload_urls_from_file(path:str): +def upload_urls_from_file(path: str) -> Response: file = {'file': open(path, 'rb')} response = http_client.post(SERVER+":"+str(PORT)+"/api/v1/upload_urls_from_file", files=file) flog.info(f"file upload response: {response.text}") @@ -685,7 +690,7 @@ def delete_all_files(): ############## Pipelines ############## -def pipeline_refresh_building_blocks(pipeline_uid:str): +def pipeline_refresh_building_blocks(pipeline_uid:str) -> Response: url = f'{BASE_API}/pipeline_refresh_building_blocks/{pipeline_uid}' response = http_client.get(url) @@ -693,7 +698,7 @@ def pipeline_refresh_building_blocks(pipeline_uid:str): return response -def pipeline_refresh_running_blocks(pipeline_uid:str): +def pipeline_refresh_running_blocks(pipeline_uid:str) -> Response: url = f'{BASE_API}/pipeline_refresh_running_blocks/{pipeline_uid}' response = http_client.get(url) @@ -701,7 +706,7 @@ def pipeline_refresh_running_blocks(pipeline_uid:str): return response -def activate_pipeline(pipeline_uid: str, project_uid: str): +def activate_pipeline(pipeline_uid: str, project_uid: str) -> Response: url = f'{BASE_API}/projects/{project_uid}/pipelines/{pipeline_uid}/activate' response = http_client.post(url) @@ -712,12 +717,14 @@ def activate_pipeline(pipeline_uid: str, project_uid: str): ############ Popups ################### """In this file all functions should have response-like return value""" -def new_popup_wrapper(args_list): +def new_popup_wrapper(args_list) -> Response: assert len(args_list) > 1 - new_popup(*args_list) + response = new_popup(*args_list) + + return response -def new_popup(pos, typ, params_dict=None): +def new_popup(pos: list[int, int], typ: str, params_dict=None) -> Response: """ Generates new Popup via API. :param pos: position of new Popup as [x, y] @@ -752,7 +759,12 @@ def new_popup(pos, typ, params_dict=None): return response -def update_popup_by_uid(popup_uid, pos=None, typ=None, params_dict=None): +def update_popup_by_uid( + popup_uid: str, + pos: Optional[list[int, int]] = None, + typ: Optional[str] = None, + params_dict: dict = None, +) -> Response: """ Updates existing Popup via API. :param popup_uid: Popup ID @@ -797,7 +809,7 @@ def update_popup_by_uid(popup_uid, pos=None, typ=None, params_dict=None): ############### Scripts ############### -def update_last_active_script(script_uid: Optional[str] = None): +def update_last_active_script(script_uid: Optional[str] = None) -> Response: payload = { "project_uid": aet.project_uid, "uid": script_uid @@ -812,7 +824,7 @@ def update_last_active_script(script_uid: Optional[str] = None): return response -def get_last_active_script(): +def get_last_active_script() -> Response: url = f'{BASE_API}/last_active_script?project_uid={aet.project_uid}' response = http_client.get(url) @@ -822,7 +834,7 @@ def get_last_active_script(): #~#~#~#~#~##~#~#~#~# SCRIPTS END #~#~#~#~#~##~#~#~#~# -def run_pipeline_to_code_conversion(): +def run_pipeline_to_code_conversion() -> Response: payload = { "pipeline_uid": aet.active_pipeline_uid, "project_uid": aet.project_uid @@ -832,7 +844,7 @@ def run_pipeline_to_code_conversion(): return response -def run_code_to_pipeline_conversion(): +def run_code_to_pipeline_conversion() -> Response: payload = { "pipeline_uid": aet.active_pipeline_uid, "project_uid": aet.project_uid @@ -842,7 +854,7 @@ def run_code_to_pipeline_conversion(): return response -def run_inspect_node_code(node_uid:str): +def run_inspect_node_code(node_uid: str) -> Response: payload = { "uid": node_uid, "project_uid": aet.project_uid @@ -851,7 +863,7 @@ def run_inspect_node_code(node_uid:str): response = http_client.post(url=url, json=payload) return response -def get_all_databases_by_project_uid(): +def get_all_databases_by_project_uid() -> Response: response = http_client.get(SERVER+":"+str(PORT)+"/api/v1/databases") if response.status_code != 200: @@ -862,7 +874,7 @@ def get_all_databases_by_project_uid(): return project_databases -def store_df_to_google_sheet(dataset_uid:str, sheet_name:str, email:str): +def store_df_to_google_sheet(dataset_uid:str, sheet_name:str, email:str) -> Response: payload = { "dataset_uid": dataset_uid, @@ -881,7 +893,7 @@ def store_df_to_google_sheet(dataset_uid:str, sheet_name:str, email:str): -def initialize_last_or_new_project_by_email(email): +def initialize_last_or_new_project_by_email(email: str) -> Response: payload = { "email": email @@ -895,7 +907,7 @@ def initialize_last_or_new_project_by_email(email): return response -def initialize_last_or_new_pipeline(project_uid): +def initialize_last_or_new_pipeline(project_uid: str) -> Response: payload = { "project_uid": project_uid @@ -909,7 +921,7 @@ def initialize_last_or_new_pipeline(project_uid): return response -def get_next_node_predictions(initial_node_uid: Optional[str] = None, is_used_for_autopilot: bool = False): +def get_next_node_predictions(initial_node_uid: Optional[str] = None, is_used_for_autopilot: bool = False) -> Response: payload = { "node_uid": initial_node_uid, "is_used_for_autopilot": is_used_for_autopilot @@ -924,7 +936,7 @@ def get_next_node_predictions(initial_node_uid: Optional[str] = None, is_used_fo return response -def confirm_selected_node_prediction(node_uid: str, is_used_for_autopilot: bool = False): +def confirm_selected_node_prediction(node_uid: str, is_used_for_autopilot: bool = False) -> Response: payload = { "node_uid": node_uid, "is_used_for_autopilot": is_used_for_autopilot @@ -939,14 +951,14 @@ def confirm_selected_node_prediction(node_uid: str, is_used_for_autopilot: bool return response -def get_form_dict_list_templates(): +def get_form_dict_list_templates() -> Response: url = f'{BASE_API}/node_defs' response = http_client.get(url) flog.debug(f'GET form dict list templates: {response.text}') return response -def get_chatgpt_adjustment(user_input_text, openai_api_key): +def get_chatgpt_adjustment(user_input_text: str, openai_api_key: str) -> Response: payload = { "user_input_text": user_input_text, "openai_api_key" : openai_api_key @@ -961,7 +973,7 @@ def get_chatgpt_adjustment(user_input_text, openai_api_key): return response -def process_api_adjustments(adjustments_dict, project_uid): +def process_api_adjustments(adjustments_dict: dict, project_uid: str) -> Response: payload = { "adjustments_dict": adjustments_dict, "project_uid": project_uid @@ -972,24 +984,23 @@ def process_api_adjustments(adjustments_dict, project_uid): url = f'{BASE_API}/api_pipeline_adjustments' response = http_client.post(url, data=json.dumps(payload)) - # flog.info(f'Chatgpt adjustment: {response.text}') return response -def get_user_logs(): +def get_user_logs() -> Response: url = f'{BASE_API}/user_logs' response = http_client.get(url) return response -def clean_data(data): +def clean_data(data) -> Response: url = f'{BASE_API}/clean_data' response = http_client.post(url, data=data) return response -def scan_website_and_take_screenshot_test(email, url): +def scan_website_and_take_screenshot_test(email: str, url: str) -> Response: payload={"email":email, "url":url, "incl_tables": True, @@ -1007,7 +1018,7 @@ def scan_website_and_take_screenshot_test(email, url): -def finalize_pipeline(project_uid): +def finalize_pipeline(project_uid: str) -> Response: payload={"project_uid":project_uid, } url = f'{BASE_API}/finalize_pipeline' @@ -1016,7 +1027,7 @@ def finalize_pipeline(project_uid): return response -def pipeline_direct_execute(pipeline_uid: str, payload: dict): +def pipeline_direct_execute(pipeline_uid: str, payload: dict) -> Response: url = f'{BASE_API}/pipelines/{pipeline_uid}/direct_execute' response = http_client.post(url=url, json=payload) response.raise_for_status() @@ -1024,7 +1035,7 @@ def pipeline_direct_execute(pipeline_uid: str, payload: dict): return response -def filter_webpage_elements_based_on_objective(elements: list[dict], objective: str): +def filter_webpage_elements_based_on_objective(elements: list[dict], objective: str) -> Response: payload = { "elements": elements, "objective": objective From 6bb091ce9e3bbc4823c895cf1c96e742e89d53b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Chm=C3=BArny?= Date: Thu, 21 Nov 2024 07:44:29 +0100 Subject: [PATCH 09/12] Fix httpx responses' reason --- forloop_modules/function_handlers/browser_handlers.py | 2 +- forloop_modules/function_handlers/database_handlers.py | 2 +- .../queries/context_request_backend_auxiliary_functions.py | 2 +- forloop_modules/queries/node_context_requests_backend.py | 2 +- forloop_modules/utils/script_utils.py | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/forloop_modules/function_handlers/browser_handlers.py b/forloop_modules/function_handlers/browser_handlers.py index fdaac22..f6b44e3 100644 --- a/forloop_modules/function_handlers/browser_handlers.py +++ b/forloop_modules/function_handlers/browser_handlers.py @@ -554,7 +554,7 @@ def direct_execute(self, elements, objective): redis_action_key = redis_config.SCRAPING_ACTION_KEY_TEMPLATE.format(pipeline_uid=aet.active_pipeline_uid) kv_redis.set(redis_action_key, result) else: - raise CriticalPipelineError(response.reason) + raise CriticalPipelineError(response.reason_phrase) browser_handlers_dict = { diff --git a/forloop_modules/function_handlers/database_handlers.py b/forloop_modules/function_handlers/database_handlers.py index 97b5b71..a9222b2 100644 --- a/forloop_modules/function_handlers/database_handlers.py +++ b/forloop_modules/function_handlers/database_handlers.py @@ -30,7 +30,7 @@ def get_all_databases_in_project(): response = ncrb.get_all_databases() if response.status_code != 200: - raise Exception(f'Error {response.status_code}: {response.reason}.') + raise Exception(f'Error {response.status_code}: {response.reason_phrase}.') databases = response.json().get("result", {}).get("databases") project_databases = [database for database in databases if database["project_uid"] == aet.project_uid] diff --git a/forloop_modules/queries/context_request_backend_auxiliary_functions.py b/forloop_modules/queries/context_request_backend_auxiliary_functions.py index f9da725..43f22bb 100644 --- a/forloop_modules/queries/context_request_backend_auxiliary_functions.py +++ b/forloop_modules/queries/context_request_backend_auxiliary_functions.py @@ -56,7 +56,7 @@ def get_and_subset_requested_objects(ncrb_function, func_args:Optional[list]=Non result = filter_items_by_project_uid(result) return result else: - flog.warning(f'Status code {response.status_code}: {response.reason}') + flog.warning(f'Status code {response.status_code}: {response.reason_phrase}') def filter_items_by_project_uid(items:list): user_items = items diff --git a/forloop_modules/queries/node_context_requests_backend.py b/forloop_modules/queries/node_context_requests_backend.py index 3a91c04..46e244c 100644 --- a/forloop_modules/queries/node_context_requests_backend.py +++ b/forloop_modules/queries/node_context_requests_backend.py @@ -867,7 +867,7 @@ def get_all_databases_by_project_uid() -> Response: response = http_client.get(SERVER+":"+str(PORT)+"/api/v1/databases") if response.status_code != 200: - raise Exception(f'Error {response.status_code}: {response.reason}.') + raise Exception(f'Error {response.status_code}: {response.reason_phrase}.') databases = response.json()['databases'] project_databases = [database for database in databases if database["project_uid"] == aet.project_uid] diff --git a/forloop_modules/utils/script_utils.py b/forloop_modules/utils/script_utils.py index 6bfd979..e450890 100644 --- a/forloop_modules/utils/script_utils.py +++ b/forloop_modules/utils/script_utils.py @@ -32,7 +32,7 @@ def create_new_script(script_name: str = "untitled", text: str = ""): aet.active_script_uid = new_script_uid flog.info(f'New script created, uid "{new_script_uid}" stored.') else: - flog.error(f'Error {response.status_code}: {response.reason}.') + flog.error(f'Error {response.status_code}: {response.reason_phrase}.') def update_active_script(code: str, script_name: Optional[str] = "untitled"): """ @@ -50,7 +50,7 @@ def update_active_script(code: str, script_name: Optional[str] = "untitled"): ) if not response.ok: - flog.error(f'Error {response.status_code}: {response.reason}.') + flog.error(f'Error {response.status_code}: {response.reason_phrase}.') else: create_new_script(text=code) @@ -58,7 +58,7 @@ def get_script_by_name(script_name: str): response = ncrb.get_all_scripts() if not response.ok: - raise Exception(f'Error {response.status_code} - {response.reason}.') + raise Exception(f'Error {response.status_code} - {response.reason_phrase}.') scripts = response.json()["result"]["scripts"] scripts_matching_name = [ From dda0676776df060583294fd57b22e7f6ce421311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Chm=C3=BArny?= Date: Thu, 21 Nov 2024 08:05:16 +0100 Subject: [PATCH 10/12] Update docstrings and type-hints of new and update factory funcs --- .../queries/node_context_requests_backend.py | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/forloop_modules/queries/node_context_requests_backend.py b/forloop_modules/queries/node_context_requests_backend.py index 46e244c..8fa0a74 100644 --- a/forloop_modules/queries/node_context_requests_backend.py +++ b/forloop_modules/queries/node_context_requests_backend.py @@ -5,7 +5,7 @@ import sys from inspect import Parameter, Signature from pathlib import Path -from typing import Any, Generator, Optional +from typing import Any, Generator, Optional, Union from httpx import Response @@ -69,6 +69,8 @@ "initial_variables": APIInitialVariable, } +Model = Union[tuple(DB_API_BODY_TEMPLATE.values())] + def set_stored_project_uid_and_pipeline_uid_to_factory_payload(payload: dict): """ @@ -134,7 +136,7 @@ def delete_factory(resource_name: str): Factory creating a "DELETE " request function. Args: - resource_name (str): Name of a resource DELETE (e.g. node, edge, script etc.) + resource_name (str): Name of a resource to DELETE (e.g. node, edge, script etc.) Returns: ((resource_uid: str) -> Response): "DELETE " request calling function @@ -148,7 +150,18 @@ def delete(resource_uid: str): return delete -def new_factory(resource_name: str, model): +def new_factory(resource_name: str, model: Model): + """ + Factory creating a "POST " request function. + + Args: + resource_name (str): Name of a resource to POST (e.g. node, edge, script etc.) + model (Model): API model of the resource (e.g. "node -> APINode", "edge -> APIEdge" etc.) + + Returns: + ((..., model_attr_names_wo_uid: Any = model_attr_names_wo_uid) -> Response): "POST + " request calling function + """ # list of pydantic attributes model_attribute_names = list(vars(model()).keys()) # remove the uid attribute – see VariableModel and APIVariable @@ -183,7 +196,18 @@ def new( return new -def update_factory(resource_name: str, model): +def update_factory(resource_name: str, model: Model): + """ + Factory creating a "PUT " request function. + + Args: + resource_name (str): Name of a resource to PUT (e.g. node, edge, script etc.) + model (Model): API model of the resource (e.g. "node -> APINode", "edge -> APIEdge" etc.) + + Returns: + ((..., model_attr_names_wo_uid: Any = model_attr_names_wo_uid) -> Response): "PUT + " request calling function + """ # list of pydantic class attributes model_attribute_names = list(vars(model()).keys()) # remove the uid attribute – see VariableModel and APIVariable From f7e511e7326797c660bcd9f671c1f799f802f4bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Chm=C3=BArny?= Date: Thu, 21 Nov 2024 08:06:59 +0100 Subject: [PATCH 11/12] Ruff format ncrb --- .../queries/node_context_requests_backend.py | 558 +++++++++--------- 1 file changed, 291 insertions(+), 267 deletions(-) diff --git a/forloop_modules/queries/node_context_requests_backend.py b/forloop_modules/queries/node_context_requests_backend.py index 8fa0a74..e0c3070 100644 --- a/forloop_modules/queries/node_context_requests_backend.py +++ b/forloop_modules/queries/node_context_requests_backend.py @@ -28,16 +28,16 @@ from forloop_modules.utils.http_client import HttpClient if sys.platform == "darwin": # MAC OS - config_path = 'config/server_config_remote.ini' + config_path = "config/server_config_remote.ini" else: - config_path = 'config/server_config.ini' + config_path = "config/server_config.ini" -with Path(config_path).open(mode='r') as f: +with Path(config_path).open(mode="r") as f: rows = f.readlines() SERVER = rows[0].split("=")[1].strip() PORT = rows[1].split("=")[1].strip() -BASE_API = f'{SERVER}:{PORT}/api/v1' +BASE_API = f"{SERVER}:{PORT}/api/v1" http_client = HttpClient() RESOURCES = { @@ -82,8 +82,8 @@ def set_stored_project_uid_and_pipeline_uid_to_factory_payload(payload: dict): # This approach might not be safe if API validation becomes strict (not all calls take both # project_uid and pipeline_uid as parameters) # TODO: Think about a better implementation (due to the point above) - payload['project_uid'] = aet.project_uid - payload['pipeline_uid'] = aet.active_pipeline_uid + payload["project_uid"] = aet.project_uid + payload["pipeline_uid"] = aet.active_pipeline_uid def remove_none_values_from_payload(payload: dict) -> dict: @@ -101,7 +101,7 @@ def get_all_factory(resource_name: str): Returns: (() -> Response): "GET all " request calling function - """ + """ resource_url = f"{SERVER}:{str(PORT)}/api/v1/{resource_name}" def get_all() -> Response: @@ -121,7 +121,8 @@ def get_factory(resource_name: str): Returns: ((resource_uid: str) -> Response): "GET " request calling function - """ + """ + def get(resource_uid: str): resource_url = f"{BASE_API}/{resource_name}/{resource_uid}" response = http_client.get(resource_url) @@ -140,7 +141,8 @@ def delete_factory(resource_name: str): Returns: ((resource_uid: str) -> Response): "DELETE " request calling function - """ + """ + def delete(resource_uid: str): resource_url = f"{BASE_API}/{resource_name}/{resource_uid}" response = http_client.delete(resource_url) @@ -248,25 +250,25 @@ def update( resource_name_singular = resource_name[:-1] for action in actions: function_name = None - if action == 'get_all': + if action == "get_all": fn = get_all_factory(resource_name) - function_name = f'{action}_{resource_name}' - elif action == 'get': + function_name = f"{action}_{resource_name}" + elif action == "get": fn = get_factory(resource_name_singular) - function_name = f'{action}_{resource_name_singular}_by_uid' - elif action == 'new': + function_name = f"{action}_{resource_name_singular}_by_uid" + elif action == "new": model = DB_API_BODY_TEMPLATE[resource_name] fn = new_factory(resource_name, model) - function_name = f'{action}_{resource_name_singular}' - elif action == 'delete': + function_name = f"{action}_{resource_name_singular}" + elif action == "delete": fn = delete_factory(resource_name_singular) - function_name = f'{action}_{resource_name_singular}_by_uid' - elif action == 'update': + function_name = f"{action}_{resource_name_singular}_by_uid" + elif action == "update": model = DB_API_BODY_TEMPLATE[resource_name] fn = update_factory(resource_name_singular, model) - function_name = f'{action}_{resource_name_singular}_by_uid' + function_name = f"{action}_{resource_name_singular}_by_uid" else: - raise Exception('Unknown action') + raise Exception("Unknown action") globals()[function_name] = fn @@ -280,10 +282,16 @@ def get_project_uid() -> Optional[str]: ############### Nodes ################# -def new_node(pos: list[int, int], typ: str, params_dict: Optional[dict] = None, fields: Optional[list] = None, visible: Optional[bool] = True) -> Response: +def new_node( + pos: list[int, int], + typ: str, + params_dict: Optional[dict] = None, + fields: Optional[list] = None, + visible: Optional[bool] = True, +) -> Response: project_uid = aet.project_uid pipeline_uid = aet.active_pipeline_uid - + payload = { "pos": pos, "typ": typ, @@ -291,81 +299,76 @@ def new_node(pos: list[int, int], typ: str, params_dict: Optional[dict] = None, "fields": fields, "pipeline_uid": pipeline_uid, "project_uid": project_uid, - "visible": visible + "visible": visible, } # sending None breaks node reflection, no need to send empty field on node generation - payload = {k:v for k,v in payload.items() if v is not None} - + payload = {k: v for k, v in payload.items() if v is not None} - flog.info(f'New Node payload: {payload}') + flog.info(f"New Node payload: {payload}") - url = f'{BASE_API}/nodes' + url = f"{BASE_API}/nodes" response = http_client.post(url, json=payload) - flog.info(f'New Node response: {response.text}') + flog.info(f"New Node response: {response.text}") return response def direct_execute_node(node_uid: str) -> Response: - payload = dict() - flog.info(f'Direct execute Node payload: {payload}') + flog.info(f"Direct execute Node payload: {payload}") - url = f'{BASE_API}/node_direct_execute/{node_uid}' + url = f"{BASE_API}/node_direct_execute/{node_uid}" response = http_client.post(url, json=payload) - flog.info(f'Direct execute Node response: {response.text}') - - return response + flog.info(f"Direct execute Node response: {response.text}") + return response def export_node_code(node_uid: str) -> Response: - - payload = dict() - flog.info(f'Export Node code payload: {payload}') + flog.info(f"Export Node code payload: {payload}") - url = f'{BASE_API}/nodes/{node_uid}/export_code' + url = f"{BASE_API}/nodes/{node_uid}/export_code" response = http_client.post(url, json=payload) - flog.info(f'Export Node code response: {response.text}') - + flog.info(f"Export Node code response: {response.text}") + return response - def get_node_by_uid(node_uid: str) -> Response: - url = f'{BASE_API}/nodes/{node_uid}' + url = f"{BASE_API}/nodes/{node_uid}" response = http_client.get(url) - flog.info(f'GET Node response: {response.text}') - + flog.info(f"GET Node response: {response.text}") + return response def get_all_nodes() -> Response: - url = f'{BASE_API}/nodes' + url = f"{BASE_API}/nodes" response = http_client.get(url) - flog.debug(f'GET all Nodes response: {response.text}') + flog.debug(f"GET all Nodes response: {response.text}") return response - + + def delete_node_by_uid(node_uid: str) -> Response: - url = f'{BASE_API}/nodes/{node_uid}' + url = f"{BASE_API}/nodes/{node_uid}" response = http_client.delete(url) - flog.info(f'DELETE Node response: {response.text}') + flog.info(f"DELETE Node response: {response.text}") return response - + def delete_all_nodes() -> Response: pipeline_uid = aet.active_pipeline_uid - url = f'{BASE_API}/pipelines/{pipeline_uid}/nodes' + url = f"{BASE_API}/pipelines/{pipeline_uid}/nodes" response = http_client.delete(url) response.raise_for_status() return response @@ -376,37 +379,39 @@ def move_node_by_uid(node_uid: str, new_pos: list[int, int]) -> Response: "new_pos": new_pos, } - - flog.info(f'Move Node payload: {payload}') - url = f'{BASE_API}/nodes/{node_uid}/move' + flog.info(f"Move Node payload: {payload}") + url = f"{BASE_API}/nodes/{node_uid}/move" response = http_client.put(url, json=payload) - flog.info(f'Move Node response: {response.text}') + flog.info(f"Move Node response: {response.text}") return response + def node_breakpoint_status(node_uid: str, breakpoint_status) -> Response: payload = { "uid": node_uid, "status": breakpoint_status, } - flog.info(f'Node breakpoint status: {payload}') - url = f'{BASE_API}/nodes/{node_uid}/breakpoint' + flog.info(f"Node breakpoint status: {payload}") + url = f"{BASE_API}/nodes/{node_uid}/breakpoint" response = http_client.put(url, json=payload) - flog.info(f'Node breakpoint status: {response.text}') + flog.info(f"Node breakpoint status: {response.text}") return response + def node_disabled_status(node_uid: str, disabled_status) -> Response: payload = { "uid": node_uid, "status": disabled_status, } - flog.info(f'Node disabled status: {payload}') - url = f'{BASE_API}/nodes/{node_uid}/status' + flog.info(f"Node disabled status: {payload}") + url = f"{BASE_API}/nodes/{node_uid}/status" response = http_client.put(url, json=payload) - flog.info(f'Node disabled status: {response.text}') + flog.info(f"Node disabled status: {response.text}") return response + def update_node_params_by_node_detail_form(node_detail_form) -> Response: node_params = node_detail_form.node_params.params_dict_repr() node_uid = node_detail_form.node_uid @@ -426,13 +431,13 @@ def update_node_by_uid( project_uid = aet.project_uid pipeline_uid = aet.active_pipeline_uid - #Do not erase this until checked that all variables correctly parsed from json payload - #payload_pos=json.dumps(pos) #pos was sometimes dumped as "[123,233]" which could not be processed by API - #if '"' in payload_pos: #Remove apostrophes from list: + # Do not erase this until checked that all variables correctly parsed from json payload + # payload_pos=json.dumps(pos) #pos was sometimes dumped as "[123,233]" which could not be processed by API + # if '"' in payload_pos: #Remove apostrophes from list: # payload_pos=payload_pos.replace('"','') - #payload='{"pos":'+payload_pos+',"typ":'+json.dumps(typ)+',"params":'+json.dumps(params)+',"fields":'+json.dumps(fields)+',"project_uid":"'+project_uid+'"}'#+'}' # - #response=http_client.put(SERVER+":"+str(PORT)+"/api/v1/nodes/"+str(node_uid),data=payload) - #return(response) + # payload='{"pos":'+payload_pos+',"typ":'+json.dumps(typ)+',"params":'+json.dumps(params)+',"fields":'+json.dumps(fields)+',"project_uid":"'+project_uid+'"}'#+'}' # + # response=http_client.put(SERVER+":"+str(PORT)+"/api/v1/nodes/"+str(node_uid),data=payload) + # return(response) # type=None has to be send, otherwise the icon type changes to "Custom" payload = { @@ -441,34 +446,32 @@ def update_node_by_uid( "params": params, "fields": fields, "project_uid": project_uid, - "pipeline_uid": pipeline_uid + "pipeline_uid": pipeline_uid, } - flog.info(f'Updated Node payload: {payload}') + flog.info(f"Updated Node payload: {payload}") - url = f'{BASE_API}/nodes/{node_uid}' + url = f"{BASE_API}/nodes/{node_uid}" response = http_client.put(url, json=payload) - flog.info(f'Updated Node response: {response.text}') + flog.info(f"Updated Node response: {response.text}") return response ####### SPECIAL NODE - ITEM DETAIL FORM NODE BUTTON FUNCTIONS ######### + def node_button_click(node_uid: str, button_name: str) -> Response: - - payload = { - "button_name": button_name - } + payload = {"button_name": button_name} - flog.info(f'Direct execute Node payload: {payload}') + flog.info(f"Direct execute Node payload: {payload}") - url = f'{BASE_API}/node_click_button/{node_uid}' + url = f"{BASE_API}/node_click_button/{node_uid}" response = http_client.post(url, json=payload) - flog.info(f'Direct execute Node response: {response.text}') - + flog.info(f"Direct execute Node response: {response.text}") + return response @@ -485,60 +488,63 @@ def node_button_click_wrapper(args) -> Response: def get_edges_by_node_uid(node_uid: str) -> Response: - url = f'{BASE_API}/nodes/{node_uid}/edges' + url = f"{BASE_API}/nodes/{node_uid}/edges" response = http_client.get(url) - flog.debug(f'GET Edges by Node Uid response: {response.text}') + flog.debug(f"GET Edges by Node Uid response: {response.text}") return response + def get_edge_by_connected_node_uids(from_node_uid: str, to_node_uid: str) -> Response: - url = f'{BASE_API}/edges/?from_node_uid={from_node_uid}&to_node_uid={to_node_uid}' + url = f"{BASE_API}/edges/?from_node_uid={from_node_uid}&to_node_uid={to_node_uid}" response = http_client.get(url) - flog.info(f'GET Edge by Connected Node Uids response: {response.text}') + flog.info(f"GET Edge by Connected Node Uids response: {response.text}") return response - def delete_edge_by_uid(uid: str) -> Response: - url = f'{BASE_API}/edges/{uid}' + url = f"{BASE_API}/edges/{uid}" response = http_client.delete(url) - flog.info(f'DELETE Edge response: {response.text}') + flog.info(f"DELETE Edge response: {response.text}") return response def delete_all_edges() -> Response: pipeline_uid = aet.active_pipeline_uid - url = f'{BASE_API}/pipelines/{pipeline_uid}/edges' + url = f"{BASE_API}/pipelines/{pipeline_uid}/edges" response = http_client.delete(url) response.raise_for_status() return response + ### LAST ACTIVE DF -def update_last_active_dataframe_node_uid(last_active_dataframe_node_uid: Optional[str]) -> Response: +def update_last_active_dataframe_node_uid( + last_active_dataframe_node_uid: Optional[str], +) -> Response: payload = { "project_uid": aet.project_uid, - "last_active_dataframe_node_uid": last_active_dataframe_node_uid + "last_active_dataframe_node_uid": last_active_dataframe_node_uid, } - flog.info(f'Last active DF node_uid payload: {payload}') + flog.info(f"Last active DF node_uid payload: {payload}") - url = f'{BASE_API}/last_active_dataframe_node_uid' + url = f"{BASE_API}/last_active_dataframe_node_uid" response = http_client.put(url, json=payload) - flog.info(f'Last active DF node_uid response: {response.text}') + flog.info(f"Last active DF node_uid response: {response.text}") return response def get_last_active_dataframe_node_uid() -> Response: - url = f'{BASE_API}/last_active_dataframe_node_uid?project_uid={aet.project_uid}' + url = f"{BASE_API}/last_active_dataframe_node_uid?project_uid={aet.project_uid}" response = http_client.get(url) - flog.debug(f'GET Last active DF node_uid response: {response.text}') + flog.debug(f"GET Last active DF node_uid response: {response.text}") return response @@ -550,7 +556,7 @@ def get_last_active_dataframe_node_uid() -> Response: def get_variable(uid: str) -> Optional[dict]: - url = f'{BASE_API}/variables/{uid}' + url = f"{BASE_API}/variables/{uid}" response = http_client.get(url) try: response.raise_for_status() @@ -563,44 +569,52 @@ def get_variable(uid: str) -> Optional[dict]: def get_variable_by_name(variable_name: str) -> Response: pipeline_job_uid = aet.active_pipeline_job_uid - url = f'{BASE_API}/variables?name={variable_name}&pipeline_job_uid={pipeline_job_uid}' + url = ( + f"{BASE_API}/variables?name={variable_name}&pipeline_job_uid={pipeline_job_uid}" + ) response = http_client.get(url) response.raise_for_status() - flog.info(f'GET Variable by name response: {response.text}') + flog.info(f"GET Variable by name response: {response.text}") return response - def delete_variable_by_uid(variable_uid: str) -> Response: - url = f'{BASE_API}/variables/{variable_uid}' + url = f"{BASE_API}/variables/{variable_uid}" response = http_client.delete(url) response.raise_for_status() - flog.info(f'DELETE Variable response: {response.text}') + flog.info(f"DELETE Variable response: {response.text}") return response def delete_all_variables() -> Response: pipeline_uid = aet.active_pipeline_uid - url = f'{BASE_API}/pipelines/{pipeline_uid}/variables' + url = f"{BASE_API}/pipelines/{pipeline_uid}/variables" response = http_client.delete(url) response.raise_for_status() return response -def update_variable_by_uid(variable_uid: str, name: str, value: Any, is_result: bool = None, type = None, size: Optional[int] = None) -> Response: +def update_variable_by_uid( + variable_uid: str, + name: str, + value: Any, + is_result: bool = None, + type=None, + size: Optional[int] = None, +) -> Response: project_uid = aet.project_uid pipeline_uid = aet.active_pipeline_uid pipeline_job_uid = aet.active_pipeline_job_uid if type is None: - for std_type in [str,int,list,dict,float,bool]: - if isinstance(value,std_type): - type=std_type.__name__ - + for std_type in [str, int, list, dict, float, bool]: + if isinstance(value, std_type): + type = std_type.__name__ + payload = { "name": name, "value": value, @@ -608,39 +622,42 @@ def update_variable_by_uid(variable_uid: str, name: str, value: Any, is_result: "size": size, "project_uid": project_uid, "pipeline_uid": pipeline_uid, - "pipeline_job_uid": pipeline_job_uid - } + "pipeline_job_uid": pipeline_job_uid, + } if is_result is not None: payload["is_result"] = is_result - response=http_client.put(f"{BASE_API}/variables/{variable_uid}",json=payload) + response = http_client.put(f"{BASE_API}/variables/{variable_uid}", json=payload) - return(response) + return response def get_job_variables() -> Response: job_uid = aet.active_pipeline_job_uid - url = f'{BASE_API}/jobs/{job_uid}/variables' + url = f"{BASE_API}/jobs/{job_uid}/variables" response = http_client.get(url) response.raise_for_status() return response + def cancel_pipeline_job() -> Response: job_uid = aet.active_pipeline_job_uid - url = f'{BASE_API}/jobs/{job_uid}/cancel' + url = f"{BASE_API}/jobs/{job_uid}/cancel" response = http_client.post(url) response.raise_for_status() return response + def cancel_prototype_job(uid: str) -> Response: - url = f'{BASE_API}/prototype_jobs/{uid}/cancel' + url = f"{BASE_API}/prototype_jobs/{uid}/cancel" response = http_client.post(url=url) - + return response + def consume_execution_stream(job_uid: str) -> Generator[dict, None, None]: """Run in a separate thread as this is a blocking operation.""" - url = f'{BASE_API}/jobs/{job_uid}/execution_stream' + url = f"{BASE_API}/jobs/{job_uid}/execution_stream" yield from http_client.sse_stream("GET", url, as_dict=True) @@ -649,7 +666,7 @@ def consume_execution_stream(job_uid: str) -> Generator[dict, None, None]: def get_initial_variable_by_name(uid: str) -> Response: pipeline_uid = aet.active_pipeline_uid - url = f'{BASE_API}/initial_variables?name={uid}&pipeline_uid={pipeline_uid}' + url = f"{BASE_API}/initial_variables?name={uid}&pipeline_uid={pipeline_uid}" response = http_client.get(url) response.raise_for_status() @@ -657,20 +674,27 @@ def get_initial_variable_by_name(uid: str) -> Response: def delete_initial_variable_by_uid(uid: str) -> Response: - response = http_client.delete(f'{BASE_API}/initial_variables/{uid}') + response = http_client.delete(f"{BASE_API}/initial_variables/{uid}") response.raise_for_status() return response def delete_all_initial_variables() -> Response: pipeline_uid = aet.active_pipeline_uid - response = http_client.delete(f'{BASE_API}/pipelines/{pipeline_uid}/initial_variables') + response = http_client.delete( + f"{BASE_API}/pipelines/{pipeline_uid}/initial_variables" + ) response.raise_for_status() return response def update_initial_variable_by_uid( - variable_uid: str, name: str, value: Any, is_result: bool, type=None, size: Optional[int] = None + variable_uid: str, + name: str, + value: Any, + is_result: bool, + type=None, + size: Optional[int] = None, ) -> Response: project_uid = aet.project_uid pipeline_uid = aet.active_pipeline_uid @@ -682,56 +706,71 @@ def update_initial_variable_by_uid( type = std_type.__name__ payload = { - "name": name, "value": value, "type": type, "size": size, "project_uid": project_uid, - "pipeline_uid": pipeline_uid, "pipeline_job_uid": pipeline_job_uid, "is_result": is_result + "name": name, + "value": value, + "type": type, + "size": size, + "project_uid": project_uid, + "pipeline_uid": pipeline_uid, + "pipeline_job_uid": pipeline_job_uid, + "is_result": is_result, } - response = http_client.put(f"{BASE_API}/initial_variables/{variable_uid}", json=payload) + response = http_client.put( + f"{BASE_API}/initial_variables/{variable_uid}", json=payload + ) response.raise_for_status() return response ########### Files ############### + def upload_urls_from_file(path: str) -> Response: - file = {'file': open(path, 'rb')} - response = http_client.post(SERVER+":"+str(PORT)+"/api/v1/upload_urls_from_file", files=file) + file = {"file": open(path, "rb")} + response = http_client.post( + SERVER + ":" + str(PORT) + "/api/v1/upload_urls_from_file", files=file + ) flog.info(f"file upload response: {response.text}") return response - + def delete_all_files(): # TODO FIX PROJECT UID - # project_uid = + # project_uid = project_uid = get_project_uid() - payload='{"project_uid":"'+project_uid+'"}' + payload = '{"project_uid":"' + project_uid + '"}' flog.info(payload) - response=http_client.delete(SERVER+":"+str(PORT)+"/api/v1/files",data=payload) - return(response) + response = http_client.delete( + SERVER + ":" + str(PORT) + "/api/v1/files", data=payload + ) + return response ############## Pipelines ############## -def pipeline_refresh_building_blocks(pipeline_uid:str) -> Response: - url = f'{BASE_API}/pipeline_refresh_building_blocks/{pipeline_uid}' +def pipeline_refresh_building_blocks(pipeline_uid: str) -> Response: + url = f"{BASE_API}/pipeline_refresh_building_blocks/{pipeline_uid}" response = http_client.get(url) - flog.info(f'GET Pipeline Refresh Building Blocks response: {response.text}') + flog.info(f"GET Pipeline Refresh Building Blocks response: {response.text}") return response -def pipeline_refresh_running_blocks(pipeline_uid:str) -> Response: - url = f'{BASE_API}/pipeline_refresh_running_blocks/{pipeline_uid}' + +def pipeline_refresh_running_blocks(pipeline_uid: str) -> Response: + url = f"{BASE_API}/pipeline_refresh_running_blocks/{pipeline_uid}" response = http_client.get(url) - flog.info(f'GET Pipeline Refresh Building Blocks response: {response.text}') + flog.info(f"GET Pipeline Refresh Building Blocks response: {response.text}") return response + def activate_pipeline(pipeline_uid: str, project_uid: str) -> Response: - url = f'{BASE_API}/projects/{project_uid}/pipelines/{pipeline_uid}/activate' + url = f"{BASE_API}/projects/{project_uid}/pipelines/{pipeline_uid}/activate" response = http_client.post(url) response.raise_for_status() @@ -741,6 +780,7 @@ def activate_pipeline(pipeline_uid: str, project_uid: str) -> Response: ############ Popups ################### """In this file all functions should have response-like return value""" + def new_popup_wrapper(args_list) -> Response: assert len(args_list) > 1 response = new_popup(*args_list) @@ -770,15 +810,15 @@ def new_popup(pos: list[int, int], typ: str, params_dict=None) -> Response: "pos": pos, "typ": typ, "params": params_dict, - "project_uid": project_uid + "project_uid": project_uid, } - flog.info(f'New Popup payload: {payload}') + flog.info(f"New Popup payload: {payload}") - url = f'{BASE_API}/popups' + url = f"{BASE_API}/popups" response = http_client.post(url, json=payload) - flog.info(f'New Popup response: {response.text}') + flog.info(f"New Popup response: {response.text}") return response @@ -809,23 +849,22 @@ def update_popup_by_uid( project_uid = get_project_uid() payload = {} - payload["uid"]=popup_uid - + payload["uid"] = popup_uid + if pos is not None: - payload["pos"]=pos + payload["pos"] = pos if typ is not None: - payload["typ"]=typ + payload["typ"] = typ if params_dict is not None: - payload["params"]=params_dict - payload["project_uid"]=project_uid - - - flog.info(f'Updated Popup payload: {payload}') + payload["params"] = params_dict + payload["project_uid"] = project_uid - url = f'{BASE_API}/popup/{popup_uid}' + flog.info(f"Updated Popup payload: {payload}") + + url = f"{BASE_API}/popup/{popup_uid}" response = http_client.put(url, json=payload) - flog.info(f'Updated Popup response: {response.text}') + flog.info(f"Updated Popup response: {response.text}") return response @@ -834,238 +873,223 @@ def update_popup_by_uid( def update_last_active_script(script_uid: Optional[str] = None) -> Response: - payload = { - "project_uid": aet.project_uid, - "uid": script_uid - } - flog.info(f'Last active Script payload: {payload}') + payload = {"project_uid": aet.project_uid, "uid": script_uid} + flog.info(f"Last active Script payload: {payload}") - url = f'{BASE_API}/last_active_script' + url = f"{BASE_API}/last_active_script" response = http_client.put(url, json=payload) - - flog.info(f'Last active Script response: {response.text}') + + flog.info(f"Last active Script response: {response.text}") return response def get_last_active_script() -> Response: - url = f'{BASE_API}/last_active_script?project_uid={aet.project_uid}' + url = f"{BASE_API}/last_active_script?project_uid={aet.project_uid}" response = http_client.get(url) - flog.info(f'GET Last active Script response: {response.text}') + flog.info(f"GET Last active Script response: {response.text}") return response -#~#~#~#~#~##~#~#~#~# SCRIPTS END #~#~#~#~#~##~#~#~#~# + +# ~#~#~#~#~##~#~#~#~# SCRIPTS END #~#~#~#~#~##~#~#~#~# + def run_pipeline_to_code_conversion() -> Response: - payload = { - "pipeline_uid": aet.active_pipeline_uid, - "project_uid": aet.project_uid - } + payload = {"pipeline_uid": aet.active_pipeline_uid, "project_uid": aet.project_uid} url = f"{BASE_API}/pipeline_to_code" response = http_client.post(url=url, json=payload) - + return response + def run_code_to_pipeline_conversion() -> Response: - payload = { - "pipeline_uid": aet.active_pipeline_uid, - "project_uid": aet.project_uid - } + payload = {"pipeline_uid": aet.active_pipeline_uid, "project_uid": aet.project_uid} url = f"{BASE_API}/code_to_pipeline" response = http_client.post(url=url, json=payload) - + return response + def run_inspect_node_code(node_uid: str) -> Response: - payload = { - "uid": node_uid, - "project_uid": aet.project_uid - } + payload = {"uid": node_uid, "project_uid": aet.project_uid} url = f"{BASE_API}/inspect_node_code" response = http_client.post(url=url, json=payload) return response + def get_all_databases_by_project_uid() -> Response: - response = http_client.get(SERVER+":"+str(PORT)+"/api/v1/databases") - + response = http_client.get(SERVER + ":" + str(PORT) + "/api/v1/databases") + if response.status_code != 200: - raise Exception(f'Error {response.status_code}: {response.reason_phrase}.') - - databases = response.json()['databases'] - project_databases = [database for database in databases if database["project_uid"] == aet.project_uid] - + raise Exception(f"Error {response.status_code}: {response.reason_phrase}.") + + databases = response.json()["databases"] + project_databases = [ + database for database in databases if database["project_uid"] == aet.project_uid + ] + return project_databases -def store_df_to_google_sheet(dataset_uid:str, sheet_name:str, email:str) -> Response: - - payload = { - "dataset_uid": dataset_uid, - "sheet_name": sheet_name, - "email": email - } - flog.info(f'Store Df to Google Sheet payload: {payload}') +def store_df_to_google_sheet(dataset_uid: str, sheet_name: str, email: str) -> Response: + payload = {"dataset_uid": dataset_uid, "sheet_name": sheet_name, "email": email} - url = f'{SERVER}:{PORT}/api/v1/store_df_to_google_sheet' + flog.info(f"Store Df to Google Sheet payload: {payload}") - response = http_client.post(url, json = payload) - flog.info(f'Store Df to Google Sheet response: {response.text}') + url = f"{SERVER}:{PORT}/api/v1/store_df_to_google_sheet" - return response + response = http_client.post(url, json=payload) + flog.info(f"Store Df to Google Sheet response: {response.text}") + return response def initialize_last_or_new_project_by_email(email: str) -> Response: - - payload = { - "email": email - } - + payload = {"email": email} - url = f'{BASE_API}/initialize_last_or_new_project_by_email' + url = f"{BASE_API}/initialize_last_or_new_project_by_email" response = http_client.post(url, json=payload) - flog.info(f'Response: {response.text}') + flog.info(f"Response: {response.text}") return response -def initialize_last_or_new_pipeline(project_uid: str) -> Response: - - payload = { - "project_uid": project_uid - } +def initialize_last_or_new_pipeline(project_uid: str) -> Response: + payload = {"project_uid": project_uid} - url = f'{BASE_API}/initialize_last_or_new_pipeline?project_uid='+str(project_uid) + url = f"{BASE_API}/initialize_last_or_new_pipeline?project_uid=" + str(project_uid) response = http_client.post(url, json=payload) - flog.info(f'Response: {response.text}') + flog.info(f"Response: {response.text}") return response -def get_next_node_predictions(initial_node_uid: Optional[str] = None, is_used_for_autopilot: bool = False) -> Response: + +def get_next_node_predictions( + initial_node_uid: Optional[str] = None, is_used_for_autopilot: bool = False +) -> Response: payload = { "node_uid": initial_node_uid, - "is_used_for_autopilot": is_used_for_autopilot + "is_used_for_autopilot": is_used_for_autopilot, } - flog.info(f'GET Next Node Predictions payload: {payload}') - - url = f'{SERVER}:{PORT}/api/v1/next_node_predictions' + flog.info(f"GET Next Node Predictions payload: {payload}") + + url = f"{SERVER}:{PORT}/api/v1/next_node_predictions" response = http_client.get(url, json=payload) - flog.info(f'GET Next Node Predictions response: {response.text}') + flog.info(f"GET Next Node Predictions response: {response.text}") return response -def confirm_selected_node_prediction(node_uid: str, is_used_for_autopilot: bool = False) -> Response: - payload = { - "node_uid": node_uid, - "is_used_for_autopilot": is_used_for_autopilot - } - flog.info(f'POST Next Node Predictions payload: {payload}') - - url = f'{SERVER}:{PORT}/api/v1/next_node_predictions' +def confirm_selected_node_prediction( + node_uid: str, is_used_for_autopilot: bool = False +) -> Response: + payload = {"node_uid": node_uid, "is_used_for_autopilot": is_used_for_autopilot} + + flog.info(f"POST Next Node Predictions payload: {payload}") + + url = f"{SERVER}:{PORT}/api/v1/next_node_predictions" response = http_client.post(url, json=payload) - flog.info(f'POST Next Node Predictions response: {response.text}') + flog.info(f"POST Next Node Predictions response: {response.text}") return response + def get_form_dict_list_templates() -> Response: - url = f'{BASE_API}/node_defs' + url = f"{BASE_API}/node_defs" response = http_client.get(url) - flog.debug(f'GET form dict list templates: {response.text}') + flog.debug(f"GET form dict list templates: {response.text}") return response + def get_chatgpt_adjustment(user_input_text: str, openai_api_key: str) -> Response: - payload = { - "user_input_text": user_input_text, - "openai_api_key" : openai_api_key - } + payload = {"user_input_text": user_input_text, "openai_api_key": openai_api_key} - flog.info(f'pipeline_adjustment_chatgpt_nlp payload: {payload}') + flog.info(f"pipeline_adjustment_chatgpt_nlp payload: {payload}") - url = f'{BASE_API}/pipeline_adjustment_chatgpt_nlp' + url = f"{BASE_API}/pipeline_adjustment_chatgpt_nlp" response = http_client.post(url, data=json.dumps(payload)) - flog.info(f'Chatgpt adjustment: {response.text}') + flog.info(f"Chatgpt adjustment: {response.text}") return response + def process_api_adjustments(adjustments_dict: dict, project_uid: str) -> Response: - payload = { - "adjustments_dict": adjustments_dict, - "project_uid": project_uid - } + payload = {"adjustments_dict": adjustments_dict, "project_uid": project_uid} - flog.info(f'api_pipeline_adjustments payload: {payload}') + flog.info(f"api_pipeline_adjustments payload: {payload}") - url = f'{BASE_API}/api_pipeline_adjustments' + url = f"{BASE_API}/api_pipeline_adjustments" response = http_client.post(url, data=json.dumps(payload)) return response + def get_user_logs() -> Response: - url = f'{BASE_API}/user_logs' + url = f"{BASE_API}/user_logs" response = http_client.get(url) return response def clean_data(data) -> Response: - url = f'{BASE_API}/clean_data' + url = f"{BASE_API}/clean_data" response = http_client.post(url, data=data) return response def scan_website_and_take_screenshot_test(email: str, url: str) -> Response: - payload={"email":email, - "url":url, - "incl_tables": True, - "incl_bullets": True, - "incl_texts": True, - "incl_headlines": True, - "incl_links": True, - "incl_images": True, - "incl_buttons": True, - "xpath":""} - url = f'{BASE_API}/website_screenshot_and_scan_test' - response=http_client.post(url=url,json=payload) + payload = { + "email": email, + "url": url, + "incl_tables": True, + "incl_bullets": True, + "incl_texts": True, + "incl_headlines": True, + "incl_links": True, + "incl_images": True, + "incl_buttons": True, + "xpath": "", + } + url = f"{BASE_API}/website_screenshot_and_scan_test" + response = http_client.post(url=url, json=payload) print(json.loads(response.content)) return response - def finalize_pipeline(project_uid: str) -> Response: - payload={"project_uid":project_uid, - } - url = f'{BASE_API}/finalize_pipeline' - response=http_client.post(url=url,json=payload) + payload = { + "project_uid": project_uid, + } + url = f"{BASE_API}/finalize_pipeline" + response = http_client.post(url=url, json=payload) print(json.loads(response.content)) return response def pipeline_direct_execute(pipeline_uid: str, payload: dict) -> Response: - url = f'{BASE_API}/pipelines/{pipeline_uid}/direct_execute' + url = f"{BASE_API}/pipelines/{pipeline_uid}/direct_execute" response = http_client.post(url=url, json=payload) response.raise_for_status() return response -def filter_webpage_elements_based_on_objective(elements: list[dict], objective: str) -> Response: - payload = { - "elements": elements, - "objective": objective - } - - url = f'{BASE_API}/filter_webpage_elements_based_on_objective' +def filter_webpage_elements_based_on_objective( + elements: list[dict], objective: str +) -> Response: + payload = {"elements": elements, "objective": objective} + + url = f"{BASE_API}/filter_webpage_elements_based_on_objective" response = http_client.post(url=url, json=payload, timeout=None) - + return response From f92c20e0290eb52fd07efd0adaf76e6fe90de179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Chm=C3=BArny?= Date: Thu, 21 Nov 2024 08:09:08 +0100 Subject: [PATCH 12/12] Remove redundant comments --- forloop_modules/queries/node_context_requests_backend.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/forloop_modules/queries/node_context_requests_backend.py b/forloop_modules/queries/node_context_requests_backend.py index e0c3070..0189fe1 100644 --- a/forloop_modules/queries/node_context_requests_backend.py +++ b/forloop_modules/queries/node_context_requests_backend.py @@ -549,9 +549,6 @@ def get_last_active_dataframe_node_uid() -> Response: return response -##### VARIABLES ##### - - ########### Variables ############### @@ -778,7 +775,6 @@ def activate_pipeline(pipeline_uid: str, project_uid: str) -> Response: ############ Popups ################### -"""In this file all functions should have response-like return value""" def new_popup_wrapper(args_list) -> Response: