diff --git a/.gitignore b/.gitignore index debb386..43faab2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,6 @@ _build venv .vscode .coverage -cov.xml \ No newline at end of file +api-saagie.iml +out/ +.env diff --git a/README.md b/README.md index 7df1d24..b1296b9 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,8 @@ pip install saagieapi== | >= 2023.05 | >= 2.10.0 | | >= 2024.01 | >= 2.11.0 | | >= 2024.02 | >= 2.12.0 | +| >= 2024.03 | >= 2.13.0 | +| >= 2024.05 | >= 2.14.0 | ## Contributing diff --git a/pyproject.toml b/pyproject.toml index bcea57e..2a82945 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "saagieapi" -version = "2.13.0" +version = "2.14.0" description = "Python API to interact with Saagie" authors = ["Saagie"] license = "GLWTPL" diff --git a/saagieapi/apps/apps.py b/saagieapi/apps/apps.py index 882cd6a..b11dc38 100644 --- a/saagieapi/apps/apps.py +++ b/saagieapi/apps/apps.py @@ -1467,17 +1467,19 @@ def count_history_statuses(self, history_id, version_number, start_time): def get_logs( self, + project_id: str, app_id: str, app_execution_id: str, limit: int = None, skip: int = None, log_stream: str = None, - start_at: str = None, ): """Get logs of the app Parameters ---------- + project_id : str + UUID of your project app_id : str UUID of your app app_execution_id : str @@ -1490,9 +1492,6 @@ def get_logs( Stream of logs to follow. Values accepted : [ENVVARS_STDOUT, ENVVARS_STDERR, ORCHESTRATION_STDOUT, ORCHESTRATION_STDERR, STDERR, STDOUT] By default, all the streams are retrieved - start_at: str, optional - Get logs since a specific datetime. - Following formats accepted : "2024-04-09 10:00:00" and "2024-04-09T10:00:00" Returns ------- @@ -1508,43 +1507,35 @@ def get_logs( ... skip=5, ... start_at="2024-04-09 10:00:00" ... ) + { - "appLogs": { - "count": 25, - "content": [ - { - "index": 5, - "value": "[I 2024-04-09 13:38:36.982 ServerApp] jupyterlab_git | extension was successfully linked.", - "containerId": "d7104fa7371c5ed6ef540fa8b0620a654a0e02c57136e29f0fcc03d16e36d74f", - "stream": "STDERR", - "recordAt": "2024-04-09T13:38:36.982473892Z" - }, - { - "index": 6, - "value": "[W 2024-04-09 13:38:36.987 NotebookApp] 'ip' has moved from NotebookApp to ServerApp. This config will be passed to ServerApp. Be sure to update your config before our next release.", - "containerId": "d7104fa7371c5ed6ef540fa8b0620a654a0e02c57136e29f0fcc03d16e36d74f", - "stream": "STDERR", - "recordAt": "2024-04-09T13:38:36.987400105Z" - } - ] + "logs": [ + { + "index": 0, + "stream": "STDERR", + "time": "2024-12-11T14:27:42.858298425Z", + "value": "[I 2024-04-09 13:38:36.982 ServerApp] jupyterlab_git | extension was successfully linked.", + }, + { + "index": 1, + "stream": "STDERR", + "time": "2024-12-11T14:27:42.859697094Z", + "value": "AH00558: httpd: Could not reliably determine the server's fully qualified domain name, using 10.4.3.20. Set the 'ServerName' directive globally to suppress this message" } + ], + "limit": 10000, + "total": 5, + "order": "asc", + "source": "elastic" } """ - params = { - "appId": app_id, - "appExecutionId": app_execution_id, - } - - if limit: - params["limit"] = limit - - if skip: - params["skip"] = skip - - if log_stream: - params["stream"] = log_stream - - if start_at: - params["recordAt"] = start_at - - return self.saagie_api.client.execute(query=gql(GQL_GET_APP_LOG), variable_values=params) + if limit is None: + limit = 10000 + if skip is None: + skip = 0 + if log_stream is None: + log_stream = "ENVVARS_STDOUT,ENVVARS_STDERR,ORCHESTRATION_STDOUT,ORCHESTRATION_STDERR,STDERR,STDOUT" + url = f"{self.saagie_api.url_saagie}log-proxy/api/logs/{self.saagie_api.realm}/platform/{self.saagie_api.platform}/project/{project_id}/app_execution/{app_execution_id}?limit={limit}&skip={skip}&streams={log_stream}" + response = self.saagie_api.request_client.send(method="GET", url=url, raise_for_status=True) + + return response.json() diff --git a/saagieapi/apps/gql_queries.py b/saagieapi/apps/gql_queries.py index f8a6770..3435f9c 100644 --- a/saagieapi/apps/gql_queries.py +++ b/saagieapi/apps/gql_queries.py @@ -452,19 +452,3 @@ countAppHistoryStatuses(appHistoryId: $appHistoryId, versionNumber: $versionNumber, startTime: $startTime) } """ - -GQL_GET_APP_LOG = """ -query appLogs($appId: UUID!, $appExecutionId: UUID!, $limit: Int, $skip: Int, $stream: LogStream, $recordAt: String) { - appLogs(appId: $appId, appExecutionId: $appExecutionId, limit: $limit, skip: $skip, stream: $stream, recordAt: $recordAt) - { - count - content { - index - value - containerId - stream - recordAt - } - } -} -""" diff --git a/saagieapi/gql_queries.py b/saagieapi/gql_queries.py index 433e6e0..1cb087b 100644 --- a/saagieapi/gql_queries.py +++ b/saagieapi/gql_queries.py @@ -132,59 +132,3 @@ ) } """ - -GQL_COUNT_CONDITION_LOGS = """ -query conditionPipelineCountFilteredLogs($conditionInstanceId: UUID!, - $projectID: UUID!, - $streams: [LogStream]!) { - conditionPipelineCountFilteredLogs ( - conditionInstanceID: $conditionInstanceId, - projectID: $projectID, - streams: $streams - ) -} -""" - -GQL_GET_CONDITION_LOGS_BY_CONDITION = """ -query conditionPipelineByNodeIdFilteredLogs($pipelineInstanceID: UUID!, - $conditionNodeID: UUID!, - $projectID: UUID!, - $streams: [LogStream]!) { - conditionPipelineByNodeIdFilteredLogs( - pipelineInstanceID: $pipelineInstanceID, - conditionNodeID: $conditionNodeID, - projectID: $projectID, - streams: $streams - ) { - count - content { - index - value - stream - } - } -} -""" - -GQL_GET_CONDITION_LOGS_BY_INSTANCE = """ -query conditionPipelineFilteredLogs($conditionInstanceId: UUID!, - $projectId: UUID!, - $limit: Int, - $skip: Int, - $streams: [LogStream]!) { - conditionPipelineFilteredLogs( - conditionInstanceID: $conditionInstanceId, - projectID:$projectId, - limit: $limit, - skip: $skip, - streams: $streams - ) { - count - content { - index - value - stream - } - } -} -""" diff --git a/saagieapi/saagie_api.py b/saagieapi/saagie_api.py index 4139f09..ca0770e 100644 --- a/saagieapi/saagie_api.py +++ b/saagieapi/saagie_api.py @@ -11,10 +11,7 @@ from .env_vars import EnvVars from .gql_queries import ( GQL_CHECK_CUSTOM_EXPRESSION, - GQL_COUNT_CONDITION_LOGS, GQL_GET_CLUSTER_INFO, - GQL_GET_CONDITION_LOGS_BY_CONDITION, - GQL_GET_CONDITION_LOGS_BY_INSTANCE, GQL_GET_PLATFORM_INFO, GQL_GET_REPOSITORIES_INFO, GQL_GET_RUNTIMES, @@ -72,6 +69,7 @@ def __init__( self.url_saagie = url_saagie self.realm = realm + self.platform = id_platform self.auth = BearerAuth( realm=self.realm, url=self.url_saagie, platform=id_platform, login=user, password=password ) @@ -702,7 +700,12 @@ def check_condition_expression(self, expression: str, project_id: str, variables } return self.client.execute(query=gql(GQL_CHECK_CUSTOM_EXPRESSION), variable_values=params) - def count_condition_logs(self, condition_instance_id: str, project_id: str, streams: List[str]) -> Dict: + def count_condition_logs( + self, + condition_instance_id: str, + project_id: str, + streams: List[str], + ) -> Dict: """Get number of logs line for an instance of a condition on Environment Variable Parameters @@ -712,7 +715,7 @@ def count_condition_logs(self, condition_instance_id: str, project_id: str, stre project_id : str UUID of the project streams : List[str] - List of logs files name to see (example : STDERR, STDOUT) + List of logs files name to see (example : ENVVARS_STDOUT, ENVVARS_STDERR, ORCHESTRATION_STDOUT, ORCHESTRATION_STDERR, STDOUT, STDERR) Returns ------- @@ -726,99 +729,21 @@ def count_condition_logs(self, condition_instance_id: str, project_id: str, stre ... project_id="your_project_id", ... streams=["STDOUT"] ... ) - { - "data": { - "conditionPipelineCountFilteredLogs": 4 - } - } - """ - - params = {"conditionInstanceId": condition_instance_id, "projectID": project_id, "streams": streams} - - return self.client.execute(query=gql(GQL_COUNT_CONDITION_LOGS), variable_values=params) - def get_condition_instance_logs_by_condition( - self, - condition_id: str, - project_id: str, - pipeline_instance_id: str, - streams: List[str], - limit: int = None, - skip: int = None, - ) -> Dict: - """Get logs for a condition on Environment Variable of a pipeline instance - - Parameters - ---------- - condition_id : str - UUID of the condition - project_id : str - UUID of the project - pipeline_instance_id ! str - UUID of the pipeline instance - streams : List[str] - List of logs files name to see (example : STDERR, STDOUT) - limit : int - Number of logs lines to return from the beginning - skip : int - Number of logs lines to doesn't display from the beginning - - Returns - ------- - dict - Dict of logs lines - - Examples - -------- - >>> saagieapi.get_condition_instance_logs_by_condition( - ... condition_id="condition_node_id", - ... project_id="project_id", - ... pipeline_instance_id="pipeline_instance_id", - ... streams=["STDOUT"] - ... ) { - "data": { - "conditionPipelineByNodeIdFilteredLogs": { - "count": 4, - "content": [ - { - "index": 0, - "value": "2023/05/15 12:55:19 INFO [evaluate_condition] Condition: 'tube_name.contains(\"Tube\") ||", - "stream": "STDOUT" - }, - { - "index": 1, - "value": "double(diameter) > 1.0'", - "stream": "STDOUT" - }, - { - "index": 2, - "value": "2023/05/15 12:55:19 INFO [evaluate_condition] Condition evaluation took: 4.736725ms", - "stream": "STDOUT" - }, - { - "index": 3, - "value": "2023/05/15 12:55:19 INFO [evaluate_condition] Result: true", - "stream": "STDOUT" - } - ] - } - } + "logs": [ + ], + "limit": 1, + "total": 5, + "order": "asc", + "source": "elastic" } - """ # pylint: disable=line-too-long - params = { - "conditionNodeID": condition_id, - "projectID": project_id, - "pipelineInstanceID": pipeline_instance_id, - "streams": streams, - } - if limit: - params["limit"] = limit - - if skip: - params["skip"] = skip + """ + log_stream = ",".join(streams) + url = f"{self.url_saagie}log-proxy/api/logs/{self.realm}/platform/{self.platform}/project/{project_id}/condition_instance/{condition_instance_id}?limit=1&skip=0&streams={log_stream}" + response = self.request_client.send(method="GET", url=url, raise_for_status=True) - return self.client.execute(query=gql(GQL_GET_CONDITION_LOGS_BY_CONDITION), variable_values=params) + return response.json() def get_condition_instance_logs_by_instance( self, @@ -837,7 +762,7 @@ def get_condition_instance_logs_by_instance( project_id : str UUID of the project streams : List[str] - List of logs files name to see (example : STDERR, STDOUT) + List of logs files name to see (example : ENVVARS_STDOUT, ENVVARS_STDERR, ORCHESTRATION_STDOUT, ORCHESTRATION_STDERR, STDOUT, STDERR) limit : int Number of logs lines to return from the beginning skip : int @@ -855,44 +780,27 @@ def get_condition_instance_logs_by_instance( ... project_id="project_id" ... ) { - "data": { - "conditionPipelineByNodeIdFilteredLogs": { - "count": 4, - "content": [ - { - "index": 0, - "value": "2023/05/15 12:55:19 INFO [evaluate_condition] Condition: 'tube_name.contains(\"Tube\") ||", - "stream": "STDOUT" - }, - { - "index": 1, - "value": "double(diameter) > 1.0'", - "stream": "STDOUT" - }, - { - "index": 2, - "value": "2023/05/15 12:55:19 INFO [evaluate_condition] Condition evaluation took: 4.736725ms", - "stream": "STDOUT" - }, - { - "index": 3, - "value": "2023/05/15 12:55:19 INFO [evaluate_condition] Result: true", - "stream": "STDOUT" - } - ] + "logs": [ + { + "index": 0, + "value": "2023/05/15 12:55:19 INFO [evaluate_condition] Condition: 'tube_name.contains(\"Tube\") ||", + "stream": "STDOUT" } - } + ], + "limit": 1, + "total": 1, + "order": "asc", + "source": "elastic" } """ # pylint: disable=line-too-long - params = { - "conditionInstanceId": condition_instance_id, - "projectId": project_id, - "streams": streams, - } - if limit: - params["limit"] = limit - if skip: - params["skip"] = skip + if limit is None: + limit = 10000 + if skip is None: + skip = 0 + log_stream = ",".join(streams) + + url = f"{self.url_saagie}log-proxy/api/logs/{self.realm}/platform/{self.platform}/project/{project_id}/condition_instance/{condition_instance_id}?limit={limit}&skip={skip}&streams={log_stream}" + response = self.request_client.send(method="GET", url=url, raise_for_status=True) - return self.client.execute(query=gql(GQL_GET_CONDITION_LOGS_BY_INSTANCE), variable_values=params) + return response.json() diff --git a/tests/integration/apps_integration_test.py b/tests/integration/apps_integration_test.py index 2c9f935..79a81b3 100644 --- a/tests/integration/apps_integration_test.py +++ b/tests/integration/apps_integration_test.py @@ -351,14 +351,15 @@ def test_count_history_statuses(create_global_project, create_then_delete_app_fr @staticmethod def test_get_app_logs(create_global_project, create_then_delete_app_from_scratch): conf = create_global_project + project_id = conf.project_id + app_id = create_then_delete_app_from_scratch app_info = conf.saagie_api.apps.run(app_id=app_id) app_info = conf.saagie_api.apps.get_info(app_id=app_id) - app_execution_id = app_info["app"]["history"]["currentExecutionId"] - result = conf.saagie_api.apps.get_logs(app_id=app_id, app_execution_id=app_execution_id) + result = conf.saagie_api.apps.get_logs(project_id=project_id, app_id=app_id, app_execution_id=app_execution_id) - assert "appLogs" in result + assert "logs" in result diff --git a/tests/integration/saagie_api_integration_test.py b/tests/integration/saagie_api_integration_test.py index 846834c..4bafdc8 100644 --- a/tests/integration/saagie_api_integration_test.py +++ b/tests/integration/saagie_api_integration_test.py @@ -18,46 +18,20 @@ def test_count_condition_logs(create_global_project, create_graph_pipeline): if pipeline_info["graphPipeline"]["instances"]: for cond_inst in pipeline_info["graphPipeline"]["instances"][0]["conditionsInstance"]: if cond_inst["id"]: - cond_inst_id = cond_inst["id"] + condition_node_id = cond_inst["conditionNodeId"] + condition_instance_id = cond_inst["id"] break - nb_logs = conf.saagie_api.count_condition_logs( - condition_instance_id=cond_inst_id, project_id=conf.project_id, streams=["STDOUT", "STDERR"] - ) - - conf.saagie_api.pipelines.delete(pipeline_id) - conf.saagie_api.jobs.delete(job_id) - - # nb logs lines can be different for 2 instances, test only that result is int - assert isinstance(nb_logs["conditionPipelineCountFilteredLogs"], int) - - @staticmethod - def test_get_condition_instance_logs_by_condition(create_global_project, create_graph_pipeline): - conf = create_global_project - - pipeline_id, job_id = create_graph_pipeline - conf.saagie_api.pipelines.run_with_callback(pipeline_id=pipeline_id) - - pipeline_info = conf.saagie_api.pipelines.get_info(pipeline_id=pipeline_id) - - if pipeline_info["graphPipeline"]["instances"]: - pipeline_instance_id = pipeline_info["graphPipeline"]["instances"][0]["id"] - for cond_inst in pipeline_info["graphPipeline"]["instances"][0]["conditionsInstance"]: - if cond_inst["id"]: - cond_id = cond_inst["conditionNodeId"] - break - - logs = conf.saagie_api.get_condition_instance_logs_by_condition( - condition_id=cond_id, + logs = conf.saagie_api.count_condition_logs( + condition_instance_id=condition_instance_id, project_id=conf.project_id, - pipeline_instance_id=pipeline_instance_id, streams=["STDOUT", "STDERR"], ) conf.saagie_api.pipelines.delete(pipeline_id) conf.saagie_api.jobs.delete(job_id) - assert "content" in logs["conditionPipelineByNodeIdFilteredLogs"] + assert 5 == logs["total"] @staticmethod def test_get_condition_instance_logs_by_instance(create_global_project, create_graph_pipeline): @@ -71,11 +45,12 @@ def test_get_condition_instance_logs_by_instance(create_global_project, create_g if pipeline_info["graphPipeline"]["instances"]: for cond_inst in pipeline_info["graphPipeline"]["instances"][0]["conditionsInstance"]: if cond_inst["id"]: - cond_inst_id = cond_inst["id"] + condition_node_id = cond_inst["conditionNodeId"] + condition_instance_id = cond_inst["id"] break logs = conf.saagie_api.get_condition_instance_logs_by_instance( - condition_instance_id=cond_inst_id, + condition_instance_id=condition_instance_id, project_id=conf.project_id, streams=["STDOUT", "STDERR"], ) @@ -83,4 +58,4 @@ def test_get_condition_instance_logs_by_instance(create_global_project, create_g conf.saagie_api.pipelines.delete(pipeline_id) conf.saagie_api.jobs.delete(job_id) - assert "content" in logs["conditionPipelineFilteredLogs"] + assert 5 == logs["total"] diff --git a/tests/unit/apps_unit_test.py b/tests/unit/apps_unit_test.py index 238ff3f..2d31061 100644 --- a/tests/unit/apps_unit_test.py +++ b/tests/unit/apps_unit_test.py @@ -785,39 +785,3 @@ def test_count_history_app_statuses(self, saagie_api_mock): app.count_history_statuses(history_id=history_id, version_number=1, start_time="2024-04-10T14:26:27.073Z") saagie_api_mock.client.execute.assert_called_with(query=expected_query, variable_values=params) - - def test_get_app_logs_gql(self): - query = gql(GQL_GET_APP_LOG) - self.client.validate(query) - - def test_get_app_logs(self, saagie_api_mock): - app = Apps(saagie_api_mock) - - app_id = "70e85ade-d6cc-4a90-8d7d-639adbd25e5d" - app_execution_id = "e3e31074-4a12-450e-96e4-0eae7801dfca" - limit = 2 - skip = 5 - stream = "STDERR" - start_at = "2024-04-09 10:00:00" - - params = { - "appId": app_id, - "appExecutionId": app_execution_id, - "limit": limit, - "skip": skip, - "stream": stream, - "recordAt": start_at, - } - - expected_query = gql(GQL_GET_APP_LOG) - - app.get_logs( - app_id=app_id, - app_execution_id=app_execution_id, - limit=limit, - skip=skip, - log_stream=stream, - start_at=start_at, - ) - - saagie_api_mock.client.execute.assert_called_with(query=expected_query, variable_values=params) diff --git a/tests/unit/saagie_api_unit_test.py b/tests/unit/saagie_api_unit_test.py index 241d877..884bc29 100644 --- a/tests/unit/saagie_api_unit_test.py +++ b/tests/unit/saagie_api_unit_test.py @@ -9,10 +9,7 @@ from saagieapi import SaagieApi from saagieapi.gql_queries import ( GQL_CHECK_CUSTOM_EXPRESSION, - GQL_COUNT_CONDITION_LOGS, GQL_GET_CLUSTER_INFO, - GQL_GET_CONDITION_LOGS_BY_CONDITION, - GQL_GET_CONDITION_LOGS_BY_INSTANCE, GQL_GET_PLATFORM_INFO, GQL_GET_REPOSITORIES_INFO, ) @@ -142,15 +139,3 @@ def test_check_technology_configured_technology_not_configured(): def test_check_custom_expression(self): query = gql(GQL_CHECK_CUSTOM_EXPRESSION) self.client.validate(query) - - def test_count_condition_logs(self): - query = gql(GQL_COUNT_CONDITION_LOGS) - self.client.validate(query) - - def test_get_condition_logs_by_condition(self): - query = gql(GQL_GET_CONDITION_LOGS_BY_CONDITION) - self.client.validate(query) - - def test_get_condition_logs_by_instance(self): - query = gql(GQL_GET_CONDITION_LOGS_BY_INSTANCE) - self.client.validate(query)