From eb5e3cdeb0831725a3bd4fc76a1c939201a57ed7 Mon Sep 17 00:00:00 2001 From: Thomas Quiviger Date: Tue, 15 Mar 2022 10:51:03 +0100 Subject: [PATCH 1/7] Adding depreciation warnings --- poetry.lock | 21 +++++++++++++--- pyproject.toml | 1 + saagieapi/projects/gql_template.py | 4 ++- saagieapi/projects/saagie_api.py | 39 +++++++++++++++++++----------- tests/test_projects_integration.py | 2 +- 5 files changed, 48 insertions(+), 19 deletions(-) diff --git a/poetry.lock b/poetry.lock index fc927678..8bfe51fc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -132,6 +132,17 @@ category = "dev" optional = false python-versions = ">=3.6, <3.7" +[[package]] +name = "deprecation" +version = "2.1.0" +description = "A library to handle automated deprecations" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +packaging = "*" + [[package]] name = "docutils" version = "0.18.1" @@ -302,7 +313,7 @@ python-versions = ">=3.6" name = "packaging" version = "21.3" description = "Core utilities for Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -363,7 +374,7 @@ python-versions = ">=3.5" name = "pyparsing" version = "3.0.7" description = "Python parsing module" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -688,7 +699,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "0f1df4a51e672672251f80f245a674dcb8a3d43a8def8d74f7bc9464a50e750d" +content-hash = "f98e3c2cbc1f1f9b977f5513dcaa717abb9835252e9ee3406455060eabda1e4b" [metadata.files] atomicwrites = [ @@ -805,6 +816,10 @@ dataclasses = [ {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"}, {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"}, ] +deprecation = [ + {file = "deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a"}, + {file = "deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff"}, +] docutils = [ {file = "docutils-0.18.1-py2.py3-none-any.whl", hash = "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c"}, {file = "docutils-0.18.1.tar.gz", hash = "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06"}, diff --git a/pyproject.toml b/pyproject.toml index ae8200ec..45aea847 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ gql = "^3.0.0" pytz = "^2021.1" croniter = "^1.0.1" requests_toolbelt = "^0.9.1" +deprecation = "^2.1.0" [tool.poetry.dev-dependencies] python-semantic-release = "^7.19.2" diff --git a/saagieapi/projects/gql_template.py b/saagieapi/projects/gql_template.py index cd85f42f..19ebd2b1 100644 --- a/saagieapi/projects/gql_template.py +++ b/saagieapi/projects/gql_template.py @@ -153,11 +153,13 @@ gql_get_project_technologies = """ {{ - technologiesByCategory(projectId: "{0}"){{ + project(id: "{0}"){{ + technologiesByCategory {{ jobCategory, technologies{{ id }} + }} }} }} """ diff --git a/saagieapi/projects/saagie_api.py b/saagieapi/projects/saagie_api.py index 1c0dabdb..3c62c89b 100644 --- a/saagieapi/projects/saagie_api.py +++ b/saagieapi/projects/saagie_api.py @@ -17,6 +17,7 @@ from .auth import * from .gql_template import * from .graph_pipeline import * +import deprecation class SaagieApi: @@ -34,9 +35,9 @@ def __init__(self, url_saagie, id_platform, user, password, realm, retries=0): id_platform : int or str Platform Id (see README on how to find it) user : str - username to login with + username to log in with password : str - password to login with + password to log in with realm : str Saagie realm (see README on how to find it) retries : int @@ -98,12 +99,12 @@ def easy_connect(cls, url_saagie_platform, user, password): url_saagie_platform : str Complete platform URL (eg: https://saagie-workspace.prod.saagie.io/projects/platform/6/) user : str - username to login with + username to log in with password : str - password to login with + password to log in with """ url_regex = re.compile( - r"(https:\/\/(\w+)-(?:\w|\.)+)\/projects\/platform\/(\d+)") + r"(https://(\w+)-(?:\w|\.)+)/projects/platform/(\d+)") m = url_regex.match(url_saagie_platform) if bool(m): url_saagie = m.group(1) @@ -385,7 +386,7 @@ def get_project_technologies(self, project_id): Dict of available technologies """ query = gql(gql_get_project_technologies.format(project_id)) - return self.client.execute(query) + return self.client.execute(query)['project'] def create_project(self, name, group=None, role="Manager", description=""): """Create a new project on the platform @@ -679,7 +680,7 @@ def edit_job(self, job_id, job_name=None, description=None, is_scheduled=None, else: raise RuntimeError("Please specify a correct timezone") - elif is_scheduled == False: + elif not is_scheduled: params["isScheduled"] = False else: @@ -881,7 +882,7 @@ def get_job_info(self, job_id): """ query = gql_get_info_job.format(job_id) return self.client.execute(gql(query)) - + def upgrade_job(self, job_id, file=None, use_previous_artifact=False, runtime_version='3.6', command_line='python {file} arg1 arg2', release_note=None, extra_technology='', extra_technology_version=''): @@ -920,10 +921,14 @@ def upgrade_job(self, job_id, file=None, use_previous_artifact=False, runtime_ve technology_id = self.get_job_info(job_id)["job"]["technology"]["id"] available_runtimes = [c["label"] for c in self.get_runtimes(technology_id)["technology"]["contexts"]] if runtime_version not in available_runtimes: - raise RuntimeError(f"Specified runtime does not exist ({runtime_version}). Available runtimes : {','.join(available_runtimes)}.") + raise RuntimeError( + f"Specified runtime does not exist ({runtime_version}). " + f"Available runtimes : {','.join(available_runtimes)}.") if file and use_previous_artifact: - logging.warning("You can not specify a file and use the previous artifact. By default, the specified file will be used.") + logging.warning( + "You can not specify a file and use the previous artifact. " + "By default, the specified file will be used.") if extra_technology != '': extra_tech = gql_extra_technology.format(extra_technology, @@ -996,7 +1001,7 @@ def get_job_id(self, job_name, project_name): return job[0]["id"] else: raise NameError(f"Job {job_name} does not exist.") - + def upgrade_job_by_name(self, job_name, project_name, file=None, use_previous_artifact=False, runtime_version='3.6', command_line='python {file} arg1 arg2', release_note=None, extra_technology='', extra_technology_version='' @@ -1078,7 +1083,9 @@ def get_project_web_apps(self, project_id, instances_limit=-1): dict Dict of webApp information """ - regex_error_missing_technology = r"io\.saagie\.projectsandjobs\.domain\.exception\.NonExistingTechnologyException: Technology \S{8}-\S{4}-\S{4}-\S{4}-\S{12} does not exist" + regex_error_missing_technology = r"io\.saagie\.projectsandjobs\.domain\.exception" \ + r"\.NonExistingTechnologyException: Technology \S{8}-\S{4}-\S{4}-\S{4}-\S{" \ + r"12} does not exist " instances_limit_request = f" (limit: {str(instances_limit)})" if instances_limit != -1 else "" query = gql(gql_get_project_web_apps.format(project_id, @@ -1251,7 +1258,7 @@ def edit_pipeline(self, pipeline_id, name=None, description=None, emails=None, else: raise RuntimeError("Please specify a correct timezone") - elif is_scheduled == False: + elif not is_scheduled: params["isScheduled"] = False else: @@ -1351,6 +1358,9 @@ def run_pipeline_callback(self, pipeline_id, freq=10, timeout=-1): print('Current state : ' + state) return state + @deprecation.deprecated(deprecated_in="Saagie 2.2.1", + details="This deprecated endpoint allows to create only linear pipeline. " + "To create graph pipelines, use the endpoint `createGraphPipeline` instead.") def create_pipeline(self, name, project_id, jobs_id, description=""): """ Create a pipeline in a given project @@ -1499,7 +1509,8 @@ def upgrade_pipeline(self, pipeline_id, graph_pipeline, release_note=""): if not graph_pipeline.list_job_nodes: graph_pipeline.to_pipeline_graph_input() - params = {'id': pipeline_id, 'jobNodes': graph_pipeline.list_job_nodes, 'conditionNodes': graph_pipeline.list_conditions_nodes, 'releaseNote': release_note} + params = {'id': pipeline_id, 'jobNodes': graph_pipeline.list_job_nodes, + 'conditionNodes': graph_pipeline.list_conditions_nodes, 'releaseNote': release_note} return self.client.execute(gql(gql_upgrade_pipeline), variable_values=params) diff --git a/tests/test_projects_integration.py b/tests/test_projects_integration.py index f4ea6958..45627bd7 100644 --- a/tests/test_projects_integration.py +++ b/tests/test_projects_integration.py @@ -355,7 +355,7 @@ def test_create_graph_pipeline(self, create_then_delete_graph_pipeline): def test_get_graph_pipeline_id(self, create_then_delete_graph_pipeline): pipeline_id, _ = create_then_delete_graph_pipeline pipeline_name = 'TEST_VIA_API' - output_pipeline_id = self.saagie.get_pipeline_id(pipeline_name,self.project_name) + output_pipeline_id = self.saagie.get_pipeline_id(pipeline_name, self.project_name) assert pipeline_id == output_pipeline_id def test_delete_graph_pipeline(self, create_graph_pipeline): From 2b20236bf39e69d99c447b214c15deb0a328ee91 Mon Sep 17 00:00:00 2001 From: Thomas Quiviger Date: Wed, 16 Mar 2022 10:52:22 +0100 Subject: [PATCH 2/7] Code review --- saagieapi/projects/saagie_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/saagieapi/projects/saagie_api.py b/saagieapi/projects/saagie_api.py index 3c62c89b..87652f13 100644 --- a/saagieapi/projects/saagie_api.py +++ b/saagieapi/projects/saagie_api.py @@ -680,7 +680,7 @@ def edit_job(self, job_id, job_name=None, description=None, is_scheduled=None, else: raise RuntimeError("Please specify a correct timezone") - elif not is_scheduled: + elif is_scheduled == False: params["isScheduled"] = False else: @@ -1258,7 +1258,7 @@ def edit_pipeline(self, pipeline_id, name=None, description=None, emails=None, else: raise RuntimeError("Please specify a correct timezone") - elif not is_scheduled: + elif is_scheduled == False: params["isScheduled"] = False else: From 0f4f40cc6647b43bd48b6ffc2e51bc1647c6b79e Mon Sep 17 00:00:00 2001 From: Thomas Quiviger Date: Tue, 22 Mar 2022 15:51:33 +0100 Subject: [PATCH 3/7] Checking Saagie version --- README.md | 5 +-- saagieapi/compatibility_matrix.json | 12 +++++ saagieapi/projects/auth.py | 30 ------------- saagieapi/projects/saagie_api.py | 7 ++- saagieapi/projects/utils.py | 69 +++++++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 35 deletions(-) create mode 100644 saagieapi/compatibility_matrix.json delete mode 100644 saagieapi/projects/auth.py create mode 100644 saagieapi/projects/utils.py diff --git a/README.md b/README.md index 53fc9d89..f680b324 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,7 @@ pip install saagieapi== ### Compatibility with your Saagie platform -| **Saagie platform version** | **saagie-api release** | -|-----------------------------|------------------------| -| < 2.2.0 | < 0.6.0 | -| >= 2.2.0 | >= 0.6.0 | +Check #TODO ## Usage diff --git a/saagieapi/compatibility_matrix.json b/saagieapi/compatibility_matrix.json new file mode 100644 index 00000000..ab368398 --- /dev/null +++ b/saagieapi/compatibility_matrix.json @@ -0,0 +1,12 @@ +{ + "1.19.2203181529-2949.39": { + "min_api_saagie_version": "0.10.1" + }, + "2.1.0": { + "min_api_saagie_version": "0.5.1", + "max_api_saagie_version": "0.5.8" + }, + "2.2.0": { + "min_api_saagie_version": "0.6.0" + } +} diff --git a/saagieapi/projects/auth.py b/saagieapi/projects/auth.py deleted file mode 100644 index afd1f17c..00000000 --- a/saagieapi/projects/auth.py +++ /dev/null @@ -1,30 +0,0 @@ -import requests - - -class BearerAuth(requests.auth.AuthBase): - def __init__(self, realm, url, platform, login, password): - self.token = self._authenticate(realm, url, login, password) - self.platform = platform - self.url = url - - def __call__(self, r): - r.headers["authorization"] = "Bearer " + self.token - return r - - @staticmethod - def _authenticate(realm, url, login, password): - """ - Retrieve a Bearer connection token - :param realm: platform url prefix (eg: saagie) - :param url: platform URL (eg: https://saagie-workspace.prod.saagie.io) - :param login: username to login with - :param password: password to login with - :return: a token - """ - s = requests.session() - s.headers["Content-Type"] = "application/json" - s.headers["Saagie-Realm"] = realm - r = s.post(url + '/authentication/api/open/authenticate', - json={'login': login, 'password': password}, - verify=False) - return r.text diff --git a/saagieapi/projects/saagie_api.py b/saagieapi/projects/saagie_api.py index 87652f13..87845f24 100644 --- a/saagieapi/projects/saagie_api.py +++ b/saagieapi/projects/saagie_api.py @@ -14,11 +14,15 @@ from gql import Client from gql.transport.requests import RequestsHTTPTransport -from .auth import * +from .utils import * from .gql_template import * from .graph_pipeline import * import deprecation +logging.basicConfig(level=logging.WARN, + format="%(asctime)s [%(levelname)s] %(message)s", + datefmt="%d/%m/%Y %H:%M:%S") + class SaagieApi: """Define several methods to interact with Saagie API in Python (API for @@ -54,6 +58,7 @@ def __init__(self, url_saagie, id_platform, user, password, realm, retries=0): self.retries = retries self.auth = BearerAuth(self.realm, self.url_saagie, self.id_platform, self.login, self.password) + get_saagie_version(self.auth, url_saagie) url = self.url_saagie + self.suffix_api + 'platform/' url += str(self.id_platform) + '/graphql' self._url = url diff --git a/saagieapi/projects/utils.py b/saagieapi/projects/utils.py new file mode 100644 index 00000000..2dfd4003 --- /dev/null +++ b/saagieapi/projects/utils.py @@ -0,0 +1,69 @@ +import importlib.metadata +import json +import logging +from json import JSONDecodeError +from packaging import version + +import requests + + +class BearerAuth(requests.auth.AuthBase): + def __init__(self, realm, url, platform, login, password): + self.token = self._authenticate(realm, url, login, password) + self.platform = platform + self.url = url + + def __call__(self, r): + r.headers["authorization"] = "Bearer " + self.token + return r + + @staticmethod + def _authenticate(realm, url, login, password): + """ + Retrieve a Bearer connection token + :param realm: platform url prefix (eg: saagie) + :param url: platform URL (eg: https://saagie-workspace.prod.saagie.io) + :param login: username to login with + :param password: password to login with + :return: a token + """ + s = requests.session() + s.headers["Content-Type"] = "application/json" + s.headers["Saagie-Realm"] = realm + r = s.post(url + '/authentication/api/open/authenticate', + json={'login': login, 'password': password}, + verify=False) + return r.text + + +def get_saagie_version(auth: BearerAuth, url_saagie: str): + s = auth(requests.session()) + saagie_compatibility_matrix = json.load(open("saagieapi/compatibility_matrix.json")) + saagie_api_current_version = importlib.metadata.version("saagieapi") + logging.debug(f"Compatibility matrix : {saagie_compatibility_matrix}") + logging.debug(f"Current api-saagie version : {saagie_api_current_version}") + + try: + saagie_versions = json.loads(s.get(f"{url_saagie}version.json").text) + saagie_current_version = saagie_versions['major'] + compatible_versions = saagie_compatibility_matrix[saagie_current_version] + minimal_version = compatible_versions["min_api_saagie_version"] + maximal_version = compatible_versions["max_api_saagie_version"] + if version.parse(saagie_api_current_version) < version.parse(minimal_version): + print("babla") + logging.warning( + f"You are using a saagiepi version ({saagie_api_current_version}) " + f"not compatible with your Saagie platform (Saagie v.{saagie_current_version})") + logging.warning( + f"Your Saagie platform requires at least the version {minimal_version} " + f"of saagieapi. Please consider upgrading.") + if version.parse(saagie_api_current_version) > version.parse(maximal_version): + logging.warning( + f"You are using a saagiepi version ({saagie_api_current_version}) " + f"not compatible with your Saagie platform (Saagie v.{saagie_current_version})") + logging.warning( + f"Your Saagie platform is not compatible with versions > {maximal_version} " + f"of saagieapi. Please consider downgrading.") + + except JSONDecodeError: + logging.warning("Could not get Saagie version") From b1492c9c4b369bb69ca5eb72c3ebaaa133893386 Mon Sep 17 00:00:00 2001 From: Thomas Quiviger Date: Thu, 7 Apr 2022 17:35:25 +0200 Subject: [PATCH 4/7] Checking Saagie version --- README.md | 4 +- .../{ => projects}/compatibility_matrix.json | 3 - saagieapi/projects/saagie_api.py | 7 +- saagieapi/projects/utils.py | 91 +++++++++++++------ 4 files changed, 68 insertions(+), 37 deletions(-) rename saagieapi/{ => projects}/compatibility_matrix.json (67%) diff --git a/README.md b/README.md index f680b324..aac767b9 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,9 @@ pip install saagieapi== ### Compatibility with your Saagie platform -Check #TODO +When instantiating a SaagieApi instance, a check is made to ensure that your Saagie platform version is compatible with +the version of SaagieApi you are using. +This check is based on the `saagieapi/projects/compatibility_matrix.json` file. ## Usage diff --git a/saagieapi/compatibility_matrix.json b/saagieapi/projects/compatibility_matrix.json similarity index 67% rename from saagieapi/compatibility_matrix.json rename to saagieapi/projects/compatibility_matrix.json index ab368398..041e1e81 100644 --- a/saagieapi/compatibility_matrix.json +++ b/saagieapi/projects/compatibility_matrix.json @@ -1,7 +1,4 @@ { - "1.19.2203181529-2949.39": { - "min_api_saagie_version": "0.10.1" - }, "2.1.0": { "min_api_saagie_version": "0.5.1", "max_api_saagie_version": "0.5.8" diff --git a/saagieapi/projects/saagie_api.py b/saagieapi/projects/saagie_api.py index 93bd2b59..f8387d6d 100644 --- a/saagieapi/projects/saagie_api.py +++ b/saagieapi/projects/saagie_api.py @@ -3,11 +3,9 @@ Projects & Jobs - to interact with the manager API, see the manager subpackage) """ -import logging import time import re import pytz -from pathlib import Path from croniter import croniter from gql import gql @@ -19,10 +17,6 @@ from .graph_pipeline import * import deprecation -logging.basicConfig(level=logging.WARN, - format="%(asctime)s [%(levelname)s] %(message)s", - datefmt="%d/%m/%Y %H:%M:%S") - class SaagieApi: """Define several methods to interact with Saagie API in Python (API for @@ -89,6 +83,7 @@ def __init__(self, url_saagie, id_platform, user, password, realm, retries=0): # Valid status list of alerting self.valid_status_list = ["REQUESTED", "QUEUED", "RUNNING", "FAILED", "KILLED", "KILLING", "SUCCEEDED", "UNKNOWN", "AWAITING", "SKIPPED"] + check_saagie_version_compatibility(self.auth, self.url_saagie) @classmethod def easy_connect(cls, url_saagie_platform, user, password): diff --git a/saagieapi/projects/utils.py b/saagieapi/projects/utils.py index 2dfd4003..2923710f 100644 --- a/saagieapi/projects/utils.py +++ b/saagieapi/projects/utils.py @@ -1,10 +1,18 @@ import importlib.metadata import json import logging +from pathlib import Path from json import JSONDecodeError from packaging import version import requests +from packaging.version import Version + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%d/%m/%Y %H:%M:%S") +logging.getLogger("requests").setLevel(logging.WARN) +logging.getLogger("gql").setLevel(logging.WARN) + +dir_path = Path(__file__).resolve().parent class BearerAuth(requests.auth.AuthBase): @@ -36,34 +44,63 @@ def _authenticate(realm, url, login, password): return r.text -def get_saagie_version(auth: BearerAuth, url_saagie: str): - s = auth(requests.session()) - saagie_compatibility_matrix = json.load(open("saagieapi/compatibility_matrix.json")) - saagie_api_current_version = importlib.metadata.version("saagieapi") - logging.debug(f"Compatibility matrix : {saagie_compatibility_matrix}") - logging.debug(f"Current api-saagie version : {saagie_api_current_version}") +def __get_saagie_current_version(auth: BearerAuth, url_saagie: str) -> str: + """ + Retrieve the current Saagie version on the target platform + :param auth: BearerAuth object + :param url_saagie: Saagie URL + :return: Saagie major version + """ + try: + saagie_versions = auth(requests.session()).get(f"{url_saagie}version.json").json() + return saagie_versions['major'] + except (JSONDecodeError, KeyError): + logging.warning("Could not get Saagie version") + return "unknown-version " + +def __get_min_max_saagie_versions(saagie_current_version: str) -> (Version, Version): + """ + Retrieve the minimum and maximum Saagie version supported by the Saagie API + based on the compatibility matrix + :param saagie_current_version: Saagie platform current version + :return: (min_version, max_version) + """ + saagie_compatibility_matrix = json.load(open(dir_path.joinpath('compatibility_matrix.json'))) + logging.debug(f"Compatibility matrix : {saagie_compatibility_matrix}") try: - saagie_versions = json.loads(s.get(f"{url_saagie}version.json").text) - saagie_current_version = saagie_versions['major'] compatible_versions = saagie_compatibility_matrix[saagie_current_version] - minimal_version = compatible_versions["min_api_saagie_version"] - maximal_version = compatible_versions["max_api_saagie_version"] - if version.parse(saagie_api_current_version) < version.parse(minimal_version): - print("babla") - logging.warning( - f"You are using a saagiepi version ({saagie_api_current_version}) " - f"not compatible with your Saagie platform (Saagie v.{saagie_current_version})") - logging.warning( - f"Your Saagie platform requires at least the version {minimal_version} " - f"of saagieapi. Please consider upgrading.") - if version.parse(saagie_api_current_version) > version.parse(maximal_version): - logging.warning( - f"You are using a saagiepi version ({saagie_api_current_version}) " - f"not compatible with your Saagie platform (Saagie v.{saagie_current_version})") - logging.warning( - f"Your Saagie platform is not compatible with versions > {maximal_version} " - f"of saagieapi. Please consider downgrading.") + minimal_version = version.parse(compatible_versions.get("min_api_saagie_version", "0")) + maximal_version = version.parse(compatible_versions.get("max_api_saagie_version", "9.9")) + return minimal_version, maximal_version + except KeyError: + logging.warning(f"Could not find your Saagie version ({saagie_current_version}) in the compatibility matrix") + return version.parse("0"), version.parse("9.9") - except JSONDecodeError: - logging.warning("Could not get Saagie version") + +def check_saagie_version_compatibility(auth: BearerAuth, url_saagie: str): + """ + Check if the Saagie version is compatible with the Saagie API version and display warnings if not + :param auth: BearerAuth object + :param url_saagie: Saagie URL + """ + saagie_current_version = __get_saagie_current_version(auth, url_saagie) + saagie_api_current_version = version.parse(importlib.metadata.version("saagieapi")) + + logging.info(f"Current saagie version : {saagie_current_version}") + logging.info(f"Current api-saagie version : {saagie_api_current_version}") + min_version, max_version = __get_min_max_saagie_versions(saagie_current_version) + if saagie_api_current_version < min_version: + logging.warning( + f"You are using a saagie-api version ({saagie_api_current_version}) " + f"not compatible with your Saagie platform (Saagie v.{saagie_current_version})") + logging.warning( + f"Your Saagie platform requires at least the version {min_version} " + f"of saagieapi. Please consider upgrading.") + if saagie_api_current_version > max_version: + logging.warning( + f"You are using a saagie-api version ({saagie_api_current_version}) " + f"not compatible with your Saagie platform (Saagie v.{saagie_current_version})") + logging.warning( + f"Your Saagie platform is not compatible with versions > {max_version} " + f"of saagieapi. Please consider downgrading.") From 16436b3745752fff5104a80d84d7b406a8b52a26 Mon Sep 17 00:00:00 2001 From: Thomas Quiviger Date: Fri, 8 Apr 2022 15:19:15 +0200 Subject: [PATCH 5/7] Refactoring --- saagieapi/projects/auth.py | 30 +++++++++ saagieapi/projects/saagie_api.py | 80 +++++++++++++++++++++-- saagieapi/projects/utils.py | 106 ------------------------------- 3 files changed, 104 insertions(+), 112 deletions(-) create mode 100644 saagieapi/projects/auth.py delete mode 100644 saagieapi/projects/utils.py diff --git a/saagieapi/projects/auth.py b/saagieapi/projects/auth.py new file mode 100644 index 00000000..ab3de9b1 --- /dev/null +++ b/saagieapi/projects/auth.py @@ -0,0 +1,30 @@ +import requests + + +class BearerAuth(requests.auth.AuthBase): + def __init__(self, realm, url, platform, login, password): + self.token = self._authenticate(realm, url, login, password) + self.platform = platform + self.url = url + + def __call__(self, r): + r.headers["authorization"] = "Bearer " + self.token + return r + + @staticmethod + def _authenticate(realm, url, login, password): + """ + Retrieve a Bearer connection token + :param realm: platform url prefix (eg: saagie) + :param url: platform URL (eg: https://saagie-workspace.prod.saagie.io) + :param login: username to login with + :param password: password to login with + :return: a token + """ + s = requests.session() + s.headers["Content-Type"] = "application/json" + s.headers["Saagie-Realm"] = realm + r = s.post(url + '/authentication/api/open/authenticate', + json={'login': login, 'password': password}, + verify=False) + return r.text \ No newline at end of file diff --git a/saagieapi/projects/saagie_api.py b/saagieapi/projects/saagie_api.py index f8387d6d..49d87d85 100644 --- a/saagieapi/projects/saagie_api.py +++ b/saagieapi/projects/saagie_api.py @@ -3,19 +3,32 @@ Projects & Jobs - to interact with the manager API, see the manager subpackage) """ -import time +import importlib.metadata +import json +import logging import re +import time +from json import JSONDecodeError +from pathlib import Path + +import deprecation import pytz from croniter import croniter - -from gql import gql from gql import Client +from gql import gql from gql.transport.requests import RequestsHTTPTransport +from packaging import version +from packaging.version import Version -from .utils import * +from .auth import * from .gql_template import * from .graph_pipeline import * -import deprecation + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%d/%m/%Y %H:%M:%S") +logging.getLogger("requests").setLevel(logging.WARN) +logging.getLogger("gql").setLevel(logging.WARN) + +dir_path = Path(__file__).resolve().parent class SaagieApi: @@ -83,7 +96,7 @@ def __init__(self, url_saagie, id_platform, user, password, realm, retries=0): # Valid status list of alerting self.valid_status_list = ["REQUESTED", "QUEUED", "RUNNING", "FAILED", "KILLED", "KILLING", "SUCCEEDED", "UNKNOWN", "AWAITING", "SKIPPED"] - check_saagie_version_compatibility(self.auth, self.url_saagie) + self.check_saagie_version_compatibility() @classmethod def easy_connect(cls, url_saagie_platform, user, password): @@ -118,6 +131,61 @@ def easy_connect(cls, url_saagie_platform, user, password): # ### env vars #### # ###################################################### + def __get_saagie_current_version(self): + """ + Retrieve the current Saagie version on the target platform + :return: Saagie major version + """ + try: + saagie_versions = self.auth(requests.session()).get(f"{self.url_saagie}version.json").json() + self.saagie_current_version = saagie_versions['major'] + except (JSONDecodeError, KeyError): + logging.warning("Could not get Saagie version") + self.saagie_current_version = "unknown-version " + logging.info(f"Current saagie version : {self.saagie_current_version}") + + def __get_min_max_saagie_versions(self) -> (Version, Version): + """ + Retrieve the minimum and maximum Saagie version supported by the Saagie API + based on the compatibility matrix + :return: (min_version, max_version) + """ + saagie_compatibility_matrix = json.load(open(dir_path.joinpath('compatibility_matrix.json'))) + logging.debug(f"Compatibility matrix : {saagie_compatibility_matrix}") + try: + compatible_versions = saagie_compatibility_matrix[self.saagie_current_version] + minimal_version = version.parse(compatible_versions.get("min_api_saagie_version", "0")) + maximal_version = version.parse(compatible_versions.get("max_api_saagie_version", "9.9")) + return minimal_version, maximal_version + except KeyError: + logging.warning( + f"Could not find your Saagie version ({self.saagie_current_version}) in the compatibility matrix") + return version.parse("0"), version.parse("9.9") + + def check_saagie_version_compatibility(self): + """ + Check if the Saagie version is compatible with the Saagie API version and display warnings if not + """ + self.__get_saagie_current_version() + saagie_api_current_version = version.parse(importlib.metadata.version("saagieapi")) + + logging.info(f"Current api-saagie version : {saagie_api_current_version}") + min_version, max_version = self.__get_min_max_saagie_versions() + if saagie_api_current_version < min_version: + logging.warning( + f"You are using a saagie-api version ({saagie_api_current_version}) " + f"not compatible with your Saagie platform (Saagie v.{self.saagie_current_version})") + logging.warning( + f"Your Saagie platform requires at least the version {min_version} " + f"of saagieapi. Please consider upgrading.") + if saagie_api_current_version > max_version: + logging.warning( + f"You are using a saagie-api version ({saagie_api_current_version}) " + f"not compatible with your Saagie platform (Saagie v.{self.saagie_current_version})") + logging.warning( + f"Your Saagie platform is not compatible with versions > {max_version} " + f"of saagieapi. Please consider downgrading.") + def get_global_env_vars(self): """Get global environment variables NB: You can only list environment variables if you have at least the diff --git a/saagieapi/projects/utils.py b/saagieapi/projects/utils.py deleted file mode 100644 index 2923710f..00000000 --- a/saagieapi/projects/utils.py +++ /dev/null @@ -1,106 +0,0 @@ -import importlib.metadata -import json -import logging -from pathlib import Path -from json import JSONDecodeError -from packaging import version - -import requests -from packaging.version import Version - -logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%d/%m/%Y %H:%M:%S") -logging.getLogger("requests").setLevel(logging.WARN) -logging.getLogger("gql").setLevel(logging.WARN) - -dir_path = Path(__file__).resolve().parent - - -class BearerAuth(requests.auth.AuthBase): - def __init__(self, realm, url, platform, login, password): - self.token = self._authenticate(realm, url, login, password) - self.platform = platform - self.url = url - - def __call__(self, r): - r.headers["authorization"] = "Bearer " + self.token - return r - - @staticmethod - def _authenticate(realm, url, login, password): - """ - Retrieve a Bearer connection token - :param realm: platform url prefix (eg: saagie) - :param url: platform URL (eg: https://saagie-workspace.prod.saagie.io) - :param login: username to login with - :param password: password to login with - :return: a token - """ - s = requests.session() - s.headers["Content-Type"] = "application/json" - s.headers["Saagie-Realm"] = realm - r = s.post(url + '/authentication/api/open/authenticate', - json={'login': login, 'password': password}, - verify=False) - return r.text - - -def __get_saagie_current_version(auth: BearerAuth, url_saagie: str) -> str: - """ - Retrieve the current Saagie version on the target platform - :param auth: BearerAuth object - :param url_saagie: Saagie URL - :return: Saagie major version - """ - try: - saagie_versions = auth(requests.session()).get(f"{url_saagie}version.json").json() - return saagie_versions['major'] - except (JSONDecodeError, KeyError): - logging.warning("Could not get Saagie version") - return "unknown-version " - - -def __get_min_max_saagie_versions(saagie_current_version: str) -> (Version, Version): - """ - Retrieve the minimum and maximum Saagie version supported by the Saagie API - based on the compatibility matrix - :param saagie_current_version: Saagie platform current version - :return: (min_version, max_version) - """ - saagie_compatibility_matrix = json.load(open(dir_path.joinpath('compatibility_matrix.json'))) - logging.debug(f"Compatibility matrix : {saagie_compatibility_matrix}") - try: - compatible_versions = saagie_compatibility_matrix[saagie_current_version] - minimal_version = version.parse(compatible_versions.get("min_api_saagie_version", "0")) - maximal_version = version.parse(compatible_versions.get("max_api_saagie_version", "9.9")) - return minimal_version, maximal_version - except KeyError: - logging.warning(f"Could not find your Saagie version ({saagie_current_version}) in the compatibility matrix") - return version.parse("0"), version.parse("9.9") - - -def check_saagie_version_compatibility(auth: BearerAuth, url_saagie: str): - """ - Check if the Saagie version is compatible with the Saagie API version and display warnings if not - :param auth: BearerAuth object - :param url_saagie: Saagie URL - """ - saagie_current_version = __get_saagie_current_version(auth, url_saagie) - saagie_api_current_version = version.parse(importlib.metadata.version("saagieapi")) - - logging.info(f"Current saagie version : {saagie_current_version}") - logging.info(f"Current api-saagie version : {saagie_api_current_version}") - min_version, max_version = __get_min_max_saagie_versions(saagie_current_version) - if saagie_api_current_version < min_version: - logging.warning( - f"You are using a saagie-api version ({saagie_api_current_version}) " - f"not compatible with your Saagie platform (Saagie v.{saagie_current_version})") - logging.warning( - f"Your Saagie platform requires at least the version {min_version} " - f"of saagieapi. Please consider upgrading.") - if saagie_api_current_version > max_version: - logging.warning( - f"You are using a saagie-api version ({saagie_api_current_version}) " - f"not compatible with your Saagie platform (Saagie v.{saagie_current_version})") - logging.warning( - f"Your Saagie platform is not compatible with versions > {max_version} " - f"of saagieapi. Please consider downgrading.") From 93553b5a57ab06f63a589b2b1c886d2724796c01 Mon Sep 17 00:00:00 2001 From: Thomas Quiviger Date: Fri, 8 Apr 2022 15:19:15 +0200 Subject: [PATCH 6/7] Refactoring --- saagieapi/projects/compatibility_matrix.json | 4 ++++ saagieapi/projects/saagie_api.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/saagieapi/projects/compatibility_matrix.json b/saagieapi/projects/compatibility_matrix.json index 041e1e81..e9323207 100644 --- a/saagieapi/projects/compatibility_matrix.json +++ b/saagieapi/projects/compatibility_matrix.json @@ -5,5 +5,9 @@ }, "2.2.0": { "min_api_saagie_version": "0.6.0" + }, + "1.19.2203251648-3207.161": { + "min_api_saagie_version": "0.6.0", + "max_api_saagie_version": "0.6.2" } } diff --git a/saagieapi/projects/saagie_api.py b/saagieapi/projects/saagie_api.py index 49d87d85..75f5fec8 100644 --- a/saagieapi/projects/saagie_api.py +++ b/saagieapi/projects/saagie_api.py @@ -96,6 +96,8 @@ def __init__(self, url_saagie, id_platform, user, password, realm, retries=0): # Valid status list of alerting self.valid_status_list = ["REQUESTED", "QUEUED", "RUNNING", "FAILED", "KILLED", "KILLING", "SUCCEEDED", "UNKNOWN", "AWAITING", "SKIPPED"] + self.saagie_current_version = self.__get_saagie_current_version() + logging.info(f"Current saagie version : {self.saagie_current_version}") self.check_saagie_version_compatibility() @classmethod @@ -138,11 +140,10 @@ def __get_saagie_current_version(self): """ try: saagie_versions = self.auth(requests.session()).get(f"{self.url_saagie}version.json").json() - self.saagie_current_version = saagie_versions['major'] + return saagie_versions['major'] except (JSONDecodeError, KeyError): logging.warning("Could not get Saagie version") - self.saagie_current_version = "unknown-version " - logging.info(f"Current saagie version : {self.saagie_current_version}") + return "unknown-version " def __get_min_max_saagie_versions(self) -> (Version, Version): """ @@ -166,7 +167,6 @@ def check_saagie_version_compatibility(self): """ Check if the Saagie version is compatible with the Saagie API version and display warnings if not """ - self.__get_saagie_current_version() saagie_api_current_version = version.parse(importlib.metadata.version("saagieapi")) logging.info(f"Current api-saagie version : {saagie_api_current_version}") From 8884e8061485a5f599796b8a0ee633124f2ed77e Mon Sep 17 00:00:00 2001 From: Thomas Quiviger Date: Fri, 8 Apr 2022 16:49:29 +0200 Subject: [PATCH 7/7] Refactoring --- saagieapi/projects/compatibility_matrix.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/saagieapi/projects/compatibility_matrix.json b/saagieapi/projects/compatibility_matrix.json index e9323207..397843dd 100644 --- a/saagieapi/projects/compatibility_matrix.json +++ b/saagieapi/projects/compatibility_matrix.json @@ -6,8 +6,7 @@ "2.2.0": { "min_api_saagie_version": "0.6.0" }, - "1.19.2203251648-3207.161": { - "min_api_saagie_version": "0.6.0", - "max_api_saagie_version": "0.6.2" + "2.2.1": { + "min_api_saagie_version": "0.6.0" } }