diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..68bc17f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/duneanalytics/__version__.py b/duneanalytics/__version__.py index fade2ddd..e427cbf8 100644 --- a/duneanalytics/__version__.py +++ b/duneanalytics/__version__.py @@ -4,7 +4,7 @@ __url__ = 'https://github.com/itzmestar/duneanalytics' __version__ = '1.2.1' __build__ = 0x010001 -__author__ = 'Tarique Anwer' -__author_email__ = 'itzmetariq@gmail.com' -__license__ = 'Apache License 2.0' -__copyright__ = 'Copyright 2021 Tarique Anwer' +__author__ = "Tarique Anwer" +__author_email__ = "itzmetariq@gmail.com" +__license__ = "Apache License 2.0" +__copyright__ = "Copyright 2021 Tarique Anwer" diff --git a/duneanalytics/duneanalytics.py b/duneanalytics/duneanalytics.py index 61204f6b..9593b8b7 100644 --- a/duneanalytics/duneanalytics.py +++ b/duneanalytics/duneanalytics.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # """This provides the DuneAnalytics class implementation""" +from datetime import datetime from requests import Session import logging @@ -12,7 +13,7 @@ # --------- Constants --------- # logging.basicConfig( level=logging.INFO, - format='%(asctime)s : %(levelname)s : %(funcName)-9s : %(message)s' + format="%(asctime)s : %(levelname)s : %(funcName)-9s : %(message)s", ) logger = logging.getLogger("dune") @@ -36,16 +37,16 @@ def __init__(self, username, password): self.password = password self.session = Session() headers = { - 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,' - 'image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', - 'dnt': '1', - 'sec-ch-ua': '"Google Chrome";v="95", "Chromium";v="95", ";Not A Brand";v="99"', - 'sec-ch-ua-mobile': '?0', - 'sec-fetch-dest': 'document', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-site', - 'origin': BASE_URL, - 'upgrade-insecure-requests': '1' + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp," + "image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "dnt": "1", + "sec-ch-ua": '"Google Chrome";v="95", "Chromium";v="95", ";Not A Brand";v="99"', + "sec-ch-ua-mobile": "?0", + "sec-fetch-dest": "document", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-site", + "origin": BASE_URL, + "upgrade-insecure-requests": "1", } self.session.headers.update(headers) @@ -54,28 +55,28 @@ def login(self): Try to login to duneanalytics.com & get the token :return: """ - login_url = BASE_URL + '/auth/login' - csrf_url = BASE_URL + '/api/auth/csrf' - auth_url = BASE_URL + '/api/auth' + login_url = BASE_URL + "/auth/login" + csrf_url = BASE_URL + "/api/auth/csrf" + auth_url = BASE_URL + "/api/auth" # fetch login page self.session.get(login_url) # get csrf token self.session.post(csrf_url) - self.csrf = self.session.cookies.get('csrf') + self.csrf = self.session.cookies.get("csrf") # try to login form_data = { - 'action': 'login', - 'username': self.username, - 'password': self.password, - 'csrf': self.csrf, - 'next': BASE_URL + "action": "login", + "username": self.username, + "password": self.password, + "csrf": self.csrf, + "next": BASE_URL, } self.session.post(auth_url, data=form_data) - self.auth_refresh = self.session.cookies.get('auth-refresh') + self.auth_refresh = self.session.cookies.get("auth-refresh") if self.auth_refresh is None: logger.warning("Login Failed!") @@ -84,11 +85,11 @@ def fetch_auth_token(self): Fetch authorization token for the user :return: """ - session_url = BASE_URL + '/api/auth/session' + session_url = BASE_URL + "/api/auth/session" response = self.session.post(session_url) if response.status_code == 200: - self.token = response.json().get('token') + self.token = response.json().get("token") if self.token is None: logger.warning("Fetching Token Failed!") else: @@ -101,22 +102,24 @@ def query_result_id(self, query_id): :param query_id: provide the query_id :return: """ - query_data = {"operationName": "GetResult", "variables": {"query_id": query_id}, - "query": "query GetResult($query_id: Int!, $parameters: [Parameter!]) " - "{\n get_result_v2(query_id: $query_id, parameters: $parameters) " - "{\n job_id\n result_id\n error_id\n __typename\n }\n}\n" - } + query_data = { + "operationName": "GetResult", + "variables": {"query_id": query_id}, + "query": "query GetResult($query_id: Int!, $parameters: [Parameter!]) " + "{\n get_result_v2(query_id: $query_id, parameters: $parameters) " + "{\n job_id\n result_id\n error_id\n __typename\n }\n}\n", + } - self.session.headers.update({'authorization': f'Bearer {self.token}'}) + self.session.headers.update({"authorization": f"Bearer {self.token}"}) response = self.session.post(GRAPH_URL, json=query_data) if response.status_code == 200: data = response.json() logger.debug(data) - if 'errors' in data: - logger.error(data.get('errors')) + if "errors" in data: + logger.error(data.get("errors")) return None - result_id = data.get('data').get('get_result_v2').get('result_id') + result_id = data.get("data").get("get_result_v2").get("result_id") return result_id else: logger.error(response.text) @@ -128,17 +131,21 @@ def query_result(self, result_id): :param result_id: result id of the query :return: """ - query_data = {"operationName": "FindResultDataByResult", - "variables": {"result_id": result_id, "error_id": "00000000-0000-0000-0000-000000000000"}, - "query": "query FindResultDataByResult($result_id: uuid!, $error_id: uuid!) " - "{\n query_results(where: {id: {_eq: $result_id}}) " - "{\n id\n job_id\n runtime\n generated_at\n columns\n __typename\n }" - "\n query_errors(where: {id: {_eq: $error_id}}) {\n id\n job_id\n runtime\n" - " message\n metadata\n type\n generated_at\n __typename\n }\n" - "\n get_result_by_result_id(args: {want_result_id: $result_id}) {\n data\n __typename\n }\n}\n" - } - - self.session.headers.update({'authorization': f'Bearer {self.token}'}) + query_data = { + "operationName": "FindResultDataByResult", + "variables": { + "result_id": result_id, + "error_id": "00000000-0000-0000-0000-000000000000", + }, + "query": "query FindResultDataByResult($result_id: uuid!, $error_id: uuid!) " + "{\n query_results(where: {id: {_eq: $result_id}}) " + "{\n id\n job_id\n runtime\n generated_at\n columns\n __typename\n }" + "\n query_errors(where: {id: {_eq: $error_id}}) {\n id\n job_id\n runtime\n" + " message\n metadata\n type\n generated_at\n __typename\n }\n" + "\n get_result_by_result_id(args: {want_result_id: $result_id}) {\n data\n __typename\n }\n}\n", + } + + self.session.headers.update({"authorization": f"Bearer {self.token}"}) response = self.session.post(GRAPH_URL, json=query_data) if response.status_code == 200: @@ -148,3 +155,67 @@ def query_result(self, result_id): else: logger.error(response.text) return {} + + def force_query_update(self, query_id): + """ + Force a query to update + :param query_id: query id + :return: Job ID if successful, None otherwise + """ + query_data = { + "operationName": "ExecuteQuery", + "variables": {"query_id": query_id, "parameters": []}, + "query": "mutation ExecuteQuery($query_id: Int!, $parameters: [Parameter!]!) " + "{\n execute_query(query_id: $query_id, parameters: $parameters) " + "{\n job_id\n __typename\n }\n}\n", + } + self.session.headers.update({"authorization": f"Bearer {self.token}"}) + + response = self.session.post(GRAPH_URL, json=query_data) + if response.status_code == 200: + data = response.json() + if "errors" in data: + raise Exception( + f"{data.get('errors')[0].get('message')}\nCode: {data.get('errors')[0].get('extensions').get('code')}" + ) + job_id = data.get("data").get("execute_query").get("job_id") + return job_id + else: + print(response.text) + return None + + def job_result(self, job_id): + """ + Fetch the result for a job + :param job_id: job id + :return: Job status + """ + + query_data = { + "operationName": "FindResultJob", + "variables": {"job_id": job_id}, + "query": "query FindResultJob($job_id: uuid) {\n jobs(where: {id: {_eq: $job_id}}) " + "{\n id\n user_id\n locked_until\n created_at\n category\n __typename\n }" + "\n view_queue_positions(where: {id: {_eq: $job_id}}) {\n pos\n __typename\n }\n}\n", + } + self.session.headers.update({"authorization": f"Bearer {self.token}"}) + + response = self.session.post(GRAPH_URL, json=query_data) + if response.status_code == 200: + data = response.json() + if "errors" in data: + return data.get("errors") + jobs_result = data.get("data").get("jobs") + queue_position_result = data.get("data").get("view_queue_positions") + + if len(jobs_result) == 0 and len(queue_position_result) == 0: + return "Job executed" + else: + lock_time = jobs_result[0].get("locked_until") + if lock_time: + lock_time = datetime.strptime(lock_time, "%Y-%m-%dT%H:%M:%S.%f%z") + lock_time = datetime.strftime(lock_time, "%Y-%m-%d %H:%M:%S %Z") + return f"Job {job_id} locked until {lock_time}" + else: + print(response.text) + return None diff --git a/setup.py b/setup.py index c55c6ee5..6ac4efcf 100644 --- a/setup.py +++ b/setup.py @@ -3,32 +3,34 @@ here = os.path.abspath(os.path.dirname(__file__)) -packages = ['duneanalytics'] +packages = ["duneanalytics"] requires = [ - 'requests>=2.18.4', + "requests>=2.18.4", ] about = {} -with open(os.path.join(here, 'duneanalytics', '__version__.py'), mode='r', encoding='utf-8') as f: +with open( + os.path.join(here, "duneanalytics", "__version__.py"), mode="r", encoding="utf-8" +) as f: exec(f.read(), about) -with open('README.md', mode='r', encoding='utf-8') as f: +with open("README.md", mode="r", encoding="utf-8") as f: readme = f.read() setuptools.setup( - name=about['__title__'], - version=about['__version__'], - description=about['__description__'], + name=about["__title__"], + version=about["__version__"], + description=about["__description__"], long_description=readme, - long_description_content_type='text/markdown', - author=about['__author__'], - author_email=about['__author_email__'], - url=about['__url__'], + long_description_content_type="text/markdown", + author=about["__author__"], + author_email=about["__author_email__"], + url=about["__url__"], packages=packages, install_requires=requires, - license=about['__license__'], + license=about["__license__"], classifiers=[ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", diff --git a/tests/conftest.py b/tests/conftest.py index 6076a7b6..36b17850 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,8 +3,10 @@ from duneanalytics import DuneAnalytics -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def dune(): - print('===============Start=======================') - yield DuneAnalytics(username=os.getenv('DUNE_USER'), password=os.getenv('DUNE_PASS')) - print('===============End=======================') + print("===============Start=======================") + yield DuneAnalytics( + username=os.getenv("DUNE_USER"), password=os.getenv("DUNE_PASS") + ) + print("===============End=======================") diff --git a/tests/test_duneanalytics.py b/tests/test_duneanalytics.py index 6f97907e..5bd79ed5 100644 --- a/tests/test_duneanalytics.py +++ b/tests/test_duneanalytics.py @@ -1,11 +1,12 @@ import pytest -@pytest.mark.usefixtures('dune') +@pytest.mark.usefixtures("dune") class TestDuneAnalytics: """ Class to test DuneAnalytics """ + def test_login(self, dune): # try to login dune.login()