From 9c97bc6045951619229c9ca981ad997a87cdd480 Mon Sep 17 00:00:00 2001 From: Gabe Joseph Date: Thu, 27 Jan 2022 16:47:01 -0700 Subject: [PATCH 01/14] `get_versions` without importing packages --- distributed/tests/test_versions.py | 13 +++-- distributed/versions.py | 77 ++++++++++-------------------- 2 files changed, 32 insertions(+), 58 deletions(-) diff --git a/distributed/tests/test_versions.py b/distributed/tests/test_versions.py index 69803b8f144..af6d6a6929a 100644 --- a/distributed/tests/test_versions.py +++ b/distributed/tests/test_versions.py @@ -1,6 +1,7 @@ import re import sys +import msgpack import pytest import tornado @@ -145,17 +146,15 @@ def test_python_version(): def test_version_custom_pkgs(): out = get_versions( [ - # Use custom function - ("distributed", lambda mod: "123"), - # Use version_of_package "notexist", - ("pytest", None), # has __version__ - "tornado", # has version - "math", # has nothing + "pytest", + "tornado", + "msgpack", + "math", ] )["packages"] - assert out["distributed"] == "123" assert out["notexist"] is None assert out["pytest"] == pytest.__version__ assert out["tornado"] == tornado.version + assert out["msgpack"] == ".".join(str(v) for v in msgpack.version) assert out["math"] is None diff --git a/distributed/versions.py b/distributed/versions.py index d1e51d4c9cc..f521ef2dab8 100644 --- a/distributed/versions.py +++ b/distributed/versions.py @@ -2,35 +2,34 @@ from __future__ import annotations -import importlib +import importlib.metadata import os import platform import struct import sys -from collections.abc import Callable, Iterable +from collections.abc import Iterable from itertools import chain -from types import ModuleType from typing import Any -required_packages = [ - ("dask", lambda p: p.__version__), - ("distributed", lambda p: p.__version__), - ("msgpack", lambda p: ".".join([str(v) for v in p.version])), - ("cloudpickle", lambda p: p.__version__), - ("tornado", lambda p: p.version), - ("toolz", lambda p: p.__version__), +required_packages: list[str] = [ + "dask", + "distributed", + "msgpack", + "cloudpickle", + "tornado", + "toolz", ] -optional_packages = [ - ("numpy", lambda p: p.__version__), - ("pandas", lambda p: p.__version__), - ("lz4", lambda p: p.__version__), - ("blosc", lambda p: p.__version__), +optional_packages: list[str] = [ + "numpy", + "pandas", + "lz4", + "blosc", ] # only these scheduler packages will be checked for version mismatch -scheduler_relevant_packages = {pkg for pkg, _ in required_packages} | {"lz4", "blosc"} +scheduler_relevant_packages = set(required_packages) | {"lz4", "blosc"} # notes to be displayed for mismatch packages @@ -39,18 +38,18 @@ } -def get_versions( - packages: Iterable[str | tuple[str, Callable[[ModuleType], str | None]]] - | None = None -) -> dict[str, dict[str, Any]]: +def get_versions(packages: Iterable[str] | None = None) -> dict[str, dict[str, Any]]: """Return basic information on our software installation, and our installed versions of packages """ return { "host": get_system_info(), - "packages": get_package_info( - chain(required_packages, optional_packages, packages or []) - ), + "packages": { + "python": ".".join(map(str, sys.version_info)), + **get_package_info( + chain(required_packages, optional_packages, packages or []) + ), + }, } @@ -69,37 +68,13 @@ def get_system_info() -> dict[str, Any]: } -def version_of_package(pkg: ModuleType) -> str | None: - """Try a variety of common ways to get the version of a package""" - from contextlib import suppress - - with suppress(AttributeError): - return pkg.__version__ # type: ignore - with suppress(AttributeError): - return str(pkg.version) # type: ignore - with suppress(AttributeError): - return ".".join(map(str, pkg.version_info)) # type: ignore - return None - - -def get_package_info( - pkgs: Iterable[str | tuple[str, Callable[[ModuleType], str | None]]] -) -> dict[str, str | None]: +def get_package_info(pkgs: Iterable[str]) -> dict[str, str | None]: """get package versions for the passed required & optional packages""" - pversions: dict[str, str | None] = {"python": ".".join(map(str, sys.version_info))} - for pkg in pkgs: - if isinstance(pkg, (tuple, list)): - modname, ver_f = pkg - if ver_f is None: - ver_f = version_of_package - else: - modname = pkg - ver_f = version_of_package - + pversions: dict[str, str | None] = {} + for modname in pkgs: try: - mod = importlib.import_module(modname) - pversions[modname] = ver_f(mod) + pversions[modname] = importlib.metadata.version(modname) except Exception: pversions[modname] = None From 573128245ccd8da2f6a1d4e237a9611294745760 Mon Sep 17 00:00:00 2001 From: Gabe Joseph Date: Thu, 27 Jan 2022 17:08:17 -0700 Subject: [PATCH 02/14] importlib.metadata backport --- continuous_integration/environment-3.7.yaml | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/continuous_integration/environment-3.7.yaml b/continuous_integration/environment-3.7.yaml index 06608084279..538d8d022df 100644 --- a/continuous_integration/environment-3.7.yaml +++ b/continuous_integration/environment-3.7.yaml @@ -15,6 +15,7 @@ dependencies: - dask # overridden by git tip below - filesystem-spec - h5py + - importlib_metadata # Only needed here - ipykernel - ipywidgets - jinja2 diff --git a/requirements.txt b/requirements.txt index fd7ec507732..be351ba1966 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ click >= 6.6 cloudpickle >= 1.5.0 dask == 2022.01.0 +importlib_metadata >= 1.4;python_version<'3.8' jinja2 msgpack >= 0.6.0 packaging >= 20.0 From df3d119e683b08d5391103b5d0ff002e082c0b17 Mon Sep 17 00:00:00 2001 From: Gabe Joseph Date: Thu, 27 Jan 2022 18:29:28 -0700 Subject: [PATCH 03/14] use pkg_resources for 3.7 instead of backport --- continuous_integration/environment-3.7.yaml | 1 - distributed/versions.py | 15 +++++++++++++-- requirements.txt | 1 - 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/continuous_integration/environment-3.7.yaml b/continuous_integration/environment-3.7.yaml index 538d8d022df..06608084279 100644 --- a/continuous_integration/environment-3.7.yaml +++ b/continuous_integration/environment-3.7.yaml @@ -15,7 +15,6 @@ dependencies: - dask # overridden by git tip below - filesystem-spec - h5py - - importlib_metadata # Only needed here - ipykernel - ipywidgets - jinja2 diff --git a/distributed/versions.py b/distributed/versions.py index f521ef2dab8..3aa3ac66c66 100644 --- a/distributed/versions.py +++ b/distributed/versions.py @@ -2,7 +2,7 @@ from __future__ import annotations -import importlib.metadata +import importlib import os import platform import struct @@ -11,6 +11,17 @@ from itertools import chain from typing import Any +try: + import importlib.metadata +except ImportError: + # Python 3.7 compatibility + + import pkg_resources + + _version = lambda modname: pkg_resources.get_distribution(modname).version +else: + _version = importlib.metadata.version + required_packages: list[str] = [ "dask", "distributed", @@ -74,7 +85,7 @@ def get_package_info(pkgs: Iterable[str]) -> dict[str, str | None]: pversions: dict[str, str | None] = {} for modname in pkgs: try: - pversions[modname] = importlib.metadata.version(modname) + pversions[modname] = _version(modname) except Exception: pversions[modname] = None diff --git a/requirements.txt b/requirements.txt index be351ba1966..fd7ec507732 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ click >= 6.6 cloudpickle >= 1.5.0 dask == 2022.01.0 -importlib_metadata >= 1.4;python_version<'3.8' jinja2 msgpack >= 0.6.0 packaging >= 20.0 From 1dadc3c47f6ba16bcf26aaac2d585adae8a79482 Mon Sep 17 00:00:00 2001 From: Gabe Joseph Date: Thu, 27 Jan 2022 18:31:43 -0700 Subject: [PATCH 04/14] remove superfluous type annotation --- distributed/versions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distributed/versions.py b/distributed/versions.py index 3aa3ac66c66..ce67f6f758b 100644 --- a/distributed/versions.py +++ b/distributed/versions.py @@ -22,7 +22,7 @@ else: _version = importlib.metadata.version -required_packages: list[str] = [ +required_packages = [ "dask", "distributed", "msgpack", @@ -31,7 +31,7 @@ "toolz", ] -optional_packages: list[str] = [ +optional_packages = [ "numpy", "pandas", "lz4", From 3b4cb0ff7d0044d6e8e27f64344051a6d27ee5da Mon Sep 17 00:00:00 2001 From: Gabe Joseph Date: Thu, 27 Jan 2022 18:33:45 -0700 Subject: [PATCH 05/14] empty tuple as default --- distributed/node.py | 2 +- distributed/versions.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/distributed/node.py b/distributed/node.py index 8e59bafb22c..f1bace03f9c 100644 --- a/distributed/node.py +++ b/distributed/node.py @@ -25,7 +25,7 @@ class ServerNode(Server): # XXX avoid inheriting from Server? there is some large potential for confusion # between base and derived attribute namespaces... - def versions(self, comm=None, packages=None): + def versions(self, comm=None, packages=()): return get_versions(packages=packages) def start_services(self, default_listen_ip): diff --git a/distributed/versions.py b/distributed/versions.py index ce67f6f758b..a93d7b504a5 100644 --- a/distributed/versions.py +++ b/distributed/versions.py @@ -49,7 +49,7 @@ } -def get_versions(packages: Iterable[str] | None = None) -> dict[str, dict[str, Any]]: +def get_versions(packages: Iterable[str] = ()) -> dict[str, dict[str, Any]]: """Return basic information on our software installation, and our installed versions of packages """ @@ -57,9 +57,7 @@ def get_versions(packages: Iterable[str] | None = None) -> dict[str, dict[str, A "host": get_system_info(), "packages": { "python": ".".join(map(str, sys.version_info)), - **get_package_info( - chain(required_packages, optional_packages, packages or []) - ), + **get_package_info(chain(required_packages, optional_packages, packages)), }, } From 3b237e8aa3f38b77da4b72bd413a685b52f85cd9 Mon Sep 17 00:00:00 2001 From: Gabe Joseph Date: Fri, 28 Jan 2022 09:47:55 -0700 Subject: [PATCH 06/14] conditional on Python version --- distributed/versions.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/distributed/versions.py b/distributed/versions.py index a93d7b504a5..9e804c056d8 100644 --- a/distributed/versions.py +++ b/distributed/versions.py @@ -2,7 +2,6 @@ from __future__ import annotations -import importlib import os import platform import struct @@ -11,16 +10,14 @@ from itertools import chain from typing import Any -try: - import importlib.metadata -except ImportError: - # Python 3.7 compatibility - +if sys.version_info >= (3, 8): + from importlib.metadata import version as _version +else: import pkg_resources - _version = lambda modname: pkg_resources.get_distribution(modname).version -else: - _version = importlib.metadata.version + def _version(distribution_name: str) -> str: + return pkg_resources.get_distribution(distribution_name).version + required_packages = [ "dask", From e8aab1f7935fde5fc166d0ae8d153385598d908c Mon Sep 17 00:00:00 2001 From: Gabe Joseph Date: Tue, 22 Feb 2022 10:11:55 -0700 Subject: [PATCH 07/14] Drop 3.7 Co-authored-by: crusaderky --- distributed/versions.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/distributed/versions.py b/distributed/versions.py index 9e804c056d8..d58741092d2 100644 --- a/distributed/versions.py +++ b/distributed/versions.py @@ -10,13 +10,7 @@ from itertools import chain from typing import Any -if sys.version_info >= (3, 8): - from importlib.metadata import version as _version -else: - import pkg_resources - - def _version(distribution_name: str) -> str: - return pkg_resources.get_distribution(distribution_name).version +import importlib.metadata required_packages = [ @@ -80,7 +74,7 @@ def get_package_info(pkgs: Iterable[str]) -> dict[str, str | None]: pversions: dict[str, str | None] = {} for modname in pkgs: try: - pversions[modname] = _version(modname) + pversions[modname] = importlib.metadata.version(modname) except Exception: pversions[modname] = None From 6bdae99d514df5129ede52ea4f362472889e8b7b Mon Sep 17 00:00:00 2001 From: Gabe Joseph Date: Tue, 22 Feb 2022 10:20:19 -0700 Subject: [PATCH 08/14] un-xfail pandas --- distributed/tests/test_nanny.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py index 4a944a42aba..5fcb415be15 100644 --- a/distributed/tests/test_nanny.py +++ b/distributed/tests/test_nanny.py @@ -621,7 +621,7 @@ async def test_environ_plugin(c, s, a, b): marks=pytest.mark.xfail(reason="distributed#5723, distributed#5729"), ), "scipy", - pytest.param("pandas", marks=pytest.mark.xfail(reason="distributed#5723")), + "pandas", ], ) @gen_cluster(client=True, Worker=Nanny, nthreads=[("", 1)]) From 110956d2e64ff2ec928f7168fa9f1cb3b6eb1064 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Wed, 23 Feb 2022 13:20:57 +0000 Subject: [PATCH 09/14] Update distributed/tests/test_nanny.py --- distributed/tests/test_nanny.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py index 5fcb415be15..9994568bd83 100644 --- a/distributed/tests/test_nanny.py +++ b/distributed/tests/test_nanny.py @@ -618,7 +618,7 @@ async def test_environ_plugin(c, s, a, b): [ pytest.param( "numpy", - marks=pytest.mark.xfail(reason="distributed#5723, distributed#5729"), + marks=pytest.mark.xfail(reason="distributed#5729"), ), "scipy", "pandas", From 32d6242bb91471cf6ad9eadea9d15e70c1454c69 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Wed, 23 Feb 2022 13:22:52 +0000 Subject: [PATCH 10/14] Update distributed/tests/test_nanny.py --- distributed/tests/test_nanny.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py index 9994568bd83..c2967efb554 100644 --- a/distributed/tests/test_nanny.py +++ b/distributed/tests/test_nanny.py @@ -618,7 +618,7 @@ async def test_environ_plugin(c, s, a, b): [ pytest.param( "numpy", - marks=pytest.mark.xfail(reason="distributed#5729"), + marks=pytest.mark.xfail(reason="distributed#5729") ), "scipy", "pandas", From eb14462f23e7334556d4731ef7747afb1429c05f Mon Sep 17 00:00:00 2001 From: crusaderky Date: Wed, 23 Feb 2022 13:29:25 +0000 Subject: [PATCH 11/14] Update distributed/versions.py Co-authored-by: Thomas Grainger --- distributed/versions.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/distributed/versions.py b/distributed/versions.py index d58741092d2..99856eedc11 100644 --- a/distributed/versions.py +++ b/distributed/versions.py @@ -10,9 +10,6 @@ from itertools import chain from typing import Any -import importlib.metadata - - required_packages = [ "dask", "distributed", From 14597ae5a558a49a4674fcdb1f2f8dfde452e8d7 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Wed, 23 Feb 2022 13:29:29 +0000 Subject: [PATCH 12/14] Update distributed/versions.py Co-authored-by: Thomas Grainger --- distributed/versions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/distributed/versions.py b/distributed/versions.py index 99856eedc11..c287b7e69d9 100644 --- a/distributed/versions.py +++ b/distributed/versions.py @@ -2,6 +2,7 @@ from __future__ import annotations +import importlib.metadata import os import platform import struct From 984f5c0f481415db3491232cfef4fef8d79b1fbf Mon Sep 17 00:00:00 2001 From: crusaderky Date: Wed, 23 Feb 2022 13:43:12 +0000 Subject: [PATCH 13/14] Update distributed/tests/test_nanny.py Co-authored-by: Thomas Grainger --- distributed/tests/test_nanny.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py index c2967efb554..8e910c75c0f 100644 --- a/distributed/tests/test_nanny.py +++ b/distributed/tests/test_nanny.py @@ -618,8 +618,6 @@ async def test_environ_plugin(c, s, a, b): [ pytest.param( "numpy", - marks=pytest.mark.xfail(reason="distributed#5729") - ), "scipy", "pandas", ], From b6d99ee89348c98bfb8462af9c7af473f98a22fd Mon Sep 17 00:00:00 2001 From: crusaderky Date: Wed, 23 Feb 2022 13:43:17 +0000 Subject: [PATCH 14/14] Update distributed/tests/test_nanny.py Co-authored-by: Thomas Grainger --- distributed/tests/test_nanny.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py index 8e910c75c0f..023c36e5071 100644 --- a/distributed/tests/test_nanny.py +++ b/distributed/tests/test_nanny.py @@ -616,8 +616,7 @@ async def test_environ_plugin(c, s, a, b): @pytest.mark.parametrize( "modname", [ - pytest.param( - "numpy", + pytest.param("numpy", marks=pytest.mark.xfail(reason="distributed#5729")), "scipy", "pandas", ],