Updates for new numpy, torch and python >= 3.12

.github/workflows/release-check.yml

@@ -17,18 +17,20 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Set up Python 3.8
+    - name: Set up Python 3.9
       uses: actions/setup-python@v4
       with:
-        python-version: 3.8
+        python-version: 3.9
 
     - name: Install dependencies
-      run: pip install --user -U pip poetry
+      run: |
+        pip install --user -U pip uv
+        uv sync
 
     - name: Build
       run: |
-        poetry install
-        poetry build
+        uv pip install build
+        uv python -m build
 
     - name: Publish
       uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/release.yml

@@ -37,18 +37,20 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Set up Python 3.8
+    - name: Set up Python 3.9
       uses: actions/setup-python@v4
       with:
-        python-version: 3.8
+        python-version: 3.9
 
     - name: Install dependencies
-      run: pip install --user --upgrade pip poetry
+      run: |
+        pip install --user -U pip uv
+        uv sync
 
     - name: Build
       run: |
-        poetry install
-        poetry build
+        uv pip install build
+        uv python -m build
 
     - name: Publish
       uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/status.yml

@@ -11,19 +11,18 @@ jobs:
 
     runs-on: ubuntu-latest
 
-    container:
-      image: analysiscenter1/ds-py3:3.8
-      options: --entrypoint ""
-
     steps:
     - uses: actions/checkout@v3
 
-    - name: Update pylint
-      run: pip3 install -U pylint>=3.2
+    - name: Install from local source
+      run: |
+        pip install uv
+        uv sync --all-extras
 
-    - name: Check pylint
-      run: pylint -rn --rcfile pylintrc batchflow
+    - name: Linting
+      run: uv run ruff check batchflow
 
     - name: Run tests
       if: always()
-      run: pytest -m "not slow" --disable-pytest-warnings -v
+      run:
+        uv run pytest -m "not slow" --disable-pytest-warnings -v

.github/workflows/test-install.yml

@@ -16,7 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.8, 3.9, '3.10', 3.11]
+        python-version: [3.9, '3.10', 3.11, 3.12]
 
     runs-on: ${{ matrix.os }}
 
@@ -52,7 +52,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.8, 3.9, '3.10', 3.11]
+        python-version: [3.9, '3.10', 3.11, 3.12]
 
     runs-on: ${{ matrix.os }}
 
@@ -87,7 +87,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.8, 3.9, '3.10', 3.11]
+        python-version: [3.9, '3.10', 3.11, 3.12]
 
     runs-on: ${{ matrix.os }}
 
@@ -141,3 +141,71 @@ jobs:
       run: |
         cd tests
         poetry run python -m pytest --disable-pytest-warnings -v dataset_test.py filesindex_test.py datasetindex_test.py
+
+# -----------------------------------------
+#    Install with uv
+# -----------------------------------------
+  install_with_uv:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: [3.9, '3.10', 3.11, 3.12]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install batchflow with uv
+      run: |
+        pip install --user -U pip
+        pip install wheel uv
+
+        echo '[project]
+        name = "test_project"
+        description = "test"
+        version = "0.0.1"
+        authors = [{ name = "Test", email = "test@test.test" }]
+
+        requires-python = ">=${{ matrix.python-version }}"
+
+        dependencies = [
+          "pytest>=7.0"
+        ]
+
+        [build-system]
+        requires = ["hatchling"]
+        build-backend = "hatchling.build"
+
+        [tool.hatch.build.targets.wheel]
+        packages = ["."]
+        ' >> pyproject.toml
+
+        uv add git+https://github.com/${{ github.event.pull_request.head.repo.full_name }}.git@${{ github.head_ref }}
+
+    - name: Run 'import batchflow' in installed environment
+      run: uv run python -c 'import batchflow'
+
+    - uses: actions/checkout@v4
+      with:
+        path: src
+
+    - name: Prepare directory
+      if: runner.os != 'Windows'
+      run: |
+        cp -r src/batchflow/tests .
+
+    - name: Prepare directory
+      if: runner.os == 'Windows'
+      run: |
+        xcopy /I /S src\batchflow\tests tests
+
+    - name: Run basic tests
+      run: |
+        cd tests
+        uv run python -m pytest --disable-pytest-warnings -v dataset_test.py filesindex_test.py datasetindex_test.py

README.md

@@ -1,5 +1,5 @@
 [![License](https://img.shields.io/github/license/analysiscenter/batchflow.svg)](https://www.apache.org/licenses/LICENSE-2.0)
-[![Python](https://img.shields.io/badge/python-3.8-blue.svg)](https://python.org)
+[![Python](https://img.shields.io/badge/python-3.9-blue.svg)](https://python.org)
 [![PyTorch](https://img.shields.io/badge/PyTorch-2.0-orange.svg)](https://pytorch.org)
 [![codecov](https://codecov.io/gh/analysiscenter/batchflow/branch/master/graph/badge.svg)](https://codecov.io/gh/analysiscenter/batchflow)
 [![PyPI](https://badge.fury.io/py/batchflow.svg)](https://badge.fury.io/py/batchflow)
@@ -78,10 +78,15 @@ For more advanced cases and detailed API see [the documentation](https://analysi
 
 > `BatchFlow` module is in the beta stage. Your suggestions and improvements are very welcome.
 
-> `BatchFlow` supports python 3.6 or higher.
+> `BatchFlow` supports Python 3.9 or higher.
 
 ### Stable python package
 
+With [uv](https://docs.astral.sh/uv/)
+```
+uv add batchflow
+```
+
 With [poetry](https://python-poetry.org/)
 ```
 poetry add batchflow
@@ -94,14 +99,23 @@ pip3 install batchflow
 
 ### Development version
 
+With [uv](https://docs.astral.sh/uv/)
+```
+git clone --branch my_branch https://github.com/analysiscenter/batchflow
+uv add --editable ./batchflow
+```
+
+You can skip `--branch` if you need `master`.
+
 With [poetry](https://python-poetry.org/)
 ```
-poetry add --editable git+https://github.com/analysiscenter/batchflow
+poetry add --editable git+https://github.com/analysiscenter/batchflow#my_branch
 ```
 
 With old-fashioned [pip](https://pip.pypa.io/en/stable/)
 ```
-pip install --editable git+https://github.com/analysiscenter/batchflow
+git clone --branch my_branch https://github.com/analysiscenter/batchflow
+pip install --editable ./batchflow
 ```
 
 ### Extras
@@ -115,7 +129,7 @@ In order to use that functionality you might need to install `batchflow` with ex
 - jupyter - utility functions for notebooks
 - research - multiprocess research
 - telegram - for monitoring pipelines via a telegram bot
-- dev - batchflow development (pylint, pytest, ...)
+- dev - batchflow development (ruff, pytest, ...)
 
 You can install several extras at once, like `batchflow[image,nn,research]`.
 

batchflow/__init__.py

@@ -3,8 +3,8 @@
 import os
 import re
 
-if sys.version_info < (3, 8):
-    raise ImportError("BatchFlow module requires Python 3.8 or higher")
+if sys.version_info < (3, 9): # noqa: UP036; outdated-version-block
+    raise ImportError("BatchFlow module requires Python 3.9 or higher")
 
 from importlib.metadata import version, PackageNotFoundError
 

batchflow/base.py

@@ -86,7 +86,7 @@ def calc_split(self, shares=0.8):
         """
         _shares = [shares] if isinstance(shares, (int, float)) else shares
         _shares = _shares if len(_shares) > 2 else _shares + [.0]
-        _shares = np.array(_shares).ravel()         # pylint: disable=no-member
+        _shares = np.array(_shares).ravel()
         n_items = len(self)
 
         if _shares.shape[0] > 3:

batchflow/batch.py

@@ -1,5 +1,4 @@
 """ Contains basic Batch classes """
-# pylint: disable=ungrouped-imports
 import os
 import traceback
 import threading
@@ -36,30 +35,30 @@ class MethodsTransformingMeta(type):
            is necessary in order to allow inner calls of untransformed versions
            (e.g. `ImagesBatch.scale` calls `ImagesBatch.crop` under the hood).
     """
-    def __new__(mcs, name, bases, namespace):
+    def __new__(cls, name, bases, namespace):
         namespace_ = namespace.copy()
         for object_name, object_ in namespace.items():
             transform_kwargs = getattr(object_, 'apply_kwargs', None)
             if transform_kwargs is not None:
-                namespace_[object_name] = mcs.use_apply_parallel(object_, **transform_kwargs)
+                namespace_[object_name] = cls.use_apply_parallel(object_, **transform_kwargs)
 
                 disclaimer = f"This is an untransformed version of `{object_.__qualname__}`.\n\n"
                 object_.__doc__ = disclaimer + (object_.__doc__ or '')
                 object_.__name__ = '_' + object_name + '_'
                 object_.__qualname__ = '.'.join(object_.__qualname__.split('.')[:-1] + [object_.__name__])
                 namespace_[object_.__name__] = object_
 
-        return super().__new__(mcs, name, bases, namespace_)
+        return super().__new__(cls, name, bases, namespace_)
 
     @classmethod
-    def use_apply_parallel(mcs, method, **apply_kwargs):
+    def use_apply_parallel(cls, method, **apply_kwargs):
         """ Wrap passed `method` in accordance with `all` arg value """
         @functools.wraps(method)
         def apply_parallel_wrapper(self, *args, **kwargs):
             transform = self.apply_parallel
 
             # Bound method to class:
-            method_ = method.__get__(self, type(self)) # pylint: disable=unnecessary-dunder-call
+            method_ = method.__get__(self, type(self))
 
             kwargs_full = {**self.apply_defaults, **apply_kwargs, **kwargs}
             return transform(method_, *args, **kwargs_full)
@@ -122,7 +121,7 @@ def data(self):
                     if self._data is None and self._preloaded is not None:
                         self.load(src=self._preloaded)
             res = self._data if self.components is None else self._data_named
-        except Exception as exc:
+        except Exception as exc: # noqa: BLE001, blind-except
             print("Exception:", exc)
             traceback.print_tb(exc.__traceback__)
             raise
@@ -200,7 +199,7 @@ def __copy__(self):
 
     def deepcopy(self):
         """ Return a deep copy of the batch. """
-        return self.__copy__() # pylint: disable=unnecessary-dunder-call
+        return self.__copy__()
 
     @classmethod
     def from_data(cls, index=None, data=None):
@@ -390,7 +389,7 @@ def add_components(self, components, init=None):
         return self
 
     def __getattr__(self, name):
-        if self.components is not None and name in self.components:   # pylint: disable=unsupported-membership-test
+        if self.components is not None and name in self.components:
             return getattr(self.data, name, None)
         raise AttributeError(f"{name} not found in class {self.__class__.__name__}")
 
@@ -404,7 +403,7 @@ def __setattr__(self, name, value):
                     else:
                         self._data_named = create_item_class(self.components, self._data)
                 return
-            if name in self.components:    # pylint: disable=unsupported-membership-test
+            if name in self.components:
                 # preload data if needed
                 _ = self.data
                 if self._data_named is None or self._data_named.components != self.components:
@@ -585,7 +584,6 @@ def apply_parallel(self, func, init=None, post=None, src=None, dst=None, *args,
 
         TODO: move logic of applying `post` function from `inbatch_parallel` here, as well as remove `use_self` arg.
         """
-        #pylint: disable=keyword-arg-before-vararg
         # Parse parameters: fill with class-wide defaults
         init = init or self.apply_defaults.get('init', None)
         post = post or self.apply_defaults.get('post', None)
@@ -628,7 +626,7 @@ def apply_parallel(self, func, init=None, post=None, src=None, dst=None, *args,
 
         # Compute result. Unbind the method to pass self explicitly
         parallel = inbatch_parallel(init=init, post=post, target=target, src=src, dst=dst)
-        transform = parallel(type(self)._apply_once)
+        transform = parallel(type(self)._apply_once)  # noqa: SLF001; private-member-access
         result = transform(self, *args, func=func, p=p, src=src, dst=dst,
                            apply_parallel_id=worker_ids, apply_parallel_seeds=rng_seeds, **kwargs)
         return result
@@ -811,7 +809,7 @@ def _load_blosc(self, ix, src=None, dst=None):
             components = tuple(dst or self.components)
             try:
                 item = tuple(data[i] for i in components)
-            except Exception as e:
+            except Exception as e: # noqa: BLE001, blind-except
                 raise KeyError('Cannot find components in corresponfig file', file_name) from e
         return item
 
@@ -882,9 +880,9 @@ def _dump_table(self, dst, fmt='feather', components=None, *args, **kwargs):
         if fmt == 'feather':
             feather.write_dataframe(_data, filename, *args, **kwargs)
         elif fmt == 'hdf5':
-            _data.to_hdf(filename, *args, **kwargs)   # pylint:disable=no-member
+            _data.to_hdf(filename, *args, **kwargs)
         elif fmt == 'csv':
-            _data.to_csv(filename, *args, **kwargs)   # pylint:disable=no-member
+            _data.to_csv(filename, *args, **kwargs)
         else:
             raise ValueError(f'Unknown format {fmt}')
 

batchflow/dataset.py

@@ -211,7 +211,7 @@ def create_subset(self, index):
             raise IndexError
         return type(self).from_dataset(self, self.index.create_subset(index))
 
-    def create_batch(self, index, pos=False, *args, **kwargs): # pylint: disable=arguments-renamed
+    def create_batch(self, index, pos=False, *args, **kwargs):
         """ Create a batch from given indices.
 
             Parameters
@@ -282,7 +282,7 @@ def cv(self, n):
             raise ValueError(f"The dataset has been split into fewer splits than {n}")
         return  getattr(self, 'cv' + str(n))
 
-    def CV(self, expr):
+    def CV(self, expr): # noqa: N802; invalid-function-name
         """ Return a dataset which corresponds to the fold defined as NamedExpression """
         return  F(self.cv)(expr)
 
@@ -321,7 +321,6 @@ def cv_split(self, method='kfold', n_splits=5, shuffle=False):
             print(dataset.test.cv1.indices) # [4, 5, 6]
             print(dataset.test.cv2.indices) # [7, 8, 9]
         """
-        # pylint: disable=access-member-before-definition
         if self.n_splits is not None:
             for i in range(self.n_splits):
                 cv_attr = 'cv'+str(i)