Memory-mapped file handling and streaming API

.dockerignore

@@ -0,0 +1,27 @@
+# Build artifacts
+*.o
+*.so
+*.pyc
+imctermite
+main.cpp.cpp
+
+# Python build
+python/build/
+python/dist/
+python/*.so
+python/*.cpp
+python/lib/
+python/LICENSE
+python/README.md
+python/*.egg-info/
+__pycache__/
+
+# Git and editor
+.git/
+.venv/
+*.swp
+*.swo
+*~
+
+# Test outputs
+.pytest_cache/

.github/workflows/pypi-deploy.yml

@@ -8,15 +8,19 @@ on:
 
 jobs:
 
+  test:
+    uses: ./.github/workflows/test.yml
+
   build_setup:
     name: Prepare environment for wheel builds
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
+    needs: [test]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Prepare wheel build
         run: make -C python/ setup
       - name: Store wheel configuration files
-        uses: actions/upload-artifact@v4.6.0
+        uses: actions/upload-artifact@v4
         with:
           name: wheel-config
           path: python/
@@ -32,42 +36,41 @@ jobs:
         os: [ubuntu-latest, windows-latest]
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
       - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel==2.1.2
+        run: python -m pip install cibuildwheel
       - name: Get wheel configuration files
-        uses: actions/download-artifact@v4.1.7
+        uses: actions/download-artifact@v4
         with:
           name: wheel-config
           path: python/
       - name: Build wheels
         run: python -m cibuildwheel --output-dir wheelhouse
         working-directory: python/
       - name: Store binary wheels
-        uses: actions/upload-artifact@v4.6.0
+        uses: actions/upload-artifact@v4
         with:
           name: binary-wheels-${{matrix.os}}-${{ strategy.job-index }}
           path: python/wheelhouse/*.whl
 
   build_sdist:
     name: Build source distribution
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     needs: [build_setup]
     steps:
-      - uses: actions/checkout@v2
-      - name: Install cython
-        run: python -m pip install cython==0.29.24
+      - uses: actions/checkout@v4
+      - name: Install build tools
+        run: python -m pip install build
       - name: Get wheel configuration files
-        uses: actions/download-artifact@v4.1.7
+        uses: actions/download-artifact@v4
         with:
           name: wheel-config
           path: python/
       - name: Build sdist
-        run: python setup.py sdist
+        run: python -m build --sdist
         working-directory: python/
       - name: Store source wheels
-        uses: actions/upload-artifact@v4.6.0
+        uses: actions/upload-artifact@v4
         with:
           name: source-wheels
           path: python/dist/*.tar.gz
@@ -77,17 +80,17 @@ jobs:
 
   upload_pypi:
     name: Upload wheels to PyPI
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     needs: [build_wheels, build_sdist]
 
     steps:
       - name: Get source wheels
-        uses: actions/download-artifact@v4.1.7
+        uses: actions/download-artifact@v4
         with:
           name: source-wheels
           path: dist/
       - name: Get binary wheels
-        uses: actions/download-artifact@v4.1.7
+        uses: actions/download-artifact@v4
         with:
           path: dist/
           pattern: binary-wheels-*

.github/workflows/test.yml

@@ -0,0 +1,35 @@
+name: Run Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  workflow_call:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install test dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest
+
+    - name: Build and Test
+      shell: bash
+      run: |
+        make test

.gitignore

@@ -43,3 +43,6 @@ python/*.soc
 python/lib/
 python/*.cpp
 python/wheelhouse/
+
+__pycache__/
+.pytest_cache/

Dockerfile

@@ -1,21 +1,22 @@
-
-FROM debian:bullseye-20210111
+FROM debian:bullseye
 
 USER root
 
 RUN apt-get update && apt-get install -y \
     build-essential git vim \
     python3 python3-pip
-RUN python3 -m pip install cython
+RUN python3 -m pip install cython pytest
+RUN ln -s /usr/bin/python3 /usr/bin/python
 
 RUN g++ -v
 
-COPY ./ /IMCtermite/
+WORKDIR /IMCtermite
+COPY ./ .
 
 # install CLI tool
-RUN cd /IMCtermite && ls -lh && make install && ls -lh /usr/local/bin/imctermite
+RUN make install
 
 # install Python module
-RUN cd /IMCtermite && ls -lh && make cython-install
+RUN make python-build
 
 CMD ["sleep","infinity"]

README.md

@@ -1,8 +1,10 @@
 
 [![LICENSE](https://img.shields.io/github/license/RecordEvolution/IMCtermite)](https://img.shields.io/github/license/RecordEvolution/IMCtermite)
 [![STARS](https://img.shields.io/github/stars/RecordEvolution/IMCtermite)](https://img.shields.io/github/stars/RecordEvolution/IMCtermite)
+![Tests](https://github.com/RecordEvolution/IMCtermite/actions/workflows/test.yml/badge.svg)
 ![CI Build Wheel](https://github.com/RecordEvolution/IMCtermite/actions/workflows/pypi-deploy.yml/badge.svg?branch=&event=push)
 [![PYPI](https://img.shields.io/pypi/v/IMCtermite.svg)](https://pypi.org/project/imctermite/)
+[![Python Version](https://img.shields.io/pypi/pyversions/imctermite)](https://pypi.org/project/imctermite/)
 
 # IMCtermite
 
@@ -27,6 +29,7 @@ Python module to integrate the _.raw_  format into any ETL workflow.
 * [File format](#Fileformat)
 * [Build and Installation](#Installation)
 * [Usage and Examples](#Usage)
+* [Testing](#Testing)
 * [References](#References)
 
 ## File format
@@ -149,10 +152,13 @@ python3 -m pip install imctermite
 ```
 
 which provides binary wheels for multiple architectures on _Windows_ and _Linux_
-and most _Python 3.x_ distributions. However, if your platform/architecture is
-not supported you can still compile the source distribution yourself, which
-requires _python3_setuptools_ and an up-to-date compiler supporting C++11
-standard (e.g. _gcc version >= 10.2.0_).
+and most _Python 3.x_ distributions. **Note:** Starting from version 3.0.0, 
+imctermite requires numpy as a dependency, which will be automatically 
+installed if not already present.
+
+However, if your platform/architecture is not supported you can still compile 
+the source distribution yourself, which requires _python3_setuptools_, _numpy_, 
+and an up-to-date compiler supporting C++11 standard (e.g. _gcc version >= 10.2.0_).
 
 ## Usage
 
@@ -194,17 +200,17 @@ of it by passing a _raw_ file to the constructor:
 ```Python
 import imctermite
 
-imcraw = imctermite.imctermite(b"sample/sampleA.raw")
+imcraw = imctermite.imctermite("sample/sampleA.raw")
 ```
 
 An example of how to create an instance and obtain the list of channels is:
 
 ```Python
-import IMCtermite
+import imctermite
 
 # declare and initialize instance of "imctermite" by passing a raw-file
 try :
-    imcraw = IMCtermite.imctermite(b"samples/sampleA.raw")
+    imcraw = imctermite.imctermite("samples/sampleA.raw")
 except RuntimeError as e :
     print("failed to load/parse raw-file: " + str(e))
 
@@ -217,6 +223,16 @@ A more complete [example](python/examples/usage.py), including the methods for
 obtaining the channels, i.a. their data and/or directly printing them to files,
 can be found in the `python/examples` folder.
 
+### Chunked NumPy export (fast path)
+
+For large files, you can iterate over channel data in chunks as NumPy arrays. This avoids creating large Python lists and allows for streaming processing (e.g. writing to Parquet). See [`python/examples/usage_numpy_chunks.py`](python/examples/usage_numpy_chunks.py) for a complete example.
+
+## Testing
+
+Run end-to-end tests: `make test`
+
+See [tests/README.md](tests/README.md) for details.
+
 ## References
 
 ### IMC

lib/imc_block.hpp

@@ -34,7 +34,8 @@ namespace imc
 
     // name and buffer of associated raw file
     std::string raw_file_;
-    const std::vector<unsigned char>* buffer_;
+    const unsigned char* buffer_;
+    size_t buffer_size_;
 
     // offset of first/last byte of parameters in block (separated by ch_sep_)
     // w.r.t. to first byte of block (=0)
@@ -44,7 +45,7 @@ namespace imc
 
     // constructor
     block(key thekey, unsigned long int begin, unsigned long int end,
-                      std::string raw_file, const std::vector<unsigned char>* buffer):
+                      std::string raw_file, const unsigned char* buffer, size_t buffer_size):
       thekey_(thekey), uuid_(std::to_string(begin))
     {
       if ( !imc::check_key(thekey) ) throw std::logic_error("unknown key");
@@ -56,14 +57,15 @@ namespace imc
       }
       raw_file_ = raw_file;
       buffer_ = buffer;
+      buffer_size_ = buffer_size;
 
       // make sure "end_" does not exceed buffer size due to invalid "length" parameter of block
-      if ( end_ > buffer_->size() )
+      if ( end_ > buffer_size_ )
       {
         std::cout<<"WARNING: invalid length parameter in "<<thekey_.name_<<"-block "
-                 <<"(block-end:"<<end_<<",buffer-size:"<<buffer_->size()<<")"
+                 <<"(block-end:"<<end_<<",buffer-size:"<<buffer_size_<<")"
                  <<" => resetting block-end to buffer-size\n";
-        end_ = (unsigned long int)(buffer_->size());
+        end_ = (unsigned long int)(buffer_size_);
       }
 
       try {
@@ -86,7 +88,7 @@ namespace imc
       for ( unsigned long int b = begin_;
         b < end_ && ( ! (thekey_.name_== "CS") || count < 4 ); b++ )
       {
-        if ( buffer_->at(b) == imc::ch_sep_ )
+        if ( buffer_[b] == imc::ch_sep_ )
         {
           // define range of parameter with first byte = ch_sep_
           parameters_.push_back(imc::parameter(b,b));
@@ -124,8 +126,8 @@ namespace imc
       {
         throw std::logic_error("inconsistent parameter offsets");
       }
-      std::vector<unsigned char> parambuff(buffer_->begin()+begin_+param.begin(),
-                                           buffer_->begin()+begin_+param.end());
+      std::vector<unsigned char> parambuff(buffer_+begin_+param.begin(),
+                                           buffer_+begin_+param.end());
       return parambuff;
     }
 
@@ -140,7 +142,7 @@ namespace imc
       std::string prm("");
       for ( unsigned long int i = param.begin()+1; i <= param.end(); i++ )
       {
-        prm.push_back( (char)((*buffer_)[i]) );
+        prm.push_back( (char)(buffer_[i]) );
       }
       return prm;
     }
@@ -163,7 +165,7 @@ namespace imc
         <<std::setw(width)<<std::left<<"begin:"<<begin_<<"\n"
         <<std::setw(width)<<std::left<<"end:"<<end_<<"\n"
         <<std::setw(width)<<std::left<<"rawfile:"<<raw_file_<<"\n"
-        <<std::setw(width)<<std::left<<"buffersize:"<<buffer_->size()<<"\n"
+        <<std::setw(width)<<std::left<<"buffersize:"<<buffer_size_<<"\n"
         <<std::setw(width)<<std::left<<"parameters:"<<prsstr<<"\n";
 
       return ss.str();