diff --git a/.github/workflows/auto-assign-pr.yml b/.github/workflows/auto-assign-pr.yml index b5cc747..009036b 100644 --- a/.github/workflows/auto-assign-pr.yml +++ b/.github/workflows/auto-assign-pr.yml @@ -8,12 +8,12 @@ jobs: auto-assign: runs-on: ubuntu-latest permissions: - pull-request: write + pull-requests: write steps: - name: 'Auto assign PR' uses: pozil/auto-assign-issue@v1 with: - repo-token: ${{ secrets.MY_PERSONAL_TOKEN }} + repo-token: ${{ secrets.GITHUB_TOKEN }} assignees: willeagren numOfAssignee: 1 allowSelfAssign: true diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 482a730..bec7a7b 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -18,12 +18,14 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.8 + - name: Build run: | python3 -m pip install --upgrade pip python3 -m venv venv + python3 -m pip install -r requirements.txt source venv/bin/activate - make + make build build-macos: runs-on: macos-latest @@ -33,10 +35,11 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.8 + - name: Build run: | python3 -m pip install --upgrade pip python3 -m venv venv + python3 -m pip install -r requirements.txt source venv/bin/activate - make - + make build diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 034cd91..111c79a 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -2,13 +2,12 @@ name: Unit tests on: push: - branches: [main] + branches: [ main ] pull_request: - branches: [main] - + branches: [ main ] jobs: - testsCPU: + unit-tests-CPU: name: CPU Tests runs-on: ubuntu-latest @@ -23,10 +22,12 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.8" + - name: Dependencies, build and test run: | python3 -m pip install --upgrade pip python3 -m venv venv + python3 -m pip install -r requirements.txt source venv/bin/activate - make + make build python3 -m unittest diff --git a/Makefile b/Makefile index 177c820..18d6cba 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ SHELL := /bin/bash build: - python3 -m pip install -r requirements.txt cd leafrs && maturin develop --release clean: diff --git a/leaf/functions/_binary_ops.py b/leaf/functions/_binary_ops.py index 081b094..741eaa8 100644 --- a/leaf/functions/_binary_ops.py +++ b/leaf/functions/_binary_ops.py @@ -22,7 +22,7 @@ # SOFTWARE. # # File created: 2022-11-05 -# Last updated: 2023-01-13 +# Last updated: 2023-01-14 # import numpy as np @@ -37,7 +37,7 @@ def _unbroadcast(arr, shape) -> np.ndarray: class Add(Function): def forward(self, x, y) -> np.ndarray: self.save_for_backward(x.shape, y.shape) - return rs.add(x, y) + return x + y def backward(self, grad) -> Tuple[np.ndarray]: xshape, yshape, = self.saved_tensors diff --git a/leaf/functions/function.py b/leaf/functions/function.py index 679c9f5..85bb3ed 100644 --- a/leaf/functions/function.py +++ b/leaf/functions/function.py @@ -22,7 +22,7 @@ # SOFTWARE. # # File created: 2022-11-05 -# Last updated: 2023-01-13 +# Last updated: 2023-01-14 # import numpy as np @@ -39,11 +39,11 @@ def _verify_tensors(*tensors) -> List[np.ndarray]: return [_extract_data(t) for t in tensors] def _extract_data(tensor) -> np.ndarray: - """ TEMPORARY!!!! EDIT this function """ return tensor.data class Function(object): - """ Definition and impelmentation of the Function class. + """ + Definition and impelmentation of the Function class. Parameters ---------- @@ -72,7 +72,8 @@ def backward(self, *args, **kwargs): @classmethod def apply(cls, *tensors) -> Tensor: - """ This classmethod constructs a Function, also referred to as a context, when + """ + This classmethod constructs a Function, also referred to as a context, when a tensor invokes an operation. As such, the tensor initially invoking the call, self, is represented as part of the *tensors arg together with any optional tensors that are to be part of the context. diff --git a/leaf/nn/base.py b/leaf/nn/base.py index 890b4fb..eb8f514 100644 --- a/leaf/nn/base.py +++ b/leaf/nn/base.py @@ -22,13 +22,14 @@ # SOFTWARE. # # File created: 2022-11-01 -# Last updated: 2023-01-13 +# Last updated: 2023-01-14 # from leaf import Tensor class Module(object): - """ Parent class for the neural network building blocks, i.e. so called Modules. + """ + Parent class for the neural network building blocks, i.e. so called Modules. They each define specific forward pass functionality based on their needs that is invoked by calling the module object with the input tensor. No __init__ method is defined for parent class, please see respective implementations for specific @@ -45,13 +46,23 @@ def forward(self, input_, *args, **kwargs) -> Tensor: ) def parameters(self) -> list: - pass + params = [] + for attr in self.__dict__.values(): + if isinstance(attr, Tensor): + params.extend([attr] if attr.requires_grad else []) + if isinstance(attr, (tuple, list)): + params.extend([p for p in attr if p.requires_grad]) + if isinstance(attr, (Module, Sequential)): + # recursive call to find learnable parameter modules + params.extend([p for p in attr.parameters() if p.requires_grad]) + class Sequential(object): - """ A high-level wrapper for the module object, simplifies the forward pass when + """ + A high-level wrapper for the module object, simplifies the forward pass when multiple operations are needed to perform in order. Follows the same naming - convention as the main module object, namely, __call__() forward() and parameters(), - that make up the API for neural networks. + convention as the main module object with `__call__`, `forward` and + `parameters`, that make up the API for neural networks. Parameters ---------- diff --git a/leaf/nn/linear.py b/leaf/nn/linear.py index 2e47b56..f0e8964 100644 --- a/leaf/nn/linear.py +++ b/leaf/nn/linear.py @@ -22,14 +22,15 @@ # SOFTWARE. # # File created: 2022-11-18 -# Last updated: 2023-01-13 +# Last updated: 2023-01-14 # from leaf import Tensor from leaf.nn import Module class Linear(Module): - """ Linear layer implementation as a neural network module. + """ + Linear layer implementation as a neural network module. This module requires input to be 2D tensor, allowing standard matmul op. Parameters @@ -49,7 +50,8 @@ def __init__(self, fan_in, fan_out, bias=True) -> None: self._bias = Tensor.uniform(fan_out, requires_grad=True) if bias else None def forward(self, input_) -> Tensor: - """ Propagate data through a linear layer, performing linear transform operation. + """ + Propagate data through a linear layer, performing the linear transform operation. Parameters ---------- diff --git a/leaf/tensor.py b/leaf/tensor.py index 3bdc36e..7a8d34c 100644 --- a/leaf/tensor.py +++ b/leaf/tensor.py @@ -22,7 +22,7 @@ # SOFTWARE. # # File created: 2022-11-01 -# Last updated: 2023-01-13 +# Last updated: 2023-01-14 # from __future__ import annotations @@ -30,7 +30,8 @@ import numpy as np class Tensor(object): - """ Definition and implementation of the Tensor class. + """ + Definition and implementation of the Tensor class. Parameters ---------- @@ -103,7 +104,8 @@ def normal(cls, *shape, loc=0.0, scale=1.0, **kwargs): ) def detach(self) -> Tensor: - """ Create a copy of the current tensor that is not part of the + """ + Create a copy of the current tensor that is not part of the dynamic DAG. As such, the new tensor does not, and can not, require grad because it is not part of any context nor DAG. Subsequentially move the tensor to cpu device, if it was on other. diff --git a/leafrs/src/lib.rs b/leafrs/src/lib.rs index 52e0fa3..85ba220 100644 --- a/leafrs/src/lib.rs +++ b/leafrs/src/lib.rs @@ -22,13 +22,11 @@ // SOFTWARE. // // File created: 2022-11-01 -// Last updated: 2023-01-13 +// Last updated: 2023-01-14 // -use ndarray; use numpy::{ IntoPyArray, - PyArray1, PyArray2, PyArrayDyn, PyReadonlyArrayDyn, @@ -41,31 +39,19 @@ use pyo3::prelude::{ Python }; -mod rust_fn { - use ndarray::{arr1, Array1, Array2, ArrayD}; +// Declare the scope/module for function specific implementations, called from Python +// through the created maturin bindings of the `PyO3` crate. Inside this scope +// we operate on the dynamic arrays specified by the `ndarray` crate. The `numpy` +// crate used in the outer scope is only used as API for the native C bindings. +#[allow(non_snake_case)] +mod RUST_BACKEND { + use ndarray::{Array2, ArrayD}; use ndarray::prelude::*; use numpy::ndarray::{ArrayViewD, ArrayView4}; - use ordered_float::OrderedFloat; - pub fn max_min(x: &ArrayViewD<'_, f32>) -> Array1 { - if x.len() == 0 { return arr1(&[]); } - let max_val = x - .iter() - .map(|a| OrderedFloat(*a)) - .max() - .expect("Error calculating max value.") - .0; - let min_val = x - .iter() - .map(|a| OrderedFloat(*a)) - .min() - .expect("Error calculating min value.") - .0; - let result_array = arr1(&[max_val, min_val]); - result_array - } - - pub fn rusum(x: &ArrayView4<'_, f32>) -> Array2 { + // BACKEND FUNC, only used in Python vs Rust performance example. + // Calculates the sum of a 4 dimensional f32 matrix. + pub fn _example_matrix_sum(x: &ArrayView4<'_, f32>) -> Array2 { let xshape = x.shape(); let mut result_array = Array2::zeros((xshape[0], xshape[1])); for h in 0..xshape[2] { @@ -76,49 +62,58 @@ mod rust_fn { result_array } - pub fn add(x: &ArrayViewD<'_, f32>, y: &ArrayViewD<'_, f32>) -> ArrayD { + // BACKEND FUNC, performs addition on two ndarrays. + pub fn add( + x: &ArrayViewD<'_, f32>, + y: &ArrayViewD<'_, f32> + ) -> ArrayD { x + y } + + // BACKEND FUNC, performs subtraction on two ndarrays. + // z = y + x + pub fn sub( + x: &ArrayViewD<'_, f32>, + y: &ArrayViewD<'_, f32> + ) -> ArrayD { + x - y + } } #[pymodule] fn leafrs(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - #[pyfn(m)] - fn max_min<'py>( - py: Python<'py>, - x: PyReadonlyArrayDyn - ) -> &'py PyArray1 { - let array = x.as_array(); - let result_array = rust_fn::max_min(&array); - result_array.into_pyarray(py) - } + // Unary operator, calculates the sum of a 4D matrix/tensor. + // ONLY USED IN EXAMPLE SCRIPT COMPARING PYTHON AND RUST SPEED! #[pyfn(m)] - fn rusum<'py>( + fn example_matrix_sum<'py>( py: Python<'py>, x: PyReadonlyArray4 ) -> &'py PyArray2 { - let array = x.as_array(); - let rustsum = rust_fn::rusum(&array); - rustsum.into_pyarray(py) + let arr = x.as_array(); + let sum = RUST_BACKEND::_example_matrix_sum(&arr); + sum.into_pyarray(py) } + // Binary operator, performs addition on two ndarrays. #[pyfn(m)] - fn eye<'py>( + fn add<'py>( py: Python<'py>, - size: usize - ) -> &PyArray2 { - let array = ndarray::Array::eye(size); - array.into_pyarray(py) + x: PyReadonlyArrayDyn, + y: PyReadonlyArrayDyn + ) -> &'py PyArrayDyn { + let result = RUST_BACKEND::add(&x.as_array(), &y.as_array()); + result.into_pyarray(py) } + // Binary operator, performs subtraction on two ndarrays. #[pyfn(m)] - fn add<'py>( + fn sub<'py>( py: Python<'py>, x: PyReadonlyArrayDyn, y: PyReadonlyArrayDyn ) -> &'py PyArrayDyn { - let result = rust_fn::add(&x.as_array(), &y.as_array()); + let result = RUST_BACKEND::sub(&x.as_array(), &y.as_array()); result.into_pyarray(py) } diff --git a/pyvsrust.py b/pyvsrust.py index ab11499..a1de430 100644 --- a/pyvsrust.py +++ b/pyvsrust.py @@ -22,7 +22,7 @@ # SOFTWARE. # # File created: 2022-11-01 -# Last updated: 2023-01-13 +# Last updated: 2023-01-14 # import leafrs as rs @@ -50,7 +50,7 @@ def info(s, strftime=True): info('Running performance test Python vs Rust API\n') info(f'Generating test data with shape: {dimensions}...') -data = np.random.rand(*dimensions) +data = np.random.rand(*dimensions).astype(np.float32) info('OK\n', strftime=False) pysum = 0 @@ -66,7 +66,7 @@ def info(s, strftime=True): info('Starting Rust API job...') t_rustart = time.time() -rusum = rs.rusum(data).sum() +rusum = rs.example_matrix_sum(data).sum() t_ru = time.time() - t_rustart info('OK\n', strftime=False) info(f'Time: {t_ru:.3f} seconds\n') diff --git a/requirements.txt b/requirements.txt index b7224fd..585bbed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ maturin==0.13.7 numpy==1.23.4 tomli==2.0.1 -torch>=1.7.0+cpu +torch>=1.7.0+cpu \ No newline at end of file diff --git a/scripts/cc.ps1 b/scripts/cc.ps1 new file mode 100644 index 0000000..c9caa46 --- /dev/null +++ b/scripts/cc.ps1 @@ -0,0 +1 @@ +Get-ChildItem -Include __pycache__ -Recurse -force | Remove-Item -Force -Recurse diff --git a/cc.sh b/scripts/cc.sh old mode 100755 new mode 100644 similarity index 100% rename from cc.sh rename to scripts/cc.sh