Refactor/rust

.github/workflows/auto-assign-pr.yml

@@ -8,12 +8,12 @@ jobs:
   auto-assign:
     runs-on: ubuntu-latest
     permissions:
-      pull-request: write
+      pull-requests: write
     steps:
     - name: 'Auto assign PR'
       uses: pozil/auto-assign-issue@v1
       with:
-        repo-token: ${{ secrets.MY_PERSONAL_TOKEN }}
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
         assignees: willeagren
         numOfAssignee: 1
         allowSelfAssign: true

.github/workflows/builds.yml

@@ -18,12 +18,14 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
+
     - name: Build
       run: |
         python3 -m pip install --upgrade pip
         python3 -m venv venv
+        python3 -m pip install -r requirements.txt
         source venv/bin/activate
-        make
+        make build
 
   build-macos:
     runs-on: macos-latest
@@ -33,10 +35,11 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
+
     - name: Build
       run: |
         python3 -m pip install --upgrade pip
         python3 -m venv venv
+        python3 -m pip install -r requirements.txt
         source venv/bin/activate
-        make
-
+        make build

.github/workflows/unittests.yml

@@ -2,13 +2,12 @@ name: Unit tests
 
 on:
   push:
-    branches: [main]
+    branches: [ main ]
   pull_request:
-    branches: [main]
-
+    branches: [ main ]
 
 jobs:
-  testsCPU:
+  unit-tests-CPU:
     name: CPU Tests
     runs-on: ubuntu-latest
 
@@ -23,10 +22,12 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: "3.8"
+
     - name: Dependencies, build and test
       run: |
         python3 -m pip install --upgrade pip
         python3 -m venv venv
+        python3 -m pip install -r requirements.txt
         source venv/bin/activate
-        make
+        make build
         python3 -m unittest

Makefile

@@ -1,7 +1,6 @@
 SHELL := /bin/bash
 
 build:
-	python3 -m pip install -r requirements.txt
 	cd leafrs && maturin develop --release
 
 clean:

leaf/functions/_binary_ops.py

@@ -22,7 +22,7 @@
 # SOFTWARE.
 #
 # File created: 2022-11-05
-# Last updated: 2023-01-13
+# Last updated: 2023-01-14
 #
 
 import numpy as np
@@ -37,7 +37,7 @@ def _unbroadcast(arr, shape) -> np.ndarray:
 class Add(Function):
     def forward(self, x, y) -> np.ndarray:
         self.save_for_backward(x.shape, y.shape)
-        return rs.add(x, y)
+        return x + y
 
     def backward(self, grad) -> Tuple[np.ndarray]:
         xshape, yshape, = self.saved_tensors

leaf/functions/function.py

@@ -22,7 +22,7 @@
 # SOFTWARE.
 #
 # File created: 2022-11-05
-# Last updated: 2023-01-13
+# Last updated: 2023-01-14
 #
 
 import numpy as np
@@ -39,11 +39,11 @@ def _verify_tensors(*tensors) -> List[np.ndarray]:
     return [_extract_data(t) for t in tensors]
 
 def _extract_data(tensor) -> np.ndarray:
-    """ TEMPORARY!!!! EDIT this function """
     return tensor.data
 
 class Function(object):
-    """ Definition and impelmentation of the Function class.
+    """
+    Definition and impelmentation of the Function class.
 
     Parameters
     ----------
@@ -72,7 +72,8 @@ def backward(self, *args, **kwargs):
 
     @classmethod
     def apply(cls, *tensors) -> Tensor: 
-        """ This classmethod constructs a Function, also referred to as a context, when 
+        """
+        This classmethod constructs a Function, also referred to as a context, when 
         a tensor invokes an operation. As such, the tensor initially invoking the 
         call, self, is represented as part of the *tensors arg together with any
         optional tensors that are to be part of the context. 

leaf/nn/base.py

@@ -22,13 +22,14 @@
 # SOFTWARE.
 #
 # File created: 2022-11-01
-# Last updated: 2023-01-13
+# Last updated: 2023-01-14
 #
 
 from leaf import Tensor
 
 class Module(object):
-    """ Parent class for the neural network building blocks, i.e. so called Modules.
+    """
+    Parent class for the neural network building blocks, i.e. so called Modules.
     They each define specific forward pass functionality based on their needs that 
     is invoked by calling the module object with the input tensor. No __init__ method
     is defined for parent class, please see respective implementations for specific
@@ -45,13 +46,23 @@ def forward(self, input_, *args, **kwargs) -> Tensor:
         )
 
     def parameters(self) -> list:
-        pass
+        params = []
+        for attr in self.__dict__.values():
+            if isinstance(attr, Tensor):
+                params.extend([attr] if attr.requires_grad else [])
+            if isinstance(attr, (tuple, list)):
+                params.extend([p for p in attr if p.requires_grad])
+            if isinstance(attr, (Module, Sequential)):
+                # recursive call to find learnable parameter modules
+                params.extend([p for p in attr.parameters() if p.requires_grad])
+
 
 class Sequential(object):
-    """ A high-level wrapper for the module object, simplifies the forward pass when
+    """
+    A high-level wrapper for the module object, simplifies the forward pass when
     multiple operations are needed to perform in order. Follows the same naming
-    convention as the main module object, namely, __call__() forward() and parameters(),
-    that make up the API for neural networks.
+    convention as the main module object with `__call__`, `forward` and 
+    `parameters`, that make up the API for neural networks.
 
     Parameters
     ----------

leaf/nn/linear.py

@@ -22,14 +22,15 @@
 # SOFTWARE.
 #
 # File created: 2022-11-18
-# Last updated: 2023-01-13
+# Last updated: 2023-01-14
 #
 
 from leaf import Tensor
 from leaf.nn import Module
 
 class Linear(Module):
-    """ Linear layer implementation as a neural network module.
+    """ 
+    Linear layer implementation as a neural network module.
     This module requires input to be 2D tensor, allowing standard matmul op.
 
     Parameters
@@ -49,7 +50,8 @@ def __init__(self, fan_in, fan_out, bias=True) -> None:
         self._bias = Tensor.uniform(fan_out, requires_grad=True) if bias else None
 
     def forward(self, input_) -> Tensor:
-        """ Propagate data through a linear layer, performing linear transform operation.
+        """
+        Propagate data through a linear layer, performing the linear transform operation.
 
         Parameters
         ----------

leaf/tensor.py

@@ -22,15 +22,16 @@
 # SOFTWARE.
 #
 # File created: 2022-11-01
-# Last updated: 2023-01-13
+# Last updated: 2023-01-14
 #
 
 from __future__ import annotations
 from typing import Union, Tuple, List
 import numpy as np
 
 class Tensor(object):
-    """ Definition and implementation of the Tensor class.
+    """
+    Definition and implementation of the Tensor class.
 
     Parameters
     ----------
@@ -103,7 +104,8 @@ def normal(cls, *shape, loc=0.0, scale=1.0, **kwargs):
         )
 
     def detach(self) -> Tensor:
-        """ Create a copy of the current tensor that is not part of the 
+        """
+        Create a copy of the current tensor that is not part of the 
         dynamic DAG. As such, the new tensor does not, and can not,
         require grad because it is not part of any context nor DAG.
         Subsequentially move the tensor to cpu device, if it was on other.

leafrs/src/lib.rs

@@ -22,13 +22,11 @@
 // SOFTWARE.
 //
 // File created: 2022-11-01
-// Last updated: 2023-01-13
+// Last updated: 2023-01-14
 //
 
-use ndarray;
 use numpy::{
     IntoPyArray,
-    PyArray1,
     PyArray2,
     PyArrayDyn,
     PyReadonlyArrayDyn,
@@ -41,31 +39,19 @@ use pyo3::prelude::{
     Python
 };
 
-mod rust_fn {
-    use ndarray::{arr1, Array1, Array2, ArrayD};
+// Declare the scope/module for function specific implementations, called from Python
+// through the created maturin bindings of the `PyO3` crate. Inside this scope
+// we operate on the dynamic arrays specified by the `ndarray` crate. The `numpy`
+// crate used in the outer scope is only used as API for the native C bindings.
+#[allow(non_snake_case)]
+mod RUST_BACKEND {
+    use ndarray::{Array2, ArrayD};
     use ndarray::prelude::*;
     use numpy::ndarray::{ArrayViewD, ArrayView4};
-    use ordered_float::OrderedFloat;
 
-    pub fn max_min(x: &ArrayViewD<'_, f32>) -> Array1<f32> {
-        if x.len() == 0 { return arr1(&[]); }
-        let max_val = x
-            .iter()
-            .map(|a| OrderedFloat(*a))
-            .max()
-            .expect("Error calculating max value.")
-            .0;
-        let min_val = x
-            .iter()
-            .map(|a| OrderedFloat(*a))
-            .min()
-            .expect("Error calculating min value.")
-            .0;
-        let result_array = arr1(&[max_val, min_val]);
-        result_array
-    }
-
-    pub fn rusum(x: &ArrayView4<'_, f32>) -> Array2<f32> {
+    // BACKEND FUNC, only used in Python vs Rust performance example.
+    // Calculates the sum of a 4 dimensional f32 matrix.
+    pub fn _example_matrix_sum(x: &ArrayView4<'_, f32>) -> Array2<f32> {
         let xshape = x.shape();
         let mut result_array = Array2::zeros((xshape[0], xshape[1]));
         for h in 0..xshape[2] {
@@ -76,49 +62,58 @@ mod rust_fn {
         result_array
     }
 
-    pub fn add(x: &ArrayViewD<'_, f32>, y: &ArrayViewD<'_, f32>) -> ArrayD<f32> {
+    // BACKEND FUNC, performs addition on two ndarrays.
+    pub fn add(
+        x: &ArrayViewD<'_, f32>,
+        y: &ArrayViewD<'_, f32>
+    ) -> ArrayD<f32> {
         x + y
     }
+
+    // BACKEND FUNC, performs subtraction on two ndarrays.
+    // z = y + x
+    pub fn sub(
+        x: &ArrayViewD<'_, f32>,
+        y: &ArrayViewD<'_, f32>
+    ) -> ArrayD<f32> {
+        x - y
+    }
 }
 
 #[pymodule]
 fn leafrs(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
-    #[pyfn(m)]
-    fn max_min<'py>(
-        py: Python<'py>,
-        x: PyReadonlyArrayDyn<f32>
-    ) -> &'py PyArray1<f32> {
-        let array = x.as_array();
-        let result_array = rust_fn::max_min(&array);
-        result_array.into_pyarray(py)
-    }
 
+    // Unary operator, calculates the sum of a 4D matrix/tensor.
+    // ONLY USED IN EXAMPLE SCRIPT COMPARING PYTHON AND RUST SPEED!
     #[pyfn(m)]
-    fn rusum<'py>(
+    fn example_matrix_sum<'py>(
         py: Python<'py>,
         x: PyReadonlyArray4<f32>
     ) -> &'py PyArray2<f32> {
-        let array = x.as_array();
-        let rustsum = rust_fn::rusum(&array);
-        rustsum.into_pyarray(py)
+        let arr = x.as_array();
+        let sum = RUST_BACKEND::_example_matrix_sum(&arr);
+        sum.into_pyarray(py)
     }
 
+    // Binary operator, performs addition on two ndarrays.
     #[pyfn(m)]
-    fn eye<'py>(
+    fn add<'py>(
         py: Python<'py>,
-        size: usize
-    ) -> &PyArray2<f32> {
-        let array = ndarray::Array::eye(size);
-        array.into_pyarray(py)
+        x: PyReadonlyArrayDyn<f32>,
+        y: PyReadonlyArrayDyn<f32>
+    ) -> &'py PyArrayDyn<f32> {
+        let result = RUST_BACKEND::add(&x.as_array(), &y.as_array());
+        result.into_pyarray(py)
     }
 
+    // Binary operator, performs subtraction on two ndarrays.
     #[pyfn(m)]
-    fn add<'py>(
+    fn sub<'py>(
         py: Python<'py>,
         x: PyReadonlyArrayDyn<f32>,
         y: PyReadonlyArrayDyn<f32>
     ) -> &'py PyArrayDyn<f32> {
-        let result = rust_fn::add(&x.as_array(), &y.as_array());
+        let result = RUST_BACKEND::sub(&x.as_array(), &y.as_array());
         result.into_pyarray(py)
     }