Unittests

.github/workflows/dockerhub-main-push.yml

@@ -1,8 +1,10 @@
 name: DockerHub Main Push
 
 on:
-  push:
-    branches: [ "main" ]
+  workflow_run:
+    workflows: ["Unittest Main Push"]
+    types:
+      - completed
 
 jobs:
   build:

.github/workflows/unittest-dev.yml

@@ -0,0 +1,30 @@
+name: Unittest Dev
+
+on:
+  pull_request:
+    branches: [ "dev" ]
+  push:
+    branches: [ "dev" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Test with unittest
+      run: |
+        python -m unittest discover ./webscrapers/unittests/utilities

.github/workflows/unittest-main-pr.yml

@@ -0,0 +1,28 @@
+name: Unittest Main Pull Request
+
+on:
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Test with unittest
+      run: |
+        python -m unittest discover ./webscrapers/unittests/utilities

.github/workflows/unittest-main-push.yml

@@ -0,0 +1,28 @@
+name: Unittest Main Push
+
+on:
+  push:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Test with unittest
+      run: |
+        python -m unittest discover ./webscrapers/unittests/utilities

model/arch/crop_image.py

@@ -0,0 +1,39 @@
+import numpy as np
+from beartype import beartype
+from typing import Union
+
+@beartype
+def crop_image(
+    image_array:np.array,
+    pt1_wh:Union[tuple,list],
+    pt2_wh:Union[tuple,list]
+    ) -> np.array:
+    """Crops an image array to specified combination of two diagonal points
+
+    Parameters
+    ---------
+    image : numpy.array
+        The numpy image array to crop
+    pt1_wh : list, tuple
+        Diagonal point 1 coordinates for cropping the image
+    pt2_wh : list, tuple
+        Diagonal point 2 coordinates for cropping the image
+
+    Returns
+    -------
+    numpy.array
+        The cropped image array
+    """
+    # extract out diagonal cropping points
+    (pt1_w, pt1_h) = pt1_wh
+    (pt2_w, pt2_h) = pt2_wh
+    # group height and width points
+    wpts = (pt1_h, pt2_h)
+    hpts = (pt1_w, pt2_w)
+    # extract out image shape
+    n_image_dims = len(image_array.shape)
+    if n_image_dims == 3:
+        crop_image_array = image_array[min(hpts):max(hpts), min(wpts):max(wpts), :] 
+    else:
+        crop_image_array = image_array[min(hpts):max(hpts), min(wpts):max(wpts)] 
+    return crop_image_array

model/data_prep.py → model/arch/data_prep.py

@@ -2,24 +2,31 @@
 import os
 import sys
 import pickle
+import logging
 import numpy as np
 import pandas as pd
 from sklearn.model_selection import train_test_split
 
 # load custom modules
 sys.path.append(os.getcwd())
 import cons
-from utilities.load_image import load_image
-from utilities.greyscale_image import greyscale_image
-from utilities.pad_image import pad_image
-from utilities.plot_image import plot_image
-from utilities.resize_image import resize_image
+from arch.load_image import load_image
+from arch.greyscale_image import greyscale_image
+from arch.pad_image import pad_image
+from arch.resize_image import resize_image
 
-def data_prep(cons):
+def data_prep():
+    """
+    Data preparation pipeline for generating the model training, testing and validation data.
 
-    """"""
+    Parameters
+    ----------
 
-    print("Generating image file paths and classes ...")
+    Returns
+    -------
+    """
+
+    logging.info("Generating image file paths and classes ...")
 
 
     if False:
@@ -55,7 +62,7 @@ def data_prep(cons):
         # combine train and test files
         image_fpaths = {**train_image_fpaths, **test_image_fpaths}
 
-    print("Creating image dataframe ...")
+    logging.info("Creating image dataframe ...")
 
     # create list to hold image data
     image_data = []
@@ -78,7 +85,7 @@ def data_prep(cons):
     # convert image data object into a pandas dataframe
     image_dataframe = pd.DataFrame(image_data)
 
-    print("Padding images ...")
+    logging.info("Padding images ...")
 
     # find the largest image dimensions
     max_height = image_dataframe['image_shape'].apply(lambda x: x[0]).max() # height
@@ -90,15 +97,15 @@ def data_prep(cons):
     # apply padding to standardize all images shapes
     image_dataframe['pad_image_array'] = image_dataframe['image_array'].apply(lambda x: pad_image(x, pad_shape_wh = pad_shape))
 
-    print('Down sizing image ...')
+    logging.info('Down sizing image ...')
 
     # set down size shape
     downsize_shape = tuple([round(dim * 1/3) for dim in pad_shape])
 
     # apply resizing to downsize image shapes
     image_dataframe['pad_image_array'] = image_dataframe['pad_image_array'].apply(lambda x: resize_image(x, reshape_wh = downsize_shape))
 
-    print('Splitting train set ...')
+    logging.info('Splitting train set ...')
 
     # subset the output image data
     sub_cols = ['image_fpath', 'pad_image_array', 'target', 'dataset']
@@ -136,8 +143,6 @@ def data_prep(cons):
     with open(cons.test_data_pickle_fpath, 'wb') as handle:
         pickle.dump(test_data_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
 
-    return 0
-
 # if running as main programme
 if __name__ == "__main__":
 

model/arch/greyscale_image.py

@@ -0,0 +1,31 @@
+import numpy as np
+from beartype import beartype
+
+@beartype
+def greyscale_image(
+    rgb_image_array:np.array,
+    keep_3dim:bool=True
+    ) -> np.array:
+    """
+    Transforms an image array to greyscale
+
+    Parameters
+    ----------
+    rgb_image_array : numpy.array
+        The coloured numpy image array to transform to greyscale
+    keep_3dim : bool
+        Whether to keep the third dimension of the image array, default is True
+
+    Returns
+    -------
+    numpy.array
+        The transformed greyscale numpy image
+    """
+    # apply grey scale transformation
+    grey_image_array =  np.dot(rgb_image_array[:, :, :3], [0.2125, 0.7154, 0.0721])
+    # floor transformed pixel floats
+    gray_img = grey_image_array.astype(np.uint8)
+    # if keeping third dimension
+    if keep_3dim:
+        gray_img = gray_img[..., np.newaxis]
+    return gray_img

model/arch/keras_model.py

@@ -12,9 +12,9 @@
 sys.path.append(os.getcwd())
 import cons
 from utilities.plot_image import plot_image
-from arch.LeNet5 import LeNet5
+from keras.LeNet5 import LeNet5
 from fit_model import fit_model
-from plot_model import plot_model_fit
+from utilities.plot_model import plot_model_fit
 
 print('Loading data ...')
 

model/arch/load_image.py

@@ -0,0 +1,24 @@
+import numpy as np
+import cv2
+from beartype import beartype
+
+@beartype
+def load_image(image_fpath:str) -> np.array:
+    """
+    Loads an image file as an image array from disk
+
+    Parameters
+    ----------
+    image_fpath : str
+        The file path to the image file to load as an image array
+
+    Returns
+    -------
+    numpy.array
+        The loaded image array
+    """
+    # load image from file path
+    image_imread = cv2.imread(image_fpath)
+    # convert to numpy array
+    rgb_image_array = np.array(image_imread)
+    return rgb_image_array

model/utilities/pad_image.py → model/arch/pad_image.py

@@ -1,7 +1,27 @@
 import numpy as np
+from beartype import beartype
+from typing import Union
 
-def pad_image(image_array, pad_shape_wh):
-    """"""
+@beartype
+def pad_image(
+    image_array:np.array,
+    pad_shape_wh:Union[list,tuple]
+    ) -> np.array:
+    """
+    Pads an image array to a desired width and height
+
+    Parameters
+    ----------
+    image_array: np.array
+        The image array to pad to a specified dimension
+    pad_shape_wh : list, tuple
+        The desired dimensions to pad the input image array to
+
+    Returns
+    -------
+    numpy.array
+        The padded image array
+    """
     image_array_shape = image_array.shape
     (img_h, img_w) = image_array_shape[0:2]
     (pad_img_w, pad_img_h) = pad_shape_wh

model/arch/resize_image.py

@@ -0,0 +1,38 @@
+import numpy as np
+import cv2
+from beartype import beartype
+from typing import Union
+
+@beartype
+def resize_image(
+    image_array:np.array,
+    reshape_wh:Union[list,tuple],
+    interpolation=cv2.INTER_LINEAR,
+    keep_3dim:bool=True
+    ) -> np.array:
+    """
+    Resizes a numpy image array to a specified shape width and height
+
+    Parameters
+    ----------
+    image_array : numpy.array
+        The image array to reshape to new dimensions
+    reshape_wh : list, tuple
+        The dimensions to reshape the image array to
+    interpolation : cv2.INTER_LINEAR
+        The interpolation function for reshaping the image array, default is cv2.INTER_LINEAR
+    keep_3dim : bool
+        Whether to maintain the third dimension of the input numpy image array, default is True
+
+    Returns
+    -------
+    numpy.array
+        The reshaped numpy image array
+    """
+    # rescale the image either by shrinking or expanding
+    res_image_array = cv2.resize(image_array, dsize = reshape_wh, interpolation=interpolation)
+    # keep 3dim; when applying resizing to (:, :, 1) shape images
+    if keep_3dim and len(res_image_array.shape) == 2:
+        res_image_array = res_image_array[..., np.newaxis]
+
+    return res_image_array

model/exeKerasModel.cmd

@@ -0,0 +1 @@
+call python -m pdb prg_keras_model.py --run_model_training --run_testset_prediction

model/exeKerasModel.sh

@@ -0,0 +1 @@
+python -m pdb prg_keras_model.py --run_model_training --run_testset_prediction

model/exeUnittests.cmd

@@ -0,0 +1 @@
+call python -m unittest discover unittests\utilities