From 457c2bf62d3aec74555c7a4cfca79b6abc6fb5b6 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 11:02:34 +0000 Subject: [PATCH 01/22] Moved playwright and scrapy folders to arch --- webscrapers/{ => arch}/playwright/cons.py | 0 webscrapers/{ => arch}/playwright/webscraper.py | 0 webscrapers/{ => arch}/scrapy/scrapy.cfg | 0 webscrapers/{ => arch}/scrapy/scrapy_webscraper/__init__.py | 0 webscrapers/{ => arch}/scrapy/scrapy_webscraper/items.py | 0 webscrapers/{ => arch}/scrapy/scrapy_webscraper/middlewares.py | 0 webscrapers/{ => arch}/scrapy/scrapy_webscraper/pipelines.py | 0 webscrapers/{ => arch}/scrapy/scrapy_webscraper/settings.py | 0 .../{ => arch}/scrapy/scrapy_webscraper/spiders/FreeImages.py | 0 .../{ => arch}/scrapy/scrapy_webscraper/spiders/__init__.py | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename webscrapers/{ => arch}/playwright/cons.py (100%) rename webscrapers/{ => arch}/playwright/webscraper.py (100%) rename webscrapers/{ => arch}/scrapy/scrapy.cfg (100%) rename webscrapers/{ => arch}/scrapy/scrapy_webscraper/__init__.py (100%) rename webscrapers/{ => arch}/scrapy/scrapy_webscraper/items.py (100%) rename webscrapers/{ => arch}/scrapy/scrapy_webscraper/middlewares.py (100%) rename webscrapers/{ => arch}/scrapy/scrapy_webscraper/pipelines.py (100%) rename webscrapers/{ => arch}/scrapy/scrapy_webscraper/settings.py (100%) rename webscrapers/{ => arch}/scrapy/scrapy_webscraper/spiders/FreeImages.py (100%) rename webscrapers/{ => arch}/scrapy/scrapy_webscraper/spiders/__init__.py (100%) diff --git a/webscrapers/playwright/cons.py b/webscrapers/arch/playwright/cons.py similarity index 100% rename from webscrapers/playwright/cons.py rename to webscrapers/arch/playwright/cons.py diff --git a/webscrapers/playwright/webscraper.py b/webscrapers/arch/playwright/webscraper.py similarity index 100% rename from webscrapers/playwright/webscraper.py rename to webscrapers/arch/playwright/webscraper.py diff --git a/webscrapers/scrapy/scrapy.cfg b/webscrapers/arch/scrapy/scrapy.cfg similarity index 100% rename from webscrapers/scrapy/scrapy.cfg rename to webscrapers/arch/scrapy/scrapy.cfg diff --git a/webscrapers/scrapy/scrapy_webscraper/__init__.py b/webscrapers/arch/scrapy/scrapy_webscraper/__init__.py similarity index 100% rename from webscrapers/scrapy/scrapy_webscraper/__init__.py rename to webscrapers/arch/scrapy/scrapy_webscraper/__init__.py diff --git a/webscrapers/scrapy/scrapy_webscraper/items.py b/webscrapers/arch/scrapy/scrapy_webscraper/items.py similarity index 100% rename from webscrapers/scrapy/scrapy_webscraper/items.py rename to webscrapers/arch/scrapy/scrapy_webscraper/items.py diff --git a/webscrapers/scrapy/scrapy_webscraper/middlewares.py b/webscrapers/arch/scrapy/scrapy_webscraper/middlewares.py similarity index 100% rename from webscrapers/scrapy/scrapy_webscraper/middlewares.py rename to webscrapers/arch/scrapy/scrapy_webscraper/middlewares.py diff --git a/webscrapers/scrapy/scrapy_webscraper/pipelines.py b/webscrapers/arch/scrapy/scrapy_webscraper/pipelines.py similarity index 100% rename from webscrapers/scrapy/scrapy_webscraper/pipelines.py rename to webscrapers/arch/scrapy/scrapy_webscraper/pipelines.py diff --git a/webscrapers/scrapy/scrapy_webscraper/settings.py b/webscrapers/arch/scrapy/scrapy_webscraper/settings.py similarity index 100% rename from webscrapers/scrapy/scrapy_webscraper/settings.py rename to webscrapers/arch/scrapy/scrapy_webscraper/settings.py diff --git a/webscrapers/scrapy/scrapy_webscraper/spiders/FreeImages.py b/webscrapers/arch/scrapy/scrapy_webscraper/spiders/FreeImages.py similarity index 100% rename from webscrapers/scrapy/scrapy_webscraper/spiders/FreeImages.py rename to webscrapers/arch/scrapy/scrapy_webscraper/spiders/FreeImages.py diff --git a/webscrapers/scrapy/scrapy_webscraper/spiders/__init__.py b/webscrapers/arch/scrapy/scrapy_webscraper/spiders/__init__.py similarity index 100% rename from webscrapers/scrapy/scrapy_webscraper/spiders/__init__.py rename to webscrapers/arch/scrapy/scrapy_webscraper/spiders/__init__.py From df30114f295bca93ee2d351bc8ffbd42189ee198 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 11:04:00 +0000 Subject: [PATCH 02/22] Moved beautifulsoup, commandline interface and download comp data into a utilities folder --- webscrapers/{ => utilities}/commandline_interface.py | 0 webscrapers/{ => utilities}/download_comp_data.py | 0 webscrapers/{beautifulsoup => utilities}/webscraper.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename webscrapers/{ => utilities}/commandline_interface.py (100%) rename webscrapers/{ => utilities}/download_comp_data.py (100%) rename webscrapers/{beautifulsoup => utilities}/webscraper.py (100%) diff --git a/webscrapers/commandline_interface.py b/webscrapers/utilities/commandline_interface.py similarity index 100% rename from webscrapers/commandline_interface.py rename to webscrapers/utilities/commandline_interface.py diff --git a/webscrapers/download_comp_data.py b/webscrapers/utilities/download_comp_data.py similarity index 100% rename from webscrapers/download_comp_data.py rename to webscrapers/utilities/download_comp_data.py diff --git a/webscrapers/beautifulsoup/webscraper.py b/webscrapers/utilities/webscraper.py similarity index 100% rename from webscrapers/beautifulsoup/webscraper.py rename to webscrapers/utilities/webscraper.py From affb5d7e90e1d795c60170e6bb4a63eb3d40942f Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 11:05:50 +0000 Subject: [PATCH 03/22] Loading modules from utilities folder --- webscrapers/prg_scrape_imgs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/webscrapers/prg_scrape_imgs.py b/webscrapers/prg_scrape_imgs.py index 27acfb0..7d45bc6 100644 --- a/webscrapers/prg_scrape_imgs.py +++ b/webscrapers/prg_scrape_imgs.py @@ -1,9 +1,9 @@ import logging -import cons -from commandline_interface import commandline_interface -from webscrapers.download_comp_data import download_comp_data -from webscrapers.beautifulsoup.webscraper import webscraper from beartype import beartype +import cons +from utilities.commandline_interface import commandline_interface +from utilities.download_comp_data import download_comp_data +from utilities.webscraper import webscraper @beartype def scrape_imags( From 38d132d2730c1ee707fb1f618e6e786d486f859b Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 11:06:22 +0000 Subject: [PATCH 04/22] Created unittests --- webscrapers/unittests/test_gen_urls.py | 33 ++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 webscrapers/unittests/test_gen_urls.py diff --git a/webscrapers/unittests/test_gen_urls.py b/webscrapers/unittests/test_gen_urls.py new file mode 100644 index 0000000..0a88354 --- /dev/null +++ b/webscrapers/unittests/test_gen_urls.py @@ -0,0 +1,33 @@ +import unittest +import os +import sys +import numpy as np +import pandas as pd +import random + +sys.path.append(os.path.join(os.getcwd(), "webscrapers")) + +import cons +from beautifulsoup.webscraper import gen_urls + +exp_urls = ['https://free-images.com/search/?q=cat&skip=0', 'https://free-images.com/search/?q=cat&skip=100', 'https://free-images.com/search/?q=cat&skip=200'] +obs_urls = gen_urls(search="cat", n_images=300, home_url=cons.home_url) + +class Test_gen_urls(unittest.TestCase): + """""" + + def setUp(self): + self.obs_urls = obs_urls + self.exp_urls = exp_urls + + def test_type(self): + self.assertEqual(type(self.obs_urls), type(self.exp_urls)) + + def test_len(self): + self.assertEqual(len(self.obs_urls), len(self.exp_urls)) + + def test_values(self): + self.assertEqual(self.obs_urls, self.exp_urls) + +if __name__ == "__main__": + unittest.main() From 57a5574f6666f501ff110389d65e0a57d6f67d4f Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 11:06:56 +0000 Subject: [PATCH 05/22] Created .cmd file to execute unittests --- webscrapers/exeUnittests.cmd | 1 + 1 file changed, 1 insertion(+) create mode 100644 webscrapers/exeUnittests.cmd diff --git a/webscrapers/exeUnittests.cmd b/webscrapers/exeUnittests.cmd new file mode 100644 index 0000000..682acaa --- /dev/null +++ b/webscrapers/exeUnittests.cmd @@ -0,0 +1 @@ +call python -m unittest discover unittests\beautifulsoup \ No newline at end of file From fcd7e0b9685292d246336ab0db6015dd12881eb4 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 11:09:56 +0000 Subject: [PATCH 06/22] Fixed imports --- webscrapers/exeUnittests.cmd | 2 +- webscrapers/unittests/test_gen_urls.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/webscrapers/exeUnittests.cmd b/webscrapers/exeUnittests.cmd index 682acaa..d468136 100644 --- a/webscrapers/exeUnittests.cmd +++ b/webscrapers/exeUnittests.cmd @@ -1 +1 @@ -call python -m unittest discover unittests\beautifulsoup \ No newline at end of file +call python -m unittest discover unittests \ No newline at end of file diff --git a/webscrapers/unittests/test_gen_urls.py b/webscrapers/unittests/test_gen_urls.py index 0a88354..dcf2cb6 100644 --- a/webscrapers/unittests/test_gen_urls.py +++ b/webscrapers/unittests/test_gen_urls.py @@ -1,14 +1,11 @@ import unittest import os import sys -import numpy as np -import pandas as pd -import random sys.path.append(os.path.join(os.getcwd(), "webscrapers")) import cons -from beautifulsoup.webscraper import gen_urls +from utilities.webscraper import gen_urls exp_urls = ['https://free-images.com/search/?q=cat&skip=0', 'https://free-images.com/search/?q=cat&skip=100', 'https://free-images.com/search/?q=cat&skip=200'] obs_urls = gen_urls(search="cat", n_images=300, home_url=cons.home_url) From 7b34a63216b8a96728cf61c0f3473222ce07dbd3 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 12:27:13 +0000 Subject: [PATCH 07/22] Running docker push pipeline after main unittests run --- .github/workflows/dockerhub-main-push.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-main-push.yml b/.github/workflows/dockerhub-main-push.yml index 8f8dc0d..dbec67d 100644 --- a/.github/workflows/dockerhub-main-push.yml +++ b/.github/workflows/dockerhub-main-push.yml @@ -1,8 +1,10 @@ name: DockerHub Main Push on: - push: - branches: [ "main" ] + workflow_run: + workflows: ["Unittest Main Push"] + types: + - completed jobs: build: From d9809698d158b9a20aae9860c4db4884bf94c55b Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 12:27:52 +0000 Subject: [PATCH 08/22] Added github action pipelines for running unittests after push to dev, and merge request and push to main. --- .github/workflows/unittest-dev.yml | 30 ++++++++++++++++++++++++ .github/workflows/unittest-main-pr.yml | 28 ++++++++++++++++++++++ .github/workflows/unittest-main-push.yml | 28 ++++++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 .github/workflows/unittest-dev.yml create mode 100644 .github/workflows/unittest-main-pr.yml create mode 100644 .github/workflows/unittest-main-push.yml diff --git a/.github/workflows/unittest-dev.yml b/.github/workflows/unittest-dev.yml new file mode 100644 index 0000000..918ddd3 --- /dev/null +++ b/.github/workflows/unittest-dev.yml @@ -0,0 +1,30 @@ +name: Unittest Dev + +on: + pull_request: + branches: [ "dev" ] + push: + branches: [ "dev" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Test with unittest + run: | + python -m unittest discover ./webscrapers/unittests/utilities diff --git a/.github/workflows/unittest-main-pr.yml b/.github/workflows/unittest-main-pr.yml new file mode 100644 index 0000000..73fe9d1 --- /dev/null +++ b/.github/workflows/unittest-main-pr.yml @@ -0,0 +1,28 @@ +name: Unittest Main Pull Request + +on: + pull_request: + branches: [ "main" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Test with unittest + run: | + python -m unittest discover ./webscrapers/unittests/utilities diff --git a/.github/workflows/unittest-main-push.yml b/.github/workflows/unittest-main-push.yml new file mode 100644 index 0000000..5fe6263 --- /dev/null +++ b/.github/workflows/unittest-main-push.yml @@ -0,0 +1,28 @@ +name: Unittest Main Push + +on: + push: + branches: [ "main" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Test with unittest + run: | + python -m unittest discover ./webscrapers/unittests/utilities From 09054fa30915b37bf34d6c459b9101405020df08 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 12:35:10 +0000 Subject: [PATCH 09/22] Added unittest for scraping image srcs from urls --- webscrapers/unittests/test_scrape_srcs.py | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 webscrapers/unittests/test_scrape_srcs.py diff --git a/webscrapers/unittests/test_scrape_srcs.py b/webscrapers/unittests/test_scrape_srcs.py new file mode 100644 index 0000000..7d283ff --- /dev/null +++ b/webscrapers/unittests/test_scrape_srcs.py @@ -0,0 +1,31 @@ +import unittest +import os +import sys + +sys.path.append(os.path.join(os.getcwd(), "webscrapers")) + +import cons +from utilities.webscraper import scrape_srcs + +exp_srcs = ['https://free-images.com/sm/d790/cat_home_cat_looking.jpg', 'https://free-images.com/sm/e396/cat_hangover_red_cute_51.jpg', 'https://free-images.com/sm/bc44/cat_hangover_siamese_cat_2.jpg'] +urls = ['https://free-images.com/search/?q=cat&skip=0'] +obs_srcs = scrape_srcs(urls=urls, n_images=3, home_url=cons.home_url) + +class Test_scrape_srcs(unittest.TestCase): + """""" + + def setUp(self): + self.obs_srcs = obs_srcs + self.exp_srcs = exp_srcs + + def test_type(self): + self.assertEqual(type(self.obs_srcs), type(self.exp_srcs)) + + def test_len(self): + self.assertEqual(len(self.obs_srcs), len(self.exp_srcs)) + + def test_values(self): + self.assertEqual(self.obs_srcs, self.exp_srcs) + +if __name__ == "__main__": + unittest.main() From 8a469a549164a02c6755a228cb9352a2ce034354 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 12:44:20 +0000 Subject: [PATCH 10/22] Moved unused modelling scripts to archive directory --- model/{utilities => arch}/crop_image.py | 0 model/{ => arch}/data_prep.py | 0 model/{utilities => arch}/greyscale_image.py | 0 model/{utilities => arch}/load_image.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename model/{utilities => arch}/crop_image.py (100%) rename model/{ => arch}/data_prep.py (100%) rename model/{utilities => arch}/greyscale_image.py (100%) rename model/{utilities => arch}/load_image.py (100%) diff --git a/model/utilities/crop_image.py b/model/arch/crop_image.py similarity index 100% rename from model/utilities/crop_image.py rename to model/arch/crop_image.py diff --git a/model/data_prep.py b/model/arch/data_prep.py similarity index 100% rename from model/data_prep.py rename to model/arch/data_prep.py diff --git a/model/utilities/greyscale_image.py b/model/arch/greyscale_image.py similarity index 100% rename from model/utilities/greyscale_image.py rename to model/arch/greyscale_image.py diff --git a/model/utilities/load_image.py b/model/arch/load_image.py similarity index 100% rename from model/utilities/load_image.py rename to model/arch/load_image.py From 002b6d2e84068c4d858e73b653e9181cedfd1478 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 14:10:41 +0000 Subject: [PATCH 11/22] Moved unused functions to arch directory --- model/{utilities => arch}/pad_image.py | 0 model/{utilities => arch}/resize_image.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename model/{utilities => arch}/pad_image.py (100%) rename model/{utilities => arch}/resize_image.py (100%) diff --git a/model/utilities/pad_image.py b/model/arch/pad_image.py similarity index 100% rename from model/utilities/pad_image.py rename to model/arch/pad_image.py diff --git a/model/utilities/resize_image.py b/model/arch/resize_image.py similarity index 100% rename from model/utilities/resize_image.py rename to model/arch/resize_image.py From af808823f86a95382033e22c13dbd7c53cc7f81b Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 15:55:28 +0000 Subject: [PATCH 12/22] Updated docstrings and beartyping --- model/utilities/TimeIt.py | 28 +++++++++++++++++++----- model/utilities/plot_model.py | 41 ++++++++++++++--------------------- model/utilities/plot_preds.py | 21 ++++++++++++++---- 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/model/utilities/TimeIt.py b/model/utilities/TimeIt.py index 3be6215..637f3ae 100644 --- a/model/utilities/TimeIt.py +++ b/model/utilities/TimeIt.py @@ -1,12 +1,12 @@ import logging +from beartype import beartype +from types import UnionType from time import time class TimeIt(): - """ - """ def __init__(self): - """ + """An object for timing code execution times, records time step and cumulative time. """ self.log = [] self.currentTime = time() @@ -14,8 +14,16 @@ def __init__(self): self.stepTime = None self.cumulativeTime = 0.0 - def logTime(self, parentKey, subKey=None): - """ + @beartype + def logTime(self, parentKey:str, subKey:UnionType[str,None]=None): + """Sets a timestamp for code execution step + + Parameters + ---------- + paraentKey : str + The parent key to label the code execution step + subKey : str + The sub key to label the code execution step, default is None """ self.previousTime = self.currentTime self.currentTime = time() @@ -24,4 +32,12 @@ def logTime(self, parentKey, subKey=None): logEntry = {"parentKey":parentKey, "subKey":subKey, "stepTime":self.stepTime, "cumulativeTime":self.cumulativeTime} logging.info(logEntry) self.log.append(logEntry) - \ No newline at end of file + + def reset(self): + """Resets the timing object + """ + self.log = [] + self.currentTime = time() + self.previousTime = None + self.stepTime = None + self.cumulativeTime = 0.0 \ No newline at end of file diff --git a/model/utilities/plot_model.py b/model/utilities/plot_model.py index 179e280..4666eed 100644 --- a/model/utilities/plot_model.py +++ b/model/utilities/plot_model.py @@ -6,38 +6,31 @@ # load relevant libraries import os +from beartype import beartype +from types import UnionType from matplotlib import pyplot as plt -def plot_model_fit(model_fit, output_fdir = None, show_plot:bool=True): +@beartype +def plot_model_fit(model_fit, output_fdir:UnionType[str,None]=None, show_plot:bool=True): - """ - - Plot Model Fit Documentation - - Function Overview - - This function plots the model's fit during training in relation to a validation set. - - Defaults - - plot_model_fit(model_fit) + """This function plots the model's fit during training in relation to a validation set. Parameters + ---------- - model_fit - model.predict(), the Keras model predict object + model_fit : model.predict(), + the Keras model predict object + output_fdir : str + The file path to save the model fit plots to disk, default is None show_plot : bool Whether to show the generated plot, default is True Returns - - 0 for successful execution - - Exmaple - - plot_model_fit(model_fit = model_fit) - - Source - + ------- + + Reference + ---------- + https://github.com/jiadaizhao/Advanced-Machine-Learning-Specialization """ @@ -88,6 +81,4 @@ def plot_model_fit(model_fit, output_fdir = None, show_plot:bool=True): plt.savefig(loss_output_fpath) if show_plot: plt.show() - plt.close() - - return 0 \ No newline at end of file + plt.close() \ No newline at end of file diff --git a/model/utilities/plot_preds.py b/model/utilities/plot_preds.py index 615a726..30e33ad 100644 --- a/model/utilities/plot_preds.py +++ b/model/utilities/plot_preds.py @@ -1,11 +1,25 @@ import os +from beartype import beartype import matplotlib.pyplot as plt +import pandas as pd from tensorflow.keras.preprocessing.image import load_img +import cons -def plot_preds(data, cons, output_fpath = None, show_plot:bool=True): - """ +@beartype +def plot_preds(data:pd.DataFrame, output_fpath:str = None, show_plot:bool=True): + """Shows model predictions as a grid of images with labels + + Parameters + ---------- + data : pandas.DataFrame + A dataframe of image arrays and associated labels + output_fpath : str + The file path to save the plot of predictions, default is None show_plot : bool Whether to show the generated plot, default is True + + Returns + ------- """ sample_test = data.head(18) plt.figure(figsize=(12, 24)) @@ -23,5 +37,4 @@ def plot_preds(data, cons, output_fpath = None, show_plot:bool=True): plt.savefig(output_fpath) if show_plot: plt.show() - plt.close() - return 0 \ No newline at end of file + plt.close() \ No newline at end of file From 44ca7f8c207051d4e5ea3e4c76d1021374fd6fee Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 18:02:54 +0000 Subject: [PATCH 13/22] Updated docstrings and beartyping --- model/arch/crop_image.py | 28 +++++++++++++++++++++++++-- model/arch/data_prep.py | 32 +++++++++++++++++-------------- model/arch/greyscale_image.py | 23 ++++++++++++++++++++-- model/arch/keras_model.py | 4 ++-- model/arch/load_image.py | 18 +++++++++++++++-- model/arch/pad_image.py | 24 +++++++++++++++++++++-- model/arch/resize_image.py | 30 +++++++++++++++++++++++++++-- model/utilities/plot_generator.py | 10 ++++++++-- model/utilities/plot_image.py | 10 ++++++++-- model/utilities/plot_model.py | 10 +++++++--- model/utilities/plot_preds.py | 9 +++++++-- 11 files changed, 163 insertions(+), 35 deletions(-) diff --git a/model/arch/crop_image.py b/model/arch/crop_image.py index e09a3c7..6e40136 100644 --- a/model/arch/crop_image.py +++ b/model/arch/crop_image.py @@ -1,5 +1,29 @@ -def crop_image(image_array, pt1_wh, pt2_wh): - """""" +import numpy as np +from beartype import beartype +from types import UnionType + +@beartype +def crop_image( + image_array:np.array, + pt1_wh:UnionType[tuple,list], + pt2_wh:UnionType[tuple,list] + ) -> np.array: + """Crops an image array to specified combination of two diagonal points + + Parameters + --------- + image : numpy.array + The numpy image array to crop + pt1_wh : list, tuple + Diagonal point 1 coordinates for cropping the image + pt2_wh : list, tuple + Diagonal point 2 coordinates for cropping the image + + Returns + ------- + numpy.array + The cropped image array + """ # extract out diagonal cropping points (pt1_w, pt1_h) = pt1_wh (pt2_w, pt2_h) = pt2_wh diff --git a/model/arch/data_prep.py b/model/arch/data_prep.py index 1d66338..7146941 100644 --- a/model/arch/data_prep.py +++ b/model/arch/data_prep.py @@ -2,6 +2,7 @@ import os import sys import pickle +import logging import numpy as np import pandas as pd from sklearn.model_selection import train_test_split @@ -9,17 +10,22 @@ # load custom modules sys.path.append(os.getcwd()) import cons -from utilities.load_image import load_image -from utilities.greyscale_image import greyscale_image -from utilities.pad_image import pad_image -from utilities.plot_image import plot_image -from utilities.resize_image import resize_image +from arch.load_image import load_image +from arch.greyscale_image import greyscale_image +from arch.pad_image import pad_image +from arch.resize_image import resize_image -def data_prep(cons): +def data_prep(): + """Data preparation pipeline for generating the model training, testing and validation data. - """""" + Parameters + ---------- - print("Generating image file paths and classes ...") + Returns + ------- + """ + + logging.info("Generating image file paths and classes ...") if False: @@ -55,7 +61,7 @@ def data_prep(cons): # combine train and test files image_fpaths = {**train_image_fpaths, **test_image_fpaths} - print("Creating image dataframe ...") + logging.info("Creating image dataframe ...") # create list to hold image data image_data = [] @@ -78,7 +84,7 @@ def data_prep(cons): # convert image data object into a pandas dataframe image_dataframe = pd.DataFrame(image_data) - print("Padding images ...") + logging.info("Padding images ...") # find the largest image dimensions max_height = image_dataframe['image_shape'].apply(lambda x: x[0]).max() # height @@ -90,7 +96,7 @@ def data_prep(cons): # apply padding to standardize all images shapes image_dataframe['pad_image_array'] = image_dataframe['image_array'].apply(lambda x: pad_image(x, pad_shape_wh = pad_shape)) - print('Down sizing image ...') + logging.info('Down sizing image ...') # set down size shape downsize_shape = tuple([round(dim * 1/3) for dim in pad_shape]) @@ -98,7 +104,7 @@ def data_prep(cons): # apply resizing to downsize image shapes image_dataframe['pad_image_array'] = image_dataframe['pad_image_array'].apply(lambda x: resize_image(x, reshape_wh = downsize_shape)) - print('Splitting train set ...') + logging.info('Splitting train set ...') # subset the output image data sub_cols = ['image_fpath', 'pad_image_array', 'target', 'dataset'] @@ -136,8 +142,6 @@ def data_prep(cons): with open(cons.test_data_pickle_fpath, 'wb') as handle: pickle.dump(test_data_dict, handle, protocol=pickle.HIGHEST_PROTOCOL) - return 0 - # if running as main programme if __name__ == "__main__": diff --git a/model/arch/greyscale_image.py b/model/arch/greyscale_image.py index b7a30e4..d6a6a73 100644 --- a/model/arch/greyscale_image.py +++ b/model/arch/greyscale_image.py @@ -1,7 +1,26 @@ import numpy as np +from beartype import beartype -def greyscale_image(rgb_image_array, keep_3dim = True): - """""" +@beartype +def greyscale_image( + rgb_image_array:np.array, + keep_3dim:bool=True + ) -> np.array: + """ + Transforms an image array to greyscale + + Parameters + ---------- + rgb_image_array : numpy.array + The coloured numpy image array to transform to greyscale + keep_3dim : bool + Whether to keep the third dimension of the image array, default is True + + Returns + ------- + numpy.array + The transformed greyscale numpy image + """ # apply grey scale transformation grey_image_array = np.dot(rgb_image_array[:, :, :3], [0.2125, 0.7154, 0.0721]) # floor transformed pixel floats diff --git a/model/arch/keras_model.py b/model/arch/keras_model.py index 566f4ea..c4c04ef 100644 --- a/model/arch/keras_model.py +++ b/model/arch/keras_model.py @@ -12,9 +12,9 @@ sys.path.append(os.getcwd()) import cons from utilities.plot_image import plot_image -from arch.LeNet5 import LeNet5 +from keras.LeNet5 import LeNet5 from fit_model import fit_model -from plot_model import plot_model_fit +from utilities.plot_model import plot_model_fit print('Loading data ...') diff --git a/model/arch/load_image.py b/model/arch/load_image.py index a5db503..23d2e16 100644 --- a/model/arch/load_image.py +++ b/model/arch/load_image.py @@ -1,8 +1,22 @@ import numpy as np import cv2 +from beartype import beartype -def load_image(image_fpath): - """""" +@beartype +def load_image(image_fpath:str) -> np.array: + """ + Loads an image file as an image array from disk + + Parameters + ---------- + image_fpath : str + The file path to the image file to load as an image array + + Returns + ------- + numpy.array + The loaded image array + """ # load image from file path image_imread = cv2.imread(image_fpath) # convert to numpy array diff --git a/model/arch/pad_image.py b/model/arch/pad_image.py index 21dbbcf..2f8d786 100644 --- a/model/arch/pad_image.py +++ b/model/arch/pad_image.py @@ -1,7 +1,27 @@ import numpy as np +from beartype import beartype +from types import UnionType -def pad_image(image_array, pad_shape_wh): - """""" +@beartype +def pad_image( + image_array:np.array, + pad_shape_wh:UnionType[list,tuple] + ) -> np.array: + """ + Pads an image array to a desired width and height + + Parameters + ---------- + image_array: np.array + The image array to pad to a specified dimension + pad_shape_wh : list, tuple + The desired dimensions to pad the input image array to + + Returns + ------- + numpy.array + The padded image array + """ image_array_shape = image_array.shape (img_h, img_w) = image_array_shape[0:2] (pad_img_w, pad_img_h) = pad_shape_wh diff --git a/model/arch/resize_image.py b/model/arch/resize_image.py index 6c44240..68f2aaf 100644 --- a/model/arch/resize_image.py +++ b/model/arch/resize_image.py @@ -1,8 +1,34 @@ import numpy as np import cv2 +from beartype import beartype +from types import UnionType -def resize_image(image_array, reshape_wh, interpolation = cv2.INTER_LINEAR, keep_3dim = True): - """""" +@beartype +def resize_image( + image_array:np.array, + reshape_wh:UnionType[list,tuple], + interpolation=cv2.INTER_LINEAR, + keep_3dim:bool=True + ) -> np.array: + """ + Resizes a numpy image array to a specified shape width and height + + Parameters + ---------- + image_array : numpy.array + The image array to reshape to new dimensions + reshape_wh : list, tuple + The dimensions to reshape the image array to + interpolation : cv2.INTER_LINEAR + The interpolation function for reshaping the image array, default is cv2.INTER_LINEAR + keep_3dim : bool + Whether to maintain the third dimension of the input numpy image array, default is True + + Returns + ------- + numpy.array + The reshaped numpy image array + """ # rescale the image either by shrinking or expanding res_image_array = cv2.resize(image_array, dsize = reshape_wh, interpolation=interpolation) # keep 3dim; when applying resizing to (:, :, 1) shape images diff --git a/model/utilities/plot_generator.py b/model/utilities/plot_generator.py index bf76cca..5e0a7cd 100644 --- a/model/utilities/plot_generator.py +++ b/model/utilities/plot_generator.py @@ -3,8 +3,14 @@ from beartype import beartype @beartype -def plot_generator(generator, mode:str='keras', output_fpath:Union[str,None]=None, show_plot:bool=True): - """Plots multiple images from a generator. +def plot_generator( + generator, + mode:str='keras', + output_fpath:Union[str,None]=None, + show_plot:bool=True + ): + """ + Plots multiple images from a generator. Parameters ---------- diff --git a/model/utilities/plot_image.py b/model/utilities/plot_image.py index 187cae1..636b802 100644 --- a/model/utilities/plot_image.py +++ b/model/utilities/plot_image.py @@ -1,10 +1,16 @@ import matplotlib.pyplot as plt +import numpy as np from beartype import beartype from typing import Union @beartype -def plot_image(image_array, output_fpath:Union[str,None]=None, show_plot:bool=True): - """Plots an image array. +def plot_image( + image_array:np.array, + output_fpath:Union[str,None]=None, + show_plot:bool=True + ): + """ + Plots an image array. Parameters ---------- diff --git a/model/utilities/plot_model.py b/model/utilities/plot_model.py index 4666eed..5c703ed 100644 --- a/model/utilities/plot_model.py +++ b/model/utilities/plot_model.py @@ -11,9 +11,13 @@ from matplotlib import pyplot as plt @beartype -def plot_model_fit(model_fit, output_fdir:UnionType[str,None]=None, show_plot:bool=True): - - """This function plots the model's fit during training in relation to a validation set. +def plot_model_fit( + model_fit, + output_fdir:UnionType[str,None]=None, + show_plot:bool=True + ): + """ + Plots the model's fit during training in relation to a validation set. Parameters ---------- diff --git a/model/utilities/plot_preds.py b/model/utilities/plot_preds.py index 30e33ad..7850dee 100644 --- a/model/utilities/plot_preds.py +++ b/model/utilities/plot_preds.py @@ -6,8 +6,13 @@ import cons @beartype -def plot_preds(data:pd.DataFrame, output_fpath:str = None, show_plot:bool=True): - """Shows model predictions as a grid of images with labels +def plot_preds( + data:pd.DataFrame, + output_fpath:str = None, + show_plot:bool=True + ): + """ + Shows model predictions as a grid of images with labels Parameters ---------- From 7810c30e8677bcab92055f5c259daf5b8e868945 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 18:17:03 +0000 Subject: [PATCH 14/22] Updated docstrings and beartypes --- model/keras/AlexNet8.py | 28 +++++++++--- model/keras/LeNet5.py | 97 ++++++++--------------------------------- model/keras/VGG16.py | 28 +++++++++--- 3 files changed, 64 insertions(+), 89 deletions(-) diff --git a/model/keras/AlexNet8.py b/model/keras/AlexNet8.py index 1300584..f36019d 100644 --- a/model/keras/AlexNet8.py +++ b/model/keras/AlexNet8.py @@ -1,14 +1,32 @@ # load in relevant libraries +from beartype import beartype +from types import UnionType from keras.models import Model from keras.layers import Flatten, Dense, Input, Dropout from keras.layers import Conv2D, MaxPooling2D -def AlexNet8(input_shape = (227, 227, 3), - n_classes = 1000, - output_activation = 'softmax' - ): - +@beartype +def AlexNet8( + input_shape:UnionType[list,tuple]=(227,227,3), + n_classes:int=1000, + output_activation:str='softmax' + ) -> Model: """ + AlexNet8 Keras model + + Parameters + ---------- + input_shape : list,tuple + The dimensions of the input image arrays, default is (227.227,3) + n_classes : int + The number of output classes to classify for, default is 1000 + output_activation : str + The type of activation function to use, default is softmax + + Returns + ------- + Model + The keras AlexNet8 model """ # set input shapes diff --git a/model/keras/LeNet5.py b/model/keras/LeNet5.py index a3b7373..9a4ce62 100644 --- a/model/keras/LeNet5.py +++ b/model/keras/LeNet5.py @@ -1,90 +1,31 @@ -# -*- coding: utf-8 -*- -""" -Created on Sun Jan 31 15:22:57 2021 -@author: oislen -""" - -# load in relevant libraries +from beartype import beartype +from types import UnionType from keras.models import Model from keras.layers import Flatten, Dense, Input, Dropout from keras.layers import Conv2D, AveragePooling2D -def LeNet5(input_shape = (28, 28, 1), - n_classes = 10, - output_activation = 'softmax' - ): - +@beartype +def LeNet5( + input_shape:UnionType[list,tuple]=(28,28,1), + n_classes:int=10, + output_activation:str='softmax' + ) -> Model: """ - - LeNet5 Documentation - - Function Overview - - This function generates a LeNet5 Model architecture: - - 1. Conv2D - - filters: 32 - - kernal: 5 x 5 - - activation: relu - - padding: same - - MaxPooling2D - - pool: 2 x 2 - - 2. Conv2D - - filters: 32 - - kernal: 5 x 5 - - activation: relu - - padding: same - - MaxPooling2D - - pool: 2 x 2 - - Flatten - - 3. Dense - - units: 128 - - activation: relu - - Dropout - - rate: 0.25 - - 4. Dense - - units: 64 - - activation: relu - - Dropout - - 0.25 rate - - 5. Dense - - units: n classes - - activation: output_activation - - Defaults - - LeNet5(input_shape = (28, 28, 1), - n_classes = 10, - output_activation = 'softmax', - name = 'LeNet5' - ) + LeNet5 keras model Parameters - - input_shape - the input image shape / dimensions - n_classes - the number of target classes + ---------- + input_shape : list,tuple + the input image shape / dimensions, default is (28,28,1) + n_classes : int + The number of target classes, default is 10 + output_activation : str + The type of activation function to use, default is softmax Returns - - model - keras.Model, the LeNet model - - Example - - LeNet5(input_shape = (28, 28, 1), - n_classes = 10, - output_activation = 'softmax', - name = 'LeNet5' - ) - + ------- + Model + Keras.Model, the LeNet model """ # set input shapes diff --git a/model/keras/VGG16.py b/model/keras/VGG16.py index 05ed56e..57bbeb0 100644 --- a/model/keras/VGG16.py +++ b/model/keras/VGG16.py @@ -1,14 +1,30 @@ -# load in relevant libraries +from beartype import beartype +from types import UnionType from keras.models import Model from keras.layers import Flatten, Dense, Input, Dropout from keras.layers import Conv2D, MaxPooling2D -def VGG16(input_shape = (224, 224, 3), - n_classes = 1000, - output_activation = 'softmax' - ): - +@beartype +def VGG16(input_shape:UnionType[list,tuple]=(224,224,3), + n_classes:int=1000, + output_activation:str='softmax' + ) -> Model: """ + VGG16 keras model + + Parameters + ---------- + input_shape : list,tuple + the input image shape / dimensions, default is (224,224,3) + n_classes : int + The number of target classes, default is 1000 + output_activation : str + The type of activation function to use, default is softmax + + Returns + ------- + Model + Keras.Model, the VGG16 model """ # set input shapes From 7a42adcd46fc86701680a39ac912a4d97b3019ea Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 18:28:12 +0000 Subject: [PATCH 15/22] Updated docstrings and beartyping --- model/keras/ResNet50_pretrained.py | 36 ++++++++++++++++++++++-------- model/keras/UNet.py | 5 +++++ model/keras/VGG16_pretrained.py | 30 +++++++++++++++++++++---- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/model/keras/ResNet50_pretrained.py b/model/keras/ResNet50_pretrained.py index 6d56720..935f026 100644 --- a/model/keras/ResNet50_pretrained.py +++ b/model/keras/ResNet50_pretrained.py @@ -1,13 +1,34 @@ +from beartype import beartype +from types import UnionType from keras.layers import Dropout, Dense, Flatten from keras.applications import ResNet50 from keras.models import Model -def ResNet50_pretrained(input_shape = (224, 224, 3), - n_classes = 2, - output_activation = 'softmax' - ): - +@beartype +def ResNet50_pretrained( + input_shape:UnionType[list,tuple]=(224,224,3), + n_classes:int=2, + output_activation:str='softmax' + ) -> Model: """ + Pretrained ResNet50 keras model + + Parameters + ---------- + input_shape : list,tuple + the input image shape / dimensions, default is (224,224,3) + n_classes : int + The number of target classes, default is 2 + output_activation : str + The type of activation function to use, default is softmax + + Returns + ------- + Model + Keras.Model, the pretrained ResNet50 model + + Example + ------- model = ResNet50_pretrained(input_shape = (224, 224, 3), n_classes = 2, output_activation = 'softmax', @@ -41,7 +62,4 @@ def ResNet50_pretrained(input_shape = (224, 224, 3), model = Model(inputs = ResNet50_pretrained.input, outputs = x, name = 'ResNet50_pretrained') - return model - - - + return model \ No newline at end of file diff --git a/model/keras/UNet.py b/model/keras/UNet.py index e860445..8988f75 100644 --- a/model/keras/UNet.py +++ b/model/keras/UNet.py @@ -1,3 +1,8 @@ +from beartype import beartype +from types import UnionType +from keras.layers import Input, Conv2D, MaxPooling2D, Conv2DTranspose +from keras.layers.merge import concatenate +import tensorflow as tf def conv_block(inputs=None, n_filters=32, dropout_prob=0, max_pooling=True): """ diff --git a/model/keras/VGG16_pretrained.py b/model/keras/VGG16_pretrained.py index 85af564..6896129 100644 --- a/model/keras/VGG16_pretrained.py +++ b/model/keras/VGG16_pretrained.py @@ -1,12 +1,34 @@ +from beartype import beartype +from types import UnionType from keras.layers import Dropout, Dense, GlobalMaxPooling2D from keras.applications import VGG16 from keras.models import Model -def VGG16_pretrained(input_shape = (224, 224, 3), - n_classes = 1, - output_activation = 'sigmoid' - ): +@beartype +def VGG16_pretrained( + input_shape:UnionType[list,tuple]=(224,224,3), + n_classes:int=1, + output_activation:str='sigmoid' + ) -> Model: """ + Pretrained VGG16 keras model + + Parameters + ---------- + input_shape : list,tuple + the input image shape / dimensions, default is (224,224,3) + n_classes : int + The number of target classes, default is 1 + output_activation : str + The type of activation function to use, default is sigmoid + + Returns + ------- + Model + Keras.Model, the pretrained VGG16 model + + Example + ------- model = VGG16_pretrained(input_shape = (224, 224, 3), n_classes = 1, output_activation = 'sigmoid', From 0a10867fb9971dcb563215cf669a9feb699e8ee0 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 18:34:19 +0000 Subject: [PATCH 16/22] Restructured unittests directory --- webscrapers/exeUnittests.cmd | 2 +- webscrapers/unittests/{ => utilities}/test_gen_urls.py | 0 webscrapers/unittests/{ => utilities}/test_scrape_srcs.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename webscrapers/unittests/{ => utilities}/test_gen_urls.py (100%) rename webscrapers/unittests/{ => utilities}/test_scrape_srcs.py (100%) diff --git a/webscrapers/exeUnittests.cmd b/webscrapers/exeUnittests.cmd index d468136..780f944 100644 --- a/webscrapers/exeUnittests.cmd +++ b/webscrapers/exeUnittests.cmd @@ -1 +1 @@ -call python -m unittest discover unittests \ No newline at end of file +call python -m unittest discover unittests\utilities \ No newline at end of file diff --git a/webscrapers/unittests/test_gen_urls.py b/webscrapers/unittests/utilities/test_gen_urls.py similarity index 100% rename from webscrapers/unittests/test_gen_urls.py rename to webscrapers/unittests/utilities/test_gen_urls.py diff --git a/webscrapers/unittests/test_scrape_srcs.py b/webscrapers/unittests/utilities/test_scrape_srcs.py similarity index 100% rename from webscrapers/unittests/test_scrape_srcs.py rename to webscrapers/unittests/utilities/test_scrape_srcs.py From 6effc82b6ef4ebc318c3ed207de03451134a3555 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 18:49:55 +0000 Subject: [PATCH 17/22] Updated UnionType with Union --- model/arch/crop_image.py | 6 ++--- model/arch/data_prep.py | 3 ++- model/arch/pad_image.py | 4 +-- model/arch/resize_image.py | 4 +-- model/exeUnittests.cmd | 1 + model/keras/LeNet5.py | 4 +-- model/keras/ResNet50_pretrained.py | 4 +-- model/keras/UNet.py | 2 +- model/keras/VGG16.py | 4 +-- model/keras/VGG16_pretrained.py | 4 +-- model/unittests/utilities/test_TimeIt.py | 34 ++++++++++++++++++++++++ model/utilities/TimeIt.py | 8 ++++-- model/utilities/plot_model.py | 10 ++----- 13 files changed, 61 insertions(+), 27 deletions(-) create mode 100644 model/exeUnittests.cmd create mode 100644 model/unittests/utilities/test_TimeIt.py diff --git a/model/arch/crop_image.py b/model/arch/crop_image.py index 6e40136..03960a4 100644 --- a/model/arch/crop_image.py +++ b/model/arch/crop_image.py @@ -1,12 +1,12 @@ import numpy as np from beartype import beartype -from types import UnionType +from typing import Union @beartype def crop_image( image_array:np.array, - pt1_wh:UnionType[tuple,list], - pt2_wh:UnionType[tuple,list] + pt1_wh:Union[tuple,list], + pt2_wh:Union[tuple,list] ) -> np.array: """Crops an image array to specified combination of two diagonal points diff --git a/model/arch/data_prep.py b/model/arch/data_prep.py index 7146941..f76a54f 100644 --- a/model/arch/data_prep.py +++ b/model/arch/data_prep.py @@ -16,7 +16,8 @@ from arch.resize_image import resize_image def data_prep(): - """Data preparation pipeline for generating the model training, testing and validation data. + """ + Data preparation pipeline for generating the model training, testing and validation data. Parameters ---------- diff --git a/model/arch/pad_image.py b/model/arch/pad_image.py index 2f8d786..d471c9c 100644 --- a/model/arch/pad_image.py +++ b/model/arch/pad_image.py @@ -1,11 +1,11 @@ import numpy as np from beartype import beartype -from types import UnionType +from typing import Union @beartype def pad_image( image_array:np.array, - pad_shape_wh:UnionType[list,tuple] + pad_shape_wh:Union[list,tuple] ) -> np.array: """ Pads an image array to a desired width and height diff --git a/model/arch/resize_image.py b/model/arch/resize_image.py index 68f2aaf..e5f80b3 100644 --- a/model/arch/resize_image.py +++ b/model/arch/resize_image.py @@ -1,12 +1,12 @@ import numpy as np import cv2 from beartype import beartype -from types import UnionType +from typing import Union @beartype def resize_image( image_array:np.array, - reshape_wh:UnionType[list,tuple], + reshape_wh:Union[list,tuple], interpolation=cv2.INTER_LINEAR, keep_3dim:bool=True ) -> np.array: diff --git a/model/exeUnittests.cmd b/model/exeUnittests.cmd new file mode 100644 index 0000000..780f944 --- /dev/null +++ b/model/exeUnittests.cmd @@ -0,0 +1 @@ +call python -m unittest discover unittests\utilities \ No newline at end of file diff --git a/model/keras/LeNet5.py b/model/keras/LeNet5.py index 9a4ce62..cf08cb4 100644 --- a/model/keras/LeNet5.py +++ b/model/keras/LeNet5.py @@ -1,12 +1,12 @@ from beartype import beartype -from types import UnionType +from typing import Union from keras.models import Model from keras.layers import Flatten, Dense, Input, Dropout from keras.layers import Conv2D, AveragePooling2D @beartype def LeNet5( - input_shape:UnionType[list,tuple]=(28,28,1), + input_shape:Union[list,tuple]=(28,28,1), n_classes:int=10, output_activation:str='softmax' ) -> Model: diff --git a/model/keras/ResNet50_pretrained.py b/model/keras/ResNet50_pretrained.py index 935f026..e561bf4 100644 --- a/model/keras/ResNet50_pretrained.py +++ b/model/keras/ResNet50_pretrained.py @@ -1,12 +1,12 @@ from beartype import beartype -from types import UnionType +from typing import Union from keras.layers import Dropout, Dense, Flatten from keras.applications import ResNet50 from keras.models import Model @beartype def ResNet50_pretrained( - input_shape:UnionType[list,tuple]=(224,224,3), + input_shape:Union[list,tuple]=(224,224,3), n_classes:int=2, output_activation:str='softmax' ) -> Model: diff --git a/model/keras/UNet.py b/model/keras/UNet.py index 8988f75..75873d6 100644 --- a/model/keras/UNet.py +++ b/model/keras/UNet.py @@ -1,5 +1,5 @@ from beartype import beartype -from types import UnionType +from typing import Union from keras.layers import Input, Conv2D, MaxPooling2D, Conv2DTranspose from keras.layers.merge import concatenate import tensorflow as tf diff --git a/model/keras/VGG16.py b/model/keras/VGG16.py index 57bbeb0..dc26bfe 100644 --- a/model/keras/VGG16.py +++ b/model/keras/VGG16.py @@ -1,11 +1,11 @@ from beartype import beartype -from types import UnionType +from typing import Union from keras.models import Model from keras.layers import Flatten, Dense, Input, Dropout from keras.layers import Conv2D, MaxPooling2D @beartype -def VGG16(input_shape:UnionType[list,tuple]=(224,224,3), +def VGG16(input_shape:Union[list,tuple]=(224,224,3), n_classes:int=1000, output_activation:str='softmax' ) -> Model: diff --git a/model/keras/VGG16_pretrained.py b/model/keras/VGG16_pretrained.py index 6896129..c7a0729 100644 --- a/model/keras/VGG16_pretrained.py +++ b/model/keras/VGG16_pretrained.py @@ -1,12 +1,12 @@ from beartype import beartype -from types import UnionType +from typing import Union from keras.layers import Dropout, Dense, GlobalMaxPooling2D from keras.applications import VGG16 from keras.models import Model @beartype def VGG16_pretrained( - input_shape:UnionType[list,tuple]=(224,224,3), + input_shape:Union[list,tuple]=(224,224,3), n_classes:int=1, output_activation:str='sigmoid' ) -> Model: diff --git a/model/unittests/utilities/test_TimeIt.py b/model/unittests/utilities/test_TimeIt.py new file mode 100644 index 0000000..3906dff --- /dev/null +++ b/model/unittests/utilities/test_TimeIt.py @@ -0,0 +1,34 @@ +import unittest +import os +import sys + +sys.path.append(os.path.join(os.getcwd(), "models")) + +import cons +from utilities.TimeIt import TimeIt + +parentKey="Test Parent Key" +subKey="Test Sub Key" +exp_log_keys = ["parentKey", "subKey", "stepTime", "cumulativeTime"] +obs_timeit = TimeIt() +obs_timeit.logTime(parentKey=parentKey, subKey=subKey) +obs_log_keys = obs_timeit.log.keys() + +class Test_TimeIt(unittest.TestCase): + """""" + + def setUp(self): + self.exp_log_keys = exp_log_keys + self.obs_log_keys = obs_log_keys + + def test_type(self): + self.assertEqual(type(self.obs_log_keys), type(self.exp_log_keys)) + + def test_len(self): + self.assertEqual(len(self.obs_log_keys), len(self.exp_log_keys)) + + def test_values(self): + self.assertEqual(self.obs_log_keys, self.exp_log_keys) + +if __name__ == "__main__": + unittest.main() diff --git a/model/utilities/TimeIt.py b/model/utilities/TimeIt.py index 637f3ae..c056c13 100644 --- a/model/utilities/TimeIt.py +++ b/model/utilities/TimeIt.py @@ -1,6 +1,6 @@ import logging from beartype import beartype -from types import UnionType +from typing import Union from time import time class TimeIt(): @@ -15,7 +15,11 @@ def __init__(self): self.cumulativeTime = 0.0 @beartype - def logTime(self, parentKey:str, subKey:UnionType[str,None]=None): + def logTime( + self, + parentKey:str, + subKey:Union[str,None]=None + ): """Sets a timestamp for code execution step Parameters diff --git a/model/utilities/plot_model.py b/model/utilities/plot_model.py index 5c703ed..316cdde 100644 --- a/model/utilities/plot_model.py +++ b/model/utilities/plot_model.py @@ -1,19 +1,13 @@ -# -*- coding: utf-8 -*- -""" -Created on Tue Feb 2 11:37:53 2021 -@author: oislen -""" - # load relevant libraries import os from beartype import beartype -from types import UnionType +from typing import Union from matplotlib import pyplot as plt @beartype def plot_model_fit( model_fit, - output_fdir:UnionType[str,None]=None, + output_fdir:Union[str,None]=None, show_plot:bool=True ): """ From be9b85cfffd166581b4df673a080085a0ad4f42d Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 18:52:21 +0000 Subject: [PATCH 18/22] Fixed TimeIt tests --- model/unittests/utilities/test_TimeIt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/unittests/utilities/test_TimeIt.py b/model/unittests/utilities/test_TimeIt.py index 3906dff..11cfdfa 100644 --- a/model/unittests/utilities/test_TimeIt.py +++ b/model/unittests/utilities/test_TimeIt.py @@ -12,7 +12,7 @@ exp_log_keys = ["parentKey", "subKey", "stepTime", "cumulativeTime"] obs_timeit = TimeIt() obs_timeit.logTime(parentKey=parentKey, subKey=subKey) -obs_log_keys = obs_timeit.log.keys() +obs_log_keys = list(obs_timeit.log[0].keys()) class Test_TimeIt(unittest.TestCase): """""" From 3d99ba4cc9fd8bb47e525bbea37593ef68eae304 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 20:33:00 +0000 Subject: [PATCH 19/22] Removed beartyping for image_array parameter --- model/utilities/plot_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/utilities/plot_image.py b/model/utilities/plot_image.py index 636b802..abdd2bf 100644 --- a/model/utilities/plot_image.py +++ b/model/utilities/plot_image.py @@ -5,7 +5,7 @@ @beartype def plot_image( - image_array:np.array, + image_array, output_fpath:Union[str,None]=None, show_plot:bool=True ): From a79a96cb93b045ea1182767ab2a2cc9cbcf3a7c9 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 20:34:02 +0000 Subject: [PATCH 20/22] Replaced UnionType with Union --- model/keras/AlexNet8.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/keras/AlexNet8.py b/model/keras/AlexNet8.py index f36019d..40b92e9 100644 --- a/model/keras/AlexNet8.py +++ b/model/keras/AlexNet8.py @@ -1,13 +1,13 @@ # load in relevant libraries from beartype import beartype -from types import UnionType +from typing import Union from keras.models import Model from keras.layers import Flatten, Dense, Input, Dropout from keras.layers import Conv2D, MaxPooling2D @beartype def AlexNet8( - input_shape:UnionType[list,tuple]=(227,227,3), + input_shape:Union[list,tuple]=(227,227,3), n_classes:int=1000, output_activation:str='softmax' ) -> Model: From 17f860bea8d57be1e7e2a5b6bf5584b9b08a9862 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sat, 8 Feb 2025 22:04:04 +0000 Subject: [PATCH 21/22] Removed cons parameter from plot preds call --- model/prg_keras_model.py | 2 +- model/prg_torch_model.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/model/prg_keras_model.py b/model/prg_keras_model.py index 371f790..30570cd 100644 --- a/model/prg_keras_model.py +++ b/model/prg_keras_model.py @@ -147,7 +147,7 @@ logging.info("Plot example test set predictions...") # plot random sample predictions - plot_preds(data=test_df, cons=cons, output_fpath=cons.keras_pred_images_fpath, show_plot=False) + plot_preds(data=test_df, output_fpath=cons.keras_pred_images_fpath, show_plot=False) timeLogger.logTime(parentKey="Plots", subKey="TestSetPredictions") logging.info("Generate a sample submission file for kaggle...") diff --git a/model/prg_torch_model.py b/model/prg_torch_model.py index 1dc57b1..2cc37d8 100644 --- a/model/prg_torch_model.py +++ b/model/prg_torch_model.py @@ -165,7 +165,7 @@ logging.info("Plot example test set predictions...") # plot random sample predictions - plot_preds(data=test_df, cons=cons, output_fpath=cons.torch_pred_images_fpath, show_plot=False) + plot_preds(data=test_df, output_fpath=cons.torch_pred_images_fpath, show_plot=False) timeLogger.logTime(parentKey="Plots", subKey="TestSetPredictions") logging.info("Generate a sample submission file for kaggle...") From 86b6e14433b77e15dd2551bfc75c04f7c3be8319 Mon Sep 17 00:00:00 2001 From: Oisin Date: Sun, 9 Feb 2025 10:47:57 +0000 Subject: [PATCH 22/22] Added executable script for running keras model training pipeline --- model/exeKerasModel.cmd | 1 + model/exeKerasModel.sh | 1 + 2 files changed, 2 insertions(+) create mode 100644 model/exeKerasModel.cmd create mode 100644 model/exeKerasModel.sh diff --git a/model/exeKerasModel.cmd b/model/exeKerasModel.cmd new file mode 100644 index 0000000..adaca20 --- /dev/null +++ b/model/exeKerasModel.cmd @@ -0,0 +1 @@ +call python -m pdb prg_keras_model.py --run_model_training --run_testset_prediction \ No newline at end of file diff --git a/model/exeKerasModel.sh b/model/exeKerasModel.sh new file mode 100644 index 0000000..4dd185d --- /dev/null +++ b/model/exeKerasModel.sh @@ -0,0 +1 @@ +python -m pdb prg_keras_model.py --run_model_training --run_testset_prediction \ No newline at end of file