From ad5922a1287401267e9fda81aabb9587580d3bb3 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Thu, 24 Aug 2023 20:45:10 +0100 Subject: [PATCH 01/35] Add simple function and script for testing --- func.py | 2 ++ requirements.txt | 15 +++++++++++++++ test_func.py | 9 +++++++++ 3 files changed, 26 insertions(+) create mode 100644 func.py create mode 100644 requirements.txt create mode 100644 test_func.py diff --git a/func.py b/func.py new file mode 100644 index 0000000..46d0d23 --- /dev/null +++ b/func.py @@ -0,0 +1,2 @@ +def sum(x: float, y: float) -> float: + return x + y \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1b946d5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,15 @@ +annotated-types==0.5.0 +anyio==3.7.1 +fastapi==0.101.1 +idna==3.4 +numpy==1.25.2 +pandas==2.0.3 +pydantic==2.3.0 +pydantic_core==2.6.3 +python-dateutil==2.8.2 +pytz==2023.3 +six==1.16.0 +sniffio==1.3.0 +starlette==0.27.0 +typing_extensions==4.7.1 +tzdata==2023.3 diff --git a/test_func.py b/test_func.py new file mode 100644 index 0000000..981ef3b --- /dev/null +++ b/test_func.py @@ -0,0 +1,9 @@ +import unittest, func + +class Test(unittest.TestCase): + + def test_sum(self): + self.assertEqual(func.sum(2,3), 5) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From e26d3770ff12331af21eb0e631af12b41fbab795 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Fri, 25 Aug 2023 16:24:37 +0100 Subject: [PATCH 02/35] Organize src and test files and added github workflow for testing --- .github/workflows/ci.yaml | 40 ++++++++++++++++++++++++++++++ funcs/__init__.py | 0 func.py => funcs/func.py | 0 tests/__init__.py | 0 test_func.py => tests/test_func.py | 3 ++- 5 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 funcs/__init__.py rename func.py => funcs/func.py (100%) create mode 100644 tests/__init__.py rename test_func.py => tests/test_func.py (79%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..fde01ea --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + pull_request: + +jobs: + run-tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: + - "3.7" + - "3.8" + - "3.9" + - "3.10" + - "3.11" + + name: Test + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version : ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install -r requirements.txt + + - name: Run tests + run: | + python -m unittest discover + + diff --git a/funcs/__init__.py b/funcs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/func.py b/funcs/func.py similarity index 100% rename from func.py rename to funcs/func.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test_func.py b/tests/test_func.py similarity index 79% rename from test_func.py rename to tests/test_func.py index 981ef3b..2d37034 100644 --- a/test_func.py +++ b/tests/test_func.py @@ -1,4 +1,5 @@ -import unittest, func +import unittest +from funcs import func class Test(unittest.TestCase): From c1c849ef9b6c9dac5c519b3da74824a8781fb882 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Fri, 25 Aug 2023 16:28:41 +0100 Subject: [PATCH 03/35] Remove python 3.7 and 3.8 as they have no required distribution of numpy available --- .github/workflows/ci.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fde01ea..a32e040 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,8 +11,6 @@ jobs: matrix: os: [ubuntu-latest] python-version: - - "3.7" - - "3.8" - "3.9" - "3.10" - "3.11" @@ -35,6 +33,6 @@ jobs: - name: Run tests run: | - python -m unittest discover + python -m unittest discover -v From 31f9d6b158a45c76db3f3e87ce9ba431aa9c0f73 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Fri, 25 Aug 2023 16:32:35 +0100 Subject: [PATCH 04/35] Change github workflow to only run on pull_request --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a32e040..c022557 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,6 @@ name: CI on: - push: pull_request: jobs: From e8762635b74eae588f8f93e87d9f90ad7e42b614 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Fri, 25 Aug 2023 17:15:05 +0100 Subject: [PATCH 05/35] Add status badge to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ced3d09..e43ff15 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ # Helper app +[![CI](https://github.com/fungss/Dojo-Helper-App/actions/workflows/ci.yaml/badge.svg)](https://github.com/fungss/Dojo-Helper-App/actions/workflows/ci.yaml) This is an app to help the hosts of the fortnightly Dojo hosts do the common tasks quicky and effectively. This will include making the replit repos, placing invite links in the discord channel, coping the written code to the github archive and sending a link in the discord channel. Updates and tickets can be found on the projects page for bounty hunters to complete. From 333217d8dd2242e6e80862df950b30220c448a26 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Fri, 25 Aug 2023 17:19:11 +0100 Subject: [PATCH 06/35] Revise README layout --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e43ff15..86c46f6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# Helper app +# Helper app + [![CI](https://github.com/fungss/Dojo-Helper-App/actions/workflows/ci.yaml/badge.svg)](https://github.com/fungss/Dojo-Helper-App/actions/workflows/ci.yaml) + This is an app to help the hosts of the fortnightly Dojo hosts do the common tasks quicky and effectively. This will include making the replit repos, placing invite links in the discord channel, coping the written code to the github archive and sending a link in the discord channel. Updates and tickets can be found on the projects page for bounty hunters to complete. From d9063a5473976659e09dd0af6ba8042483ae9494 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Fri, 25 Aug 2023 17:50:25 +0100 Subject: [PATCH 07/35] Add test to check installed packages --- .github/workflows/ci.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c022557..08e1791 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,7 @@ name: CI on: + push: pull_request: jobs: @@ -30,6 +31,10 @@ jobs: run: | python -m pip install -r requirements.txt + - name: Check installed packages + run: | + python freeze -r requirements.txt + - name: Run tests run: | python -m unittest discover -v From bf4ec5569231e5daa61bb5b2cecb9f6eeb7bfa55 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Fri, 25 Aug 2023 17:52:41 +0100 Subject: [PATCH 08/35] Fix package-check error --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 08e1791..7511c53 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,9 +31,9 @@ jobs: run: | python -m pip install -r requirements.txt - - name: Check installed packages + - name: Check if installed packages confirm with requirements.txt run: | - python freeze -r requirements.txt + pip freeze -r requirements.txt - name: Run tests run: | From 32b85378d129fbb618b74bd8ab6605a85fec1abd Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Wed, 30 Aug 2023 15:23:26 +0100 Subject: [PATCH 09/35] Add comment to explain the usage of __init__.py --- funcs/__init__.py | 1 + tests/__init__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/funcs/__init__.py b/funcs/__init__.py index e69de29..2b8e6eb 100644 --- a/funcs/__init__.py +++ b/funcs/__init__.py @@ -0,0 +1 @@ +# placeholder file to modularize (to package) .py files under the directory "funcs", making the files importable. \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..61d14e5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1 @@ +# placeholder file to modularize (to package) .py files under the directory "tests", making the files importable. \ No newline at end of file From b4f91ae78f77c6c12ad9bf91c11382b6a673b7be Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Wed, 30 Aug 2023 15:25:05 +0100 Subject: [PATCH 10/35] Rename workflow name to Unit Test Linux --- .github/workflows/{ci.yaml => unit-test.yaml} | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) rename .github/workflows/{ci.yaml => unit-test.yaml} (51%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/unit-test.yaml similarity index 51% rename from .github/workflows/ci.yaml rename to .github/workflows/unit-test.yaml index 7511c53..5a37e33 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/unit-test.yaml @@ -1,4 +1,4 @@ -name: CI +name: Unit Tests Linux on: push: @@ -6,26 +6,27 @@ on: jobs: run-tests: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - python-version: - - "3.9" - - "3.10" - - "3.11" - - name: Test - runs-on: ${{ matrix.os }} + # strategy: + # fail-fast: false + # matrix: + # os: [ubuntu-latest] + # python-version: + # - "3.9" + # - "3.10" + # - "3.11" + + name: unit-test + # runs-on: ${{ matrix.os }} + runs-on: python:alpine steps: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version : ${{ matrix.python-version }} + # - name: Set up Python + # uses: actions/setup-python@v4 + # with: + # python-version : ${{ matrix.python-version }} - name: Install dependencies run: | From 6b3d60bfba2b17d6214f45206c5b82c3932b6eaa Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Wed, 30 Aug 2023 15:35:00 +0100 Subject: [PATCH 11/35] Comment out pull_request as trigger condition --- .github/workflows/unit-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 5a37e33..d486a98 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -2,7 +2,7 @@ name: Unit Tests Linux on: push: - pull_request: + # pull_request: jobs: run-tests: From ec814191660ef7c2be57e2c563a8cc0991fc18e5 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Wed, 30 Aug 2023 15:42:02 +0100 Subject: [PATCH 12/35] Rename badge to Unit Test Linux --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 86c46f6..c946ea6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Helper app -[![CI](https://github.com/fungss/Dojo-Helper-App/actions/workflows/ci.yaml/badge.svg)](https://github.com/fungss/Dojo-Helper-App/actions/workflows/ci.yaml) +[![Unit Test Linux](https://github.com/fungss/Dojo-Helper-App/actions/workflows/ci.yaml/badge.svg)](https://github.com/fungss/Dojo-Helper-App/actions/workflows/ci.yaml) This is an app to help the hosts of the fortnightly Dojo hosts do the common tasks quicky and effectively. This will include making the replit repos, placing invite links in the discord channel, coping the written code to the github archive and sending a link in the discord channel. Updates and tickets can be found on the projects page for bounty hunters to complete. From 73c6c53068bd31d2cb1e604f602e03a980d3fe97 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Wed, 30 Aug 2023 15:47:16 +0100 Subject: [PATCH 13/35] Update badge link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c946ea6..cfa73a6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Helper app -[![Unit Test Linux](https://github.com/fungss/Dojo-Helper-App/actions/workflows/ci.yaml/badge.svg)](https://github.com/fungss/Dojo-Helper-App/actions/workflows/ci.yaml) +[![Unit Tests Linux](https://github.com/fungss/Dojo-Helper-App/actions/workflows/unit-test.yaml/badge.svg?branch=main)](https://github.com/fungss/Dojo-Helper-App/actions/workflows/unit-test.yaml) This is an app to help the hosts of the fortnightly Dojo hosts do the common tasks quicky and effectively. This will include making the replit repos, placing invite links in the discord channel, coping the written code to the github archive and sending a link in the discord channel. Updates and tickets can be found on the projects page for bounty hunters to complete. From e181b9ca55ddf609dfde8d8e7b4999350dd29156 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Wed, 30 Aug 2023 15:48:38 +0100 Subject: [PATCH 14/35] Resume previous os setting to ensure actions job run successfully --- .github/workflows/unit-test.yaml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index d486a98..96eb036 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -1,4 +1,4 @@ -name: Unit Tests Linux +name: Unit Test Linux on: push: @@ -6,27 +6,27 @@ on: jobs: run-tests: - # strategy: - # fail-fast: false - # matrix: - # os: [ubuntu-latest] - # python-version: - # - "3.9" - # - "3.10" - # - "3.11" + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: + - "3.9" + - "3.10" + - "3.11" name: unit-test - # runs-on: ${{ matrix.os }} - runs-on: python:alpine + runs-on: ${{ matrix.os }} + # runs-on: python:alpine steps: - name: Checkout code uses: actions/checkout@v3 - # - name: Set up Python - # uses: actions/setup-python@v4 - # with: - # python-version : ${{ matrix.python-version }} + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version : ${{ matrix.python-version }} - name: Install dependencies run: | From 01895c46d898e71d2f38dffdffd025a5283d9776 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Wed, 30 Aug 2023 15:49:50 +0100 Subject: [PATCH 15/35] Remove python 3.7 in future unit test pipeline --- .github/workflows/unit-test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 96eb036..57ad213 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -11,7 +11,6 @@ jobs: matrix: os: [ubuntu-latest] python-version: - - "3.9" - "3.10" - "3.11" From c6f9f2766d7a05cb5947985406907cb1ad20e7a5 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Thu, 31 Aug 2023 17:13:29 +0100 Subject: [PATCH 16/35] Stage changes to README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cfa73a6..4136475 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Helper app -[![Unit Tests Linux](https://github.com/fungss/Dojo-Helper-App/actions/workflows/unit-test.yaml/badge.svg?branch=main)](https://github.com/fungss/Dojo-Helper-App/actions/workflows/unit-test.yaml) +[![Unit Test Linux](https://github.com/fungss/Dojo-Helper-App/actions/workflows/unit-test.yaml/badge.svg?branch=main)](https://github.com/fungss/Dojo-Helper-App/actions/workflows/unit-test.yaml) This is an app to help the hosts of the fortnightly Dojo hosts do the common tasks quicky and effectively. This will include making the replit repos, placing invite links in the discord channel, coping the written code to the github archive and sending a link in the discord channel. Updates and tickets can be found on the projects page for bounty hunters to complete. From 7262b1ac922bf774ecaeb77c84675097a4599115 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Thu, 31 Aug 2023 17:19:16 +0100 Subject: [PATCH 17/35] Update badge link in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4136475..d3474c1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Helper app -[![Unit Test Linux](https://github.com/fungss/Dojo-Helper-App/actions/workflows/unit-test.yaml/badge.svg?branch=main)](https://github.com/fungss/Dojo-Helper-App/actions/workflows/unit-test.yaml) +[![Unit Test Linux](https://github.com/Python-Dojo/Dojo-Helper-App/actions/workflows/unit-test.yaml/badge.svg?branch=main)](https://github.com/Python-Dojo/Dojo-Helper-App/actions/workflows/unit-test.yaml) This is an app to help the hosts of the fortnightly Dojo hosts do the common tasks quicky and effectively. This will include making the replit repos, placing invite links in the discord channel, coping the written code to the github archive and sending a link in the discord channel. Updates and tickets can be found on the projects page for bounty hunters to complete. From 3cebb8e5bef37ee48b0b6606704c8d7c244e0aa2 Mon Sep 17 00:00:00 2001 From: Ray <57572379+JustCallMeRay@users.noreply.github.com> Date: Thu, 31 Aug 2023 17:45:31 +0100 Subject: [PATCH 18/35] Remove python 3.10 support Was demeaned unneeded and would take too much sever time. --- .github/workflows/unit-test.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 57ad213..e846e84 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -11,7 +11,6 @@ jobs: matrix: os: [ubuntu-latest] python-version: - - "3.10" - "3.11" name: unit-test @@ -39,4 +38,3 @@ jobs: run: | python -m unittest discover -v - From 92236149bfa4987b2eb799465922a4b1fa2684ea Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Thu, 7 Sep 2023 23:31:28 +0100 Subject: [PATCH 19/35] Add package flake8 to requirements.txt --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index 1b946d5..8f6605a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,15 @@ annotated-types==0.5.0 anyio==3.7.1 fastapi==0.101.1 +flake8==6.1.0 idna==3.4 +mccabe==0.7.0 numpy==1.25.2 pandas==2.0.3 +pycodestyle==2.11.0 pydantic==2.3.0 pydantic_core==2.6.3 +pyflakes==3.1.0 python-dateutil==2.8.2 pytz==2023.3 six==1.16.0 From f9252f5e056ecae9a277299f65dca42e5d901d8b Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Fri, 8 Sep 2023 00:22:04 +0100 Subject: [PATCH 20/35] Add function-to-be-test and the corresponding unit test --- funcs/func.py | 7 ++++++- tests/test_func.py | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/funcs/func.py b/funcs/func.py index 46d0d23..a49f0e4 100644 --- a/funcs/func.py +++ b/funcs/func.py @@ -1,2 +1,7 @@ def sum(x: float, y: float) -> float: - return x + y \ No newline at end of file + return x + y + +def misformatted_func( +var1, var2, var3 +): + print(var1, var2, var3) diff --git a/tests/test_func.py b/tests/test_func.py index 2d37034..6989460 100644 --- a/tests/test_func.py +++ b/tests/test_func.py @@ -1,4 +1,5 @@ import unittest +from unittest.mock import patch from funcs import func class Test(unittest.TestCase): @@ -6,5 +7,10 @@ class Test(unittest.TestCase): def test_sum(self): self.assertEqual(func.sum(2,3), 5) + @patch('funcs.func.misformatted_func') + def test_misformatted_func(self, mock_misformatted_func): + func.misformatted_func("one", "two", "three") + mock_misformatted_func.assert_called_with("one", "two", "three") + if __name__ == "__main__": unittest.main() \ No newline at end of file From 90663acfebd065d33d0f6fe4a8916345d852fd51 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Fri, 8 Sep 2023 00:23:05 +0100 Subject: [PATCH 21/35] Add check format step in github workflow --- .github/workflows/unit-test.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index e846e84..2b7f910 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -33,6 +33,10 @@ jobs: - name: Check if installed packages confirm with requirements.txt run: | pip freeze -r requirements.txt + + - name: Check format without making corrections + run: | + flake8 funcs/*.py - name: Run tests run: | From c3e453923e29ec10db96f22f0c8bafd9c774be2d Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Fri, 8 Sep 2023 19:10:12 +0100 Subject: [PATCH 22/35] Update GitHub CI workflow to only run on pull request. --- .github/workflows/unit-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 2b7f910..f1c9bc9 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -1,8 +1,8 @@ name: Unit Test Linux on: - push: - # pull_request: + # push: + pull_request: jobs: run-tests: From 5a5e648f8711248c5e61bd6cf3421146ed80b214 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Mon, 18 Sep 2023 22:16:51 +0100 Subject: [PATCH 23/35] Back up misformatted py file as .txt as example --- funcs/func_misformatted.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 funcs/func_misformatted.txt diff --git a/funcs/func_misformatted.txt b/funcs/func_misformatted.txt new file mode 100644 index 0000000..a49f0e4 --- /dev/null +++ b/funcs/func_misformatted.txt @@ -0,0 +1,7 @@ +def sum(x: float, y: float) -> float: + return x + y + +def misformatted_func( +var1, var2, var3 +): + print(var1, var2, var3) From 546074e21b9d7f72f8437ca798640091866e1ee7 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Mon, 18 Sep 2023 22:17:44 +0100 Subject: [PATCH 24/35] Update py files formats according to flake8 --- funcs/__init__.py | 3 ++- funcs/func.py | 11 ++++++++--- tests/__init__.py | 3 ++- tests/test_func.py | 14 ++++++++++---- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/funcs/__init__.py b/funcs/__init__.py index 2b8e6eb..a9b1dd8 100644 --- a/funcs/__init__.py +++ b/funcs/__init__.py @@ -1 +1,2 @@ -# placeholder file to modularize (to package) .py files under the directory "funcs", making the files importable. \ No newline at end of file +# placeholder file to modularize (to package) .py files +# under the directory "funcs", making the files importable. diff --git a/funcs/func.py b/funcs/func.py index a49f0e4..436a0fd 100644 --- a/funcs/func.py +++ b/funcs/func.py @@ -1,7 +1,12 @@ def sum(x: float, y: float) -> float: return x + y -def misformatted_func( -var1, var2, var3 -): + +# def misformatted_func( +# var1, var2, var3 +# ): +# print(var1, var2, var3) + + +def corr_formatted_func(var1, var2, var3): print(var1, var2, var3) diff --git a/tests/__init__.py b/tests/__init__.py index 61d14e5..d35b63a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1,2 @@ -# placeholder file to modularize (to package) .py files under the directory "tests", making the files importable. \ No newline at end of file +# placeholder file to modularize (to package) .py files +# under the directory "tests", making the files importable. diff --git a/tests/test_func.py b/tests/test_func.py index 6989460..f1f0123 100644 --- a/tests/test_func.py +++ b/tests/test_func.py @@ -7,10 +7,16 @@ class Test(unittest.TestCase): def test_sum(self): self.assertEqual(func.sum(2,3), 5) - @patch('funcs.func.misformatted_func') - def test_misformatted_func(self, mock_misformatted_func): - func.misformatted_func("one", "two", "three") - mock_misformatted_func.assert_called_with("one", "two", "three") + # Any misformatted functions would not pass flake8 format checks. All related tests would not be executed. + # @patch('funcs.func.misformatted_func') + # def test_misformatted_func(self, mock_misformatted_func): + # func.misformatted_func("one", "two", "three") + # mock_misformatted_func.assert_called_with("one", "two", "three") + + @patch('funcs.func.corr_formatted_func') + def test_corr_formatted_func(self, mock_corr_formatted_func): + func.corr_formatted_func("one", "two", "three") + mock_corr_formatted_func.assert_called_with("one", "two", "three") if __name__ == "__main__": unittest.main() \ No newline at end of file From b2b1eebf47adca6c99234c1273190339e3d5cfc1 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Thu, 21 Sep 2023 20:05:50 +0100 Subject: [PATCH 25/35] Remove function and test for testing flake8 --- funcs/func.py | 6 ------ funcs/func_misformatted.txt | 7 ------- tests/test_func.py | 14 ++------------ 3 files changed, 2 insertions(+), 25 deletions(-) delete mode 100644 funcs/func_misformatted.txt diff --git a/funcs/func.py b/funcs/func.py index 436a0fd..2262e0a 100644 --- a/funcs/func.py +++ b/funcs/func.py @@ -2,11 +2,5 @@ def sum(x: float, y: float) -> float: return x + y -# def misformatted_func( -# var1, var2, var3 -# ): -# print(var1, var2, var3) - - def corr_formatted_func(var1, var2, var3): print(var1, var2, var3) diff --git a/funcs/func_misformatted.txt b/funcs/func_misformatted.txt deleted file mode 100644 index a49f0e4..0000000 --- a/funcs/func_misformatted.txt +++ /dev/null @@ -1,7 +0,0 @@ -def sum(x: float, y: float) -> float: - return x + y - -def misformatted_func( -var1, var2, var3 -): - print(var1, var2, var3) diff --git a/tests/test_func.py b/tests/test_func.py index f1f0123..0db8e65 100644 --- a/tests/test_func.py +++ b/tests/test_func.py @@ -7,16 +7,6 @@ class Test(unittest.TestCase): def test_sum(self): self.assertEqual(func.sum(2,3), 5) - # Any misformatted functions would not pass flake8 format checks. All related tests would not be executed. - # @patch('funcs.func.misformatted_func') - # def test_misformatted_func(self, mock_misformatted_func): - # func.misformatted_func("one", "two", "three") - # mock_misformatted_func.assert_called_with("one", "two", "three") - - @patch('funcs.func.corr_formatted_func') - def test_corr_formatted_func(self, mock_corr_formatted_func): - func.corr_formatted_func("one", "two", "three") - mock_corr_formatted_func.assert_called_with("one", "two", "three") - if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() + \ No newline at end of file From 78d7f6fa24e0bc7722127562f84a1cc586f67a74 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Thu, 21 Sep 2023 20:14:28 +0100 Subject: [PATCH 26/35] Remove function and test for testing flake8 --- funcs/func.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/funcs/func.py b/funcs/func.py index 2262e0a..8d531d0 100644 --- a/funcs/func.py +++ b/funcs/func.py @@ -1,6 +1,2 @@ def sum(x: float, y: float) -> float: return x + y - - -def corr_formatted_func(var1, var2, var3): - print(var1, var2, var3) From bdcc0d607b750a03e3a3e713070f84f9912ea235 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Thu, 21 Sep 2023 20:18:32 +0100 Subject: [PATCH 27/35] Remove test packages in requirements.txt --- requirements.txt | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8f6605a..9fcfdc1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,4 @@ -annotated-types==0.5.0 -anyio==3.7.1 -fastapi==0.101.1 -flake8==6.1.0 -idna==3.4 -mccabe==0.7.0 -numpy==1.25.2 -pandas==2.0.3 -pycodestyle==2.11.0 -pydantic==2.3.0 -pydantic_core==2.6.3 -pyflakes==3.1.0 -python-dateutil==2.8.2 -pytz==2023.3 -six==1.16.0 -sniffio==1.3.0 -starlette==0.27.0 -typing_extensions==4.7.1 -tzdata==2023.3 +distlib==0.3.6 +filelock==3.12.2 +platformdirs==3.8.0 +virtualenv==20.23.1 From 5ce29e4d53e3c65a261fc8145fb00801be0be51f Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Thu, 21 Sep 2023 20:30:34 +0100 Subject: [PATCH 28/35] Remove test packages in requirements.txt --- funcs/__init__.py | 3 +-- tests/__init__.py | 3 +-- tests/test_func.py | 6 +++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/funcs/__init__.py b/funcs/__init__.py index a9b1dd8..5a86c79 100644 --- a/funcs/__init__.py +++ b/funcs/__init__.py @@ -1,2 +1 @@ -# placeholder file to modularize (to package) .py files -# under the directory "funcs", making the files importable. +# placeholder file to modularize (to package) .py files under the directory "tests", making the files importable. diff --git a/tests/__init__.py b/tests/__init__.py index d35b63a..5a86c79 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,2 +1 @@ -# placeholder file to modularize (to package) .py files -# under the directory "tests", making the files importable. +# placeholder file to modularize (to package) .py files under the directory "tests", making the files importable. diff --git a/tests/test_func.py b/tests/test_func.py index 0db8e65..8618217 100644 --- a/tests/test_func.py +++ b/tests/test_func.py @@ -1,12 +1,12 @@ import unittest -from unittest.mock import patch from funcs import func + class Test(unittest.TestCase): def test_sum(self): - self.assertEqual(func.sum(2,3), 5) + self.assertEqual(func.sum(2, 3), 5) + if __name__ == "__main__": unittest.main() - \ No newline at end of file From 8372d6557d62a3f93d4c5ac06a1c730b7a220147 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Thu, 21 Sep 2023 20:54:03 +0100 Subject: [PATCH 29/35] Split workflows to different .yaml files, configured unit-test to be run only when linting return with success status, added setup.cfg for flake8 configuration --- .github/workflows/linting.yaml | 35 ++++++++++++++++++++++++++++++++ .github/workflows/unit-test.yaml | 13 +++++------- setup.cfg | 4 ++++ 3 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/linting.yaml create mode 100644 setup.cfg diff --git a/.github/workflows/linting.yaml b/.github/workflows/linting.yaml new file mode 100644 index 0000000..ed1bdac --- /dev/null +++ b/.github/workflows/linting.yaml @@ -0,0 +1,35 @@ +name: Linting Linux + +on: + # push: + pull_request: + +jobs: + run-tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: + - "3.11" + + name: linting + runs-on: ${{ matrix.os }} + # runs-on: python:alpine + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version : ${{ matrix.python-version }} + + - name: Install flake8 + run: | + python -m pip install flake8 + + - name: Check format without making corrections + run: | + flake8 diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index f1c9bc9..b3b688d 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -1,8 +1,10 @@ name: Unit Test Linux on: - # push: - pull_request: + workflow_run: + workflows: [Linting Linux] + types: + - completed jobs: run-tests: @@ -15,7 +17,7 @@ jobs: name: unit-test runs-on: ${{ matrix.os }} - # runs-on: python:alpine + if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: Checkout code @@ -33,12 +35,7 @@ jobs: - name: Check if installed packages confirm with requirements.txt run: | pip freeze -r requirements.txt - - - name: Check format without making corrections - run: | - flake8 funcs/*.py - name: Run tests run: | python -m unittest discover -v - diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..2aa0064 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 120 +exclude = + ./venv/ From d3672252050b4dce198a905a3ff512fa7c1a7911 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Fri, 22 Sep 2023 14:50:50 +0100 Subject: [PATCH 30/35] Remove commented code --- .github/workflows/linting.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/linting.yaml b/.github/workflows/linting.yaml index ed1bdac..e6e48b8 100644 --- a/.github/workflows/linting.yaml +++ b/.github/workflows/linting.yaml @@ -1,7 +1,6 @@ name: Linting Linux on: - # push: pull_request: jobs: @@ -15,7 +14,6 @@ jobs: name: linting runs-on: ${{ matrix.os }} - # runs-on: python:alpine steps: - name: Checkout code From ae56eb01b103d8f4c4ad20ea9233905a59a4cdb4 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Fri, 22 Sep 2023 15:11:32 +0100 Subject: [PATCH 31/35] Remove exclude= section in setup.cfg --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 2aa0064..6deafc2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,2 @@ [flake8] max-line-length = 120 -exclude = - ./venv/ From e98eb3457d34fdc07e2fb789161953b44a984495 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Sat, 23 Sep 2023 20:37:44 +0100 Subject: [PATCH 32/35] Exclude unnecessary directory in setup.cfg and added flake8 package in reqirements.txt for locally testing --- requirements.txt | 8 ++++---- setup.cfg | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9fcfdc1..51a2c28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -distlib==0.3.6 -filelock==3.12.2 -platformdirs==3.8.0 -virtualenv==20.23.1 +flake8==6.1.0 +mccabe==0.7.0 +pycodestyle==2.11.0 +pyflakes==3.1.0 diff --git a/setup.cfg b/setup.cfg index 6deafc2..66da313 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,10 @@ [flake8] max-line-length = 120 +exclude = + .git, + __pycache__, + docs/source/conf.py, + old, + build, + dist, + venv From 0cc39cc06c5d094a940f6effdef17d587f8ec8a0 Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Sat, 23 Sep 2023 20:39:16 +0100 Subject: [PATCH 33/35] Update workflow files --- .github/workflows/linting.yaml | 2 +- .github/workflows/unit-test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linting.yaml b/.github/workflows/linting.yaml index e6e48b8..947de61 100644 --- a/.github/workflows/linting.yaml +++ b/.github/workflows/linting.yaml @@ -4,7 +4,7 @@ on: pull_request: jobs: - run-tests: + linting: strategy: fail-fast: false matrix: diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index b3b688d..4b7dbf2 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -7,7 +7,7 @@ on: - completed jobs: - run-tests: + unit-test: strategy: fail-fast: false matrix: From 4a54b4d55acb300f448ed959edd36add888374a0 Mon Sep 17 00:00:00 2001 From: Ray <57572379+JustCallMeRay@users.noreply.github.com> Date: Mon, 25 Sep 2023 11:28:24 +0100 Subject: [PATCH 34/35] Update unit-tests to run on push --- .github/workflows/unit-test.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 4b7dbf2..ed47eb8 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -1,10 +1,6 @@ name: Unit Test Linux -on: - workflow_run: - workflows: [Linting Linux] - types: - - completed +on: push jobs: unit-test: From 8a187c4892dd5b640c8779b684c33352fa88961c Mon Sep 17 00:00:00 2001 From: Ronald Fung Date: Sat, 14 Oct 2023 12:47:42 +0100 Subject: [PATCH 35/35] Add ReplitScrapper and GithubArchiver classes --- .DS_Store | Bin 0 -> 6148 bytes .gitignore | 6 ++ Makefile | 11 +++ funcs/github_archiver.py | 121 ++++++++++++++++++++++++++++++++ funcs/replit_scrapper.py | 125 ++++++++++++++++++++++++++++++++++ main.py | 31 +++++++++ requirements.txt | 55 +++++++++++++++ tests/test_github_archiver.py | 78 +++++++++++++++++++++ tests/test_replit_scrapper.py | 46 +++++++++++++ 9 files changed, 473 insertions(+) create mode 100644 .DS_Store create mode 100644 Makefile create mode 100644 funcs/github_archiver.py create mode 100644 funcs/replit_scrapper.py create mode 100644 main.py create mode 100644 tests/test_github_archiver.py create mode 100644 tests/test_replit_scrapper.py diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..2f3651ffc2d53228b7711c1c88c98932eac82db1 GIT binary patch literal 6148 zcmeHK!A=4(5S;?*5;ftViF#_{6(Xx38ZR5vA8?Hx)F3NFH?A#V7ddQ5_O5=Aeu&@W zOk0f#T#eBVnZC|+rki<7+8qF3%|XWnC;@sZPt zc%Qd}ldwB%m6rCUNZO(3_EkdI>0-$FX(&2!*pvg&iB+zz7o4I~Y?ap1v{Gx-%G|B* z)JA2V?rv7gytcJH8Wo-8^^Luw)@^hr)* None: + self._project_name = project_name + self._file_paths = dict() + self._file_list = list() + self._commit_sha = "" + self.__github_access_token = github_access_token + self._commit_message = commit_message + + def get_project_name(self) -> str: + return self._project_name + + def identify_target_files(self) -> None: + print("GithubArchiver: Begin to parse target files...") + download_folder_path = "./screen-shots" + extracted_folder_path = os.path.join(download_folder_path, self.get_project_name()) + assert os.path.isdir(extracted_folder_path) is True, "Target folder does not exist" + assert len(os.listdir(extracted_folder_path)) != 0, "Target folder is empty" + + replit_junk = [ + '.cache', + '.upm', + '.replit', + 'poetry.lock', + 'pyproject.toml', + 'replit_zip_error_log.txt', + 'replit.nix', + ] + + # Walk through the directory and its subdirectories + for root, dirs, files in os.walk(extracted_folder_path): + for file in files: + file_full_path = os.path.join(root, file) + file_relative_path = file_full_path.replace(extracted_folder_path, self.get_project_name()) + if not any(excluded in file_relative_path for excluded in replit_junk): + self._file_paths[file_relative_path] = file_full_path + self._file_list.append(file_relative_path) + + print("GithubArchiver: Target files are parsed") + + def get_target_files(self) -> list: + return self._file_list + + def commit_to_github(self) -> None: + print("GithubArchiver: Begin to upload files to Github...") + assert len(self._file_list) != 0, "Target files are not identified" + auth = Auth.Token(self.__github_access_token) + g = Github(auth=auth) + repo = g.get_user().get_repo('The-Archive') + main_branch = repo.get_branch("main") + main_tree = repo.get_git_tree(sha=main_branch.commit.sha) + + tree = list() + for file_relative_path, file_full_path in self._file_paths.items(): + + with open(file_full_path, "rb") as file: + file_content = file.read() + + file_content_based64 = base64.b64encode(file_content) + + blob = repo.create_git_blob( + content=file_content_based64.decode('utf-8'), + encoding="base64" + ) + + tree.append( + InputGitTreeElement( + path=file_relative_path, + mode="100644", + type="blob", + sha=blob.sha, + ) + ) + + new_tree = repo.create_git_tree( + tree=tree, + base_tree=main_tree + ) + + commit = repo.create_git_commit( + message=self._commit_message, + tree=repo.get_git_tree(sha=new_tree.sha), + parents=[repo.get_git_commit(main_branch.commit.sha)], + ) + + archive_ref = repo.get_git_ref(ref='heads/main') + print(f"GithubArchiver: Archive_ref is {archive_ref}") + self._commit_sha = commit.sha + + # Commit to Github + archive_ref.edit(sha=commit.sha) + print("GithubArchiver: Upload complete") + + g.close() + + def get_commit_sha(self) -> str: + return self._commit_sha diff --git a/funcs/replit_scrapper.py b/funcs/replit_scrapper.py new file mode 100644 index 0000000..00c5dd2 --- /dev/null +++ b/funcs/replit_scrapper.py @@ -0,0 +1,125 @@ +from playwright.sync_api import sync_playwright +from playwright_stealth import stealth_sync + + +class ReplitScrapper(): + user_agent = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/116.0.0.0 " + "Safari/537.36 " + "Edg/116.0.1938.81" + ) + + def __init__(self, login_name, login_password): + self.__login_name = login_name + self.__login_password = login_password + self._replit_url = None + self._downloaded_filename = None + + def set_replit_url(self, replit_url) -> None: + if replit_url is None: + raise ValueError + self._replit_url = replit_url + + def get_replit_url(self) -> str: + if self._replit_url is None: + raise ValueError("Missing replit_url") + return self._replit_url + + def _set_downloaded_filename(self, filename) -> None: + if filename is None: + raise ValueError("ReplitScrapper._set_downloaded_filename() argument is None") + self._downloaded_filename = filename + + def get_downloaded_filename(self) -> str: + if self._downloaded_filename is None: + raise ValueError("Missing downloaded_filename") + return self._downloaded_filename + + def _visit_replit_repo(self, page) -> None: + response = page.goto(self.get_replit_url(), wait_until="domcontentloaded") + if response.status != 200: + if response.status == 404: + print(f"response.status = {response.status}") + raise ValueError("Invalid replit_url") + else: + print(f"response.status = {response.status}") + raise ValueError("ReplitScrapper._visit_replit_repo() something other than 404 happened") + + def _login_replit(self, page) -> None: + # Login + page.goto('https://replit.com/login', wait_until="domcontentloaded") + page.screenshot(path="./screen-shots/replit.png") + url_init = "https://identitytoolkit.googleapis.com/v1/accounts" + with page.expect_response(lambda response: url_init in response.url) as response_info: + page.locator( + "xpath=/html/body/div[1]/div/div[2]/div/main/div[2]/div/form/div[1]/input" + ).fill(self.__login_name) + page.locator( + "xpath=/html/body/div[1]/div/div[2]/div/main/div[2]/div/form/div[2]/div/input" + ).fill(self.__login_password) + page.locator( + "xpath=/html/body/div[1]/div/div[2]/div/main/div[2]/div/form/div[3]/button" + ).click() + response = response_info.value + if response.status != 200: + print(response) + if response.status == 400: + print(f"response.status = {response.status}") + raise ValueError("Invalid login credentials") + else: + print(f"response.status = {response.status}") + raise ValueError("ReplitScrapper._login_replit() something other than 401 happened") + page.wait_for_url("https://replit.com/~") + page.screenshot(path="./screen-shots/replit_after_login.png") + + def _download_as_zip(self, page) -> None: + # Wait for page load + page.locator( + "xpath=/html/body/div[1]/div[1]/div[1]/div[2]/div/div[1]/div/div[3]/div/div[1]/button/div/span" + ).wait_for() + while page.locator( + "xpath=/html/body/div[1]/div[1]/div[1]/div[2]/header/div[2]/button" + ).text_content() != "Run": + print(page.locator( + "xpath=/html/body/div[1]/div[1]/div[1]/div[2]/header/div[2]/button" + ).text_content()) + page.wait_for_timeout(2000) + page.screenshot(path="./screen-shots/target_page.png") + + # Begin downloading + page.locator( + "xpath=/html/body/div[1]/div[1]/div[1]/div[2]/div/div[1]/div/div[2]/div[1]/div[1]/div/button[3]" + ).click() + with page.expect_download() as download_info: + page.locator( + "xpath=/html/body/div[@class='css-1o92kwk']//div[@id='item-4']//div[@class='css-1l2rn59']" + ).click() + download = download_info.value + self._set_downloaded_filename(download.suggested_filename) + download.save_as(f"./screen-shots/{download.suggested_filename}") + + def run(self): + print("ReplitScrapper: Begin downloading repo files...") + with sync_playwright() as p: + # Context setup + browser = p.chromium.launch(slow_mo=50) + # browser = p.chromium.launch(headless=False + # , slow_mo=50 + # ) + context = browser.new_context(user_agent=ReplitScrapper.user_agent) + page = context.new_page() + stealth_sync(page) + + # Login replit + self._login_replit(page) + + # Download repo files as zip + self._visit_replit_repo(page) + self._download_as_zip(page) + + # Clean-up + context.close() + browser.close() + print("ReplitScrapper: Download complete") diff --git a/main.py b/main.py new file mode 100644 index 0000000..c56f249 --- /dev/null +++ b/main.py @@ -0,0 +1,31 @@ +from funcs.replit_scrapper import ReplitScrapper +from funcs.github_archiver import GithubArchiver +import os +import zipfile +from dotenv import load_dotenv +load_dotenv() + +WDIR = os.path.abspath(os.path.dirname(__name__)) + +if __name__ == "__main__": + test_url = "https://replit.com/@pythondojoarchi/SlipperyGargantuanDebuggers" + project_name = "SlipperyGargantuanDebuggers" + + # Download repo files as zip + scrapper = ReplitScrapper(login_name=os.environ['EMAIL'], login_password=os.environ['PASSWORD']) + scrapper.set_replit_url(test_url) + scrapper.run() + + # Unzip downloaded zip file + download_folder_path = os.path.join(WDIR, "screen-shots") + full_file_path = os.path.join(download_folder_path, project_name+".zip") + extracted_folder_path = os.path.join(download_folder_path, project_name) + zipfile.ZipFile(full_file_path).extractall(extracted_folder_path) + + # Commit target files to Github + archiver = GithubArchiver( + project_name=project_name, + github_access_token=os.environ['GITHUB_ACCESS_TOKEN'] + ) + archiver.identify_target_files() + archiver.commit_to_github() diff --git a/requirements.txt b/requirements.txt index 51a2c28..d05daec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,59 @@ +appnope==0.1.3 +asttokens==2.4.1 +certifi==2023.11.17 +cffi==1.16.0 +charset-normalizer==3.3.2 +comm==0.2.0 +cryptography==41.0.7 +debugpy==1.8.0 +decorator==5.1.1 +Deprecated==1.2.14 +executing==2.0.1 flake8==6.1.0 +greenlet==3.0.1 +idna==3.6 +iniconfig==2.0.0 +ipykernel==6.27.1 +ipython==8.19.0 +jedi==0.19.1 +jupyter_client==8.6.0 +jupyter_core==5.5.1 +matplotlib-inline==0.1.6 mccabe==0.7.0 +nest-asyncio==1.5.8 +packaging==23.2 +parso==0.8.3 +pexpect==4.9.0 +platformdirs==4.1.0 +playwright==1.40.0 +playwright-stealth==1.0.6 +pluggy==1.3.0 +prompt-toolkit==3.0.43 +psutil==5.9.7 +ptyprocess==0.7.0 +pure-eval==0.2.2 pycodestyle==2.11.0 +pycparser==2.21 +pyee==11.0.1 pyflakes==3.1.0 +PyGithub==2.1.1 +Pygments==2.17.2 +PyJWT==2.8.0 +PyNaCl==1.5.0 +pytest==7.4.3 +pytest-base-url==2.0.0 +pytest-playwright==0.4.3 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-slugify==8.0.1 +pyzmq==25.1.2 +requests==2.31.0 +six==1.16.0 +stack-data==0.6.3 +text-unidecode==1.3 +tornado==6.4 +traitlets==5.14.0 +typing_extensions==4.9.0 +urllib3==2.1.0 +wcwidth==0.2.12 +wrapt==1.16.0 diff --git a/tests/test_github_archiver.py b/tests/test_github_archiver.py new file mode 100644 index 0000000..75682dc --- /dev/null +++ b/tests/test_github_archiver.py @@ -0,0 +1,78 @@ +# from github import Github, Auth +import unittest +from funcs.github_archiver import GithubArchiver +import os +from dotenv import load_dotenv +load_dotenv() + + +class Test(unittest.TestCase): + + def test_archiver_raise_error_if_target_folder_does_not_exist(self): + archiver = GithubArchiver( + project_name="directory_not_exist", + github_access_token=os.environ['GITHUB_ACCESS_TOKEN'] + ) + with self.assertRaises(AssertionError) as ctx_manager: + archiver.identify_target_files() + self.assertEqual(str(ctx_manager.exception), "Target folder does not exist") + + def test_archiver_raise_error_if_target_folder_is_empty(self): + archiver = GithubArchiver( + project_name="empty_folder", + github_access_token=os.environ['GITHUB_ACCESS_TOKEN'] + ) + with self.assertRaises(AssertionError) as ctx_manager: + archiver.identify_target_files() + self.assertEqual(str(ctx_manager.exception), "Target folder is empty") + + # def test_archiver_return_list_of_target_files(self): + # target_list = [ + # "SlipperyGargantuanDebuggers/test-README.md", + # "SlipperyGargantuanDebuggers/road.jpg", + # "SlipperyGargantuanDebuggers/test.py", + # "SlipperyGargantuanDebuggers/main.py", + # "SlipperyGargantuanDebuggers/Group-1/test-1.txt", + # "SlipperyGargantuanDebuggers/Group-2/test-2.txt", + # ] + + # archiver = GithubArchiver( + # project_name="SlipperyGargantuanDebuggers", + # xxwgithub_access_token=os.environ['GITHUB_ACCESS_TOKEN'] + # ) + # archiver.identify_target_files() + # # https://stackoverflow.com/questions/12813633/how-to-assert-two-list-contain-the-same-elements-in-python + # self.assertCountEqual(archiver.get_target_files(), target_list) + + # def test_archiver_raise_error_if_target_files_not_set(self): + # archiver = GithubArchiver( + # project_name="SlipperyGargantuanDebuggers", + # github_access_token=os.environ['GITHUB_ACCESS_TOKEN'] + # ) + # with self.assertRaises(AssertionError) as ctx_manager: + # archiver.commit_to_github() + + # self.assertEqual(str(ctx_manager.exception), "Target files are not identified") + + # def test_archiver_upload_target_files_to_github(self): + # archiver = GithubArchiver( + # project_name="SlipperyGargantuanDebuggers", + # github_access_token=os.environ['GITHUB_ACCESS_TOKEN'] + # ) + # archiver.identify_target_files() + # archiver.commit_to_github() + + # auth = Auth.Token(os.environ['GITHUB_ACCESS_TOKEN']) + # g = Github(auth=auth) + # repo = g.get_user().get_repo('The-Archive') + # commit = repo.get_commit(archiver.get_commit_sha()) + # target_list = list() + # for file in commit.files: + # target_list.append(file.filename) + # g.close() + + # self.assertCountEqual(archiver.get_target_files(), target_list) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_replit_scrapper.py b/tests/test_replit_scrapper.py new file mode 100644 index 0000000..34b902c --- /dev/null +++ b/tests/test_replit_scrapper.py @@ -0,0 +1,46 @@ +import unittest +from funcs.replit_scrapper import ReplitScrapper +# import os +from dotenv import load_dotenv +load_dotenv() + + +class Test(unittest.TestCase): + + def test_scrapper_raise_value_error_when_replit_url_not_set(self): + scrapper = ReplitScrapper(login_name=None, login_password=None) + with self.assertRaises(ValueError) as ctx_manager: + scrapper.get_replit_url() + self.assertEqual(str(ctx_manager.exception), 'Missing replit_url') + + def test_scrapper_return_replit_url(self): + test_url = "https://replit.com/@pythondojoarchi/SlipperyGargantuanDebuggers" + + scrapper = ReplitScrapper(login_name=None, login_password=None) + scrapper.set_replit_url(test_url) + self.assertEqual(scrapper.get_replit_url(), test_url) + + # Commented out to avoid replit acount freezes + # def test_scrapper_login_with_invalid_credentials(self): + # scrapper = ReplitScrapper(login_name = os.environ['EMAIL'], login_password = "ThisIsNotTheCorrectPassword") + # with self.assertRaises(ValueError) as ctx_manager: + # scrapper.run() + # self.assertEqual(str(ctx_manager.exception), 'Invalid login credentials') + + # def test_scrapper_download_repo_as_zip(self): + # test_url = "https://replit.com/@pythondojoarchi/SlipperyGargantuanDebuggers" + # target_zip_name = "SlipperyGargantuanDebuggers.zip" + # WDIR = os.path.abspath(os.path.dirname(__name__)) + # full_target_file_path = os.path.join(WDIR, "screen-shots", target_zip_name) + # print(full_target_file_path) + + # scrapper = ReplitScrapper(login_name=os.environ['EMAIL'], login_password=os.environ['PASSWORD']) + # scrapper.set_replit_url(test_url) + # scrapper.run() + + # print(scrapper.get_downloaded_filename()) + # self.assertTrue(os.path.exists(full_target_file_path)) + + +if __name__ == "__main__": + unittest.main()