From 4fadc9979cb949292598118d2b2cc8fc426f6051 Mon Sep 17 00:00:00 2001 From: Michal Baranowski Date: Tue, 1 Apr 2025 18:31:40 -0400 Subject: [PATCH 1/3] feat: filter gitlab files by regex --- tests/test_gitlab.py | 32 ++++++++++++++++++++++++++++++++ web3_utils/gitlab.py | 7 +++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/test_gitlab.py b/tests/test_gitlab.py index 2bba2e1..7b23ffd 100644 --- a/tests/test_gitlab.py +++ b/tests/test_gitlab.py @@ -45,6 +45,38 @@ def test_download_files_from_project(gitlab_instance, mocker): repo_tree_mock.assert_called_once_with(ref="main", path="test_dir", get_all=True) repo_blob_mock.assert_has_calls([mocker.call("file_id_1"), mocker.call("file_id_2")]) +def test_download_files_from_project_with_include_only_files(gitlab_instance, mocker): + gitlab, gitlab_mock = gitlab_instance + + repo_tree_mock = mocker.patch.object( + gitlab_mock.projects.get(1), + "repository_tree", + return_value=[ + {"id": "file_id_1", "name": "hoodi-file.txt", "type": "blob"}, + {"id": "file_id_2", "name": "holesky-file.txt", "type": "blob"}, + {"id": "file_id_3", "name": "not-hoodi.txt", "type": "blob"}, + ], + ) + + repo_blob_mock = mocker.patch.object( + gitlab_mock.projects.get(1), + "repository_blob", + side_effect=[ + {"content": base64.b64encode(b"file1 content").decode("utf-8")}, + {"content": base64.b64encode(b"file2 content").decode("utf-8")}, + ], + ) + + downloaded_files = gitlab.download_files_from_project(project_id=1, dir_path="test_dir", branch="main", include_only_files=["^hoodi", "^holesky"]) + + expected_paths = [ + os.path.join(os.getcwd(), "tmp", "public_keys", "hoodi-file.txt"), + os.path.join(os.getcwd(), "tmp", "public_keys", "holesky-file.txt"), + ] + assert downloaded_files == expected_paths + repo_tree_mock.assert_called_once_with(ref="main", path="test_dir", get_all=True) + repo_blob_mock.assert_has_calls([mocker.call("file_id_1"), mocker.call("file_id_2")]) + def test_prepare_temp_directory(gitlab_instance, mocker): gitlab, gitlab_mock = gitlab_instance diff --git a/web3_utils/gitlab.py b/web3_utils/gitlab.py index b5dc6cd..bad19b3 100644 --- a/web3_utils/gitlab.py +++ b/web3_utils/gitlab.py @@ -1,20 +1,23 @@ import base64 +import re import shutil from os import path, makedirs, getcwd from gitlab import Gitlab - +from typing import Optional class GitLab: def __init__(self, url: str, token: str, tmp_dir: str = path.join(getcwd(), "tmp", "public_keys")): self.client = Gitlab(private_token=token, url=url) self.tmp_dir = tmp_dir - def download_files_from_project(self, project_id: int, dir_path: str, branch: str = "master"): + def download_files_from_project(self, project_id: int, dir_path: str, branch: str = "master", include_only_files: Optional[list[str]] = None): # ensures that access token doesn't expire self.client.auth() project = self.client.projects.get(project_id) items = project.repository_tree(ref=branch, path=dir_path, get_all=True) + if include_only_files: + items = [item for item in items if any(re.match(pattern, item["name"]) for pattern in include_only_files)] self._prepare_temp_directory() From 763e167d6090c3c9e4e41954ccab7adb19b68810 Mon Sep 17 00:00:00 2001 From: Michal Baranowski Date: Tue, 1 Apr 2025 18:32:52 -0400 Subject: [PATCH 2/3] chore: format --- tests/test_gitlab.py | 5 ++++- web3_utils/gitlab.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_gitlab.py b/tests/test_gitlab.py index 7b23ffd..fbc8019 100644 --- a/tests/test_gitlab.py +++ b/tests/test_gitlab.py @@ -45,6 +45,7 @@ def test_download_files_from_project(gitlab_instance, mocker): repo_tree_mock.assert_called_once_with(ref="main", path="test_dir", get_all=True) repo_blob_mock.assert_has_calls([mocker.call("file_id_1"), mocker.call("file_id_2")]) + def test_download_files_from_project_with_include_only_files(gitlab_instance, mocker): gitlab, gitlab_mock = gitlab_instance @@ -67,7 +68,9 @@ def test_download_files_from_project_with_include_only_files(gitlab_instance, mo ], ) - downloaded_files = gitlab.download_files_from_project(project_id=1, dir_path="test_dir", branch="main", include_only_files=["^hoodi", "^holesky"]) + downloaded_files = gitlab.download_files_from_project( + project_id=1, dir_path="test_dir", branch="main", include_only_files=["^hoodi", "^holesky"] + ) expected_paths = [ os.path.join(os.getcwd(), "tmp", "public_keys", "hoodi-file.txt"), diff --git a/web3_utils/gitlab.py b/web3_utils/gitlab.py index bad19b3..8c76642 100644 --- a/web3_utils/gitlab.py +++ b/web3_utils/gitlab.py @@ -5,12 +5,15 @@ from gitlab import Gitlab from typing import Optional + class GitLab: def __init__(self, url: str, token: str, tmp_dir: str = path.join(getcwd(), "tmp", "public_keys")): self.client = Gitlab(private_token=token, url=url) self.tmp_dir = tmp_dir - def download_files_from_project(self, project_id: int, dir_path: str, branch: str = "master", include_only_files: Optional[list[str]] = None): + def download_files_from_project( + self, project_id: int, dir_path: str, branch: str = "master", include_only_files: Optional[list[str]] = None + ): # ensures that access token doesn't expire self.client.auth() From 50399485d525a901d8725dfe201a05e02445afde Mon Sep 17 00:00:00 2001 From: Michal Baranowski Date: Tue, 1 Apr 2025 18:40:27 -0400 Subject: [PATCH 3/3] ci: fix pipeline --- .github/workflows/sast.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml index f44c1e4..86f2cfa 100644 --- a/.github/workflows/sast.yml +++ b/.github/workflows/sast.yml @@ -19,7 +19,7 @@ jobs: with: output: reports type: python, depscan - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: reports path: reports