From 2e58a7826782d3ba128b7b1523b0c7940ba1cf48 Mon Sep 17 00:00:00 2001 From: Marvin Davila Date: Thu, 20 Feb 2025 20:12:29 -0500 Subject: [PATCH 1/3] add GetTagShasStream --- tap_github/repository_streams.py | 47 ++++++++++++++++++++++++++++++++ tap_github/streams.py | 2 ++ 2 files changed, 49 insertions(+) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 1694b9ad..2b8b2ddf 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2625,6 +2625,53 @@ class TagsStream(GitHubRestStream): th.Property("tarball_url", th.StringType), th.Property("node_id", th.StringType), ).to_dict() + + def get_child_context(self, record: dict, context: dict | None) -> dict: + """Return a child context object from the record and optional provided context. + By default, will return context if provided and otherwise the record dict. + Developers may override this behavior to send specific information to child + streams for context. + """ + return { + "org": context["org"] if context else None, + "repo": context["repo"] if context else None, + "tag_name": record["name"], + "repo_id": context["repo_id"] if context else None, + } + +class GetTagShasStream(GitHubRestStream): + """A stream dedicated to fetching tag shas of a tag in a repository. + + API docs: https://docs.github.com/en/rest/git/refs#get-a-reference + """ + + name = "get_tag_shas" + path = "/repos/{org}/{repo}/git/ref/tags/{tag_name}" + primary_keys: ClassVar[list[str]] = ["node_id"] + parent_stream_type = TagsStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org", "tag_name"] + tolerated_http_errors: ClassVar[list[int]] = [404] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("tag_name", th.StringType), + # Tag Sha Details + th.Property("ref", th.StringType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property( + "object", + th.ObjectType( + th.Property("type", th.StringType), + th.Property("sha", th.StringType), + th.Property("url", th.StringType), + ), + ), + ).to_dict() class DeploymentsStream(GitHubRestStream): diff --git a/tap_github/streams.py b/tap_github/streams.py index a14ac323..da0230b6 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -24,6 +24,7 @@ DeploymentStatusesStream, EventsStream, ExtraMetricsStream, + GetTagShasStream, IssueCommentsStream, IssueEventsStream, IssuesStream, @@ -88,6 +89,7 @@ def __init__(self, valid_queries: set[str], streams: list[type[Stream]]) -> None DeploymentsStream, DeploymentStatusesStream, EventsStream, + GetTagShasStream, IssueCommentsStream, IssueEventsStream, IssuesStream, From 1d7481428f98df1bb53df6b393efecd4e250b7b3 Mon Sep 17 00:00:00 2001 From: Marvin Davila Date: Thu, 20 Feb 2025 20:18:40 -0500 Subject: [PATCH 2/3] add TagDetailsStream --- tap_github/repository_streams.py | 65 ++++++++++++++++++++++++++++++++ tap_github/streams.py | 2 + 2 files changed, 67 insertions(+) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 2b8b2ddf..0aca768a 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2673,6 +2673,71 @@ class GetTagShasStream(GitHubRestStream): ), ).to_dict() + def get_child_context(self, record: dict, context: dict | None) -> dict: + """Return a child context object from the record and optional provided context. + By default, will return context if provided and otherwise the record dict. + Developers may override this behavior to send specific information to child + streams for context. + """ + return { + "org": context["org"] if context else None, + "repo": context["repo"] if context else None, + "tag_sha": record["object"]["sha"] if record.get("object") else None, + "repo_id": context["repo_id"] if context else None, + } + + +class TagDetailsStream(GitHubRestStream): + """A stream dedicated to fetching details of a tag in a repository.""" + + name = "tag_details" + path = "/repos/{org}/{repo}/git/tags/{tag_sha}" + primary_keys: ClassVar[list[str]] = ["node_id"] + parent_stream_type = GetTagShasStream + ignore_parent_replication_key = True + state_partitioning_keys: ClassVar[list[str]] = ["repo", "org", "tag_sha"] + tolerated_http_errors: ClassVar[list[int]] = [404] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + th.Property("tag_sha", th.StringType), + # Tag Details + th.Property("node_id", th.StringType), + th.Property("tag", th.StringType), + th.Property("sha", th.StringType), + th.Property("url", th.StringType), + th.Property("message", th.StringType), + th.Property( + "tagger", + th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.DateTimeType), + ), + ), + th.Property( + "object", + th.ObjectType( + th.Property("type", th.StringType), + th.Property("sha", th.StringType), + th.Property("url", th.StringType), + ), + ), + th.Property( + "verification", + th.ObjectType( + th.Property("verified", th.BooleanType), + th.Property("reason", th.StringType), + th.Property("signature", th.StringType), + th.Property("payload", th.StringType), + th.Property("verified_at", th.StringType), + ), + ), + ).to_dict() + class DeploymentsStream(GitHubRestStream): """A stream dedicated to fetching deployments in a repository.""" diff --git a/tap_github/streams.py b/tap_github/streams.py index da0230b6..6494d750 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -47,6 +47,7 @@ StargazersStream, StatsContributorsStream, TagsStream, + TagDetailsStream, TrafficClonesStream, TrafficPageViewsStream, TrafficReferralPathsStream, @@ -113,6 +114,7 @@ def __init__(self, valid_queries: set[str], streams: list[type[Stream]]) -> None StargazersStream, StatsContributorsStream, TagsStream, + TagDetailsStream, TrafficClonesStream, TrafficPageViewsStream, TrafficReferralPathsStream, From 02ba0433d9160abd853de82b7286ba6e12ec4208 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 21 Feb 2025 01:22:41 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tap_github/repository_streams.py | 7 ++++--- tap_github/streams.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 0aca768a..23946d3d 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2625,7 +2625,7 @@ class TagsStream(GitHubRestStream): th.Property("tarball_url", th.StringType), th.Property("node_id", th.StringType), ).to_dict() - + def get_child_context(self, record: dict, context: dict | None) -> dict: """Return a child context object from the record and optional provided context. By default, will return context if provided and otherwise the record dict. @@ -2638,10 +2638,11 @@ def get_child_context(self, record: dict, context: dict | None) -> dict: "tag_name": record["name"], "repo_id": context["repo_id"] if context else None, } - + + class GetTagShasStream(GitHubRestStream): """A stream dedicated to fetching tag shas of a tag in a repository. - + API docs: https://docs.github.com/en/rest/git/refs#get-a-reference """ diff --git a/tap_github/streams.py b/tap_github/streams.py index 6494d750..537a38be 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -46,8 +46,8 @@ StargazersGraphqlStream, StargazersStream, StatsContributorsStream, - TagsStream, TagDetailsStream, + TagsStream, TrafficClonesStream, TrafficPageViewsStream, TrafficReferralPathsStream,