From e9b3dd74d0117261b3cd4891589aff2d703fcfc6 Mon Sep 17 00:00:00 2001 From: Cory Bullinger Date: Tue, 13 Jan 2026 16:13:49 -0500 Subject: [PATCH 1/3] Fix: Add refs/heads/ prefix to BranchPath in workflow processor The workflow processor was creating UploadKey with BranchPath set to just the branch name (e.g., 'main') instead of the full ref path (e.g., 'refs/heads/main'). This caused GitHub API calls to fail with 404 errors when trying to access the branch ref. This fix ensures BranchPath is always set with the 'refs/heads/' prefix, consistent with how it's used throughout the rest of the codebase. --- services/workflow_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/workflow_processor.go b/services/workflow_processor.go index a766907..ee1b9bc 100644 --- a/services/workflow_processor.go +++ b/services/workflow_processor.go @@ -340,7 +340,7 @@ func (wp *workflowProcessor) addToUploadQueue( // Create upload key key := UploadKey{ RepoName: workflow.Destination.Repo, - BranchPath: workflow.Destination.Branch, + BranchPath: "refs/heads/" + workflow.Destination.Branch, } // Get existing entries from FileStateService From 5a7d6bcacdd28ff6c55df222546ece7ecf861c5c Mon Sep 17 00:00:00 2001 From: Cory Bullinger Date: Wed, 14 Jan 2026 16:59:23 -0500 Subject: [PATCH 2/3] Fix org-specific GitHub API access for multi-org installations - Add GetRestClientForOrg() to get installation-specific tokens - Fix GraphQL query to use node(id:) instead of repository(owner:) - Update RetrieveFileContentsWithConfigAndBranch to use org-specific client - Remove refs/heads/ prefix duplication in workflow processor - Fixes 404 errors when accessing repos in different orgs --- services/github_auth.go | 38 ++++++++++++++++++++++++++++++ services/github_read.go | 5 ++-- services/github_write_to_target.go | 28 ++++++++++++++++++++-- services/webhook_handler_new.go | 6 ++++- services/workflow_processor.go | 2 +- 5 files changed, 73 insertions(+), 6 deletions(-) diff --git a/services/github_auth.go b/services/github_auth.go index 683463f..b5a5aac 100644 --- a/services/github_auth.go +++ b/services/github_auth.go @@ -287,6 +287,44 @@ func GetGraphQLClient() (*graphql.Client, error) { return client, nil } +// GetGraphQLClientForOrg returns a GitHub GraphQL API client authenticated for a specific organization +func GetGraphQLClientForOrg(org string) (*graphql.Client, error) { + // Check if we have a cached token for this org + if token, ok := installationTokenCache[org]; ok && token != "" { + client := graphql.NewClient("https://api.github.com/graphql", &http.Client{ + Transport: &transport{token: token}, + }) + return client, nil + } + + // Get installation ID for the organization + installationID, err := getInstallationIDForOrg(org) + if err != nil { + return nil, fmt.Errorf("failed to get installation ID for org %s: %w", org, err) + } + + // Get JWT token + token, err := getOrRefreshJWT() + if err != nil { + return nil, fmt.Errorf("failed to get JWT: %w", err) + } + + // Get installation access token + installationToken, err := getInstallationAccessToken(installationID, token, HTTPClient) + if err != nil { + return nil, fmt.Errorf("failed to get installation token for org %s: %w", org, err) + } + + // Cache the token + installationTokenCache[org] = installationToken + + // Create and return client + client := graphql.NewClient("https://api.github.com/graphql", &http.Client{ + Transport: &transport{token: installationToken}, + }) + return client, nil +} + // getOrRefreshJWT returns a valid JWT token, generating a new one if expired func getOrRefreshJWT() (string, error) { // Check if we have a valid cached JWT diff --git a/services/github_read.go b/services/github_read.go index e213712..716c575 100644 --- a/services/github_read.go +++ b/services/github_read.go @@ -26,9 +26,10 @@ func GetFilesChangedInPr(owner string, repo string, pr_number int) ([]ChangedFil } } - client, err := GetGraphQLClient() + // Use org-specific client to ensure we have the right installation token + client, err := GetGraphQLClientForOrg(owner) if err != nil { - return nil, fmt.Errorf("failed to get GraphQL client: %w", err) + return nil, fmt.Errorf("failed to get GraphQL client for org %s: %w", owner, err) } ctx := context.Background() diff --git a/services/github_write_to_target.go b/services/github_write_to_target.go index e77066e..79b2ef6 100644 --- a/services/github_write_to_target.go +++ b/services/github_write_to_target.go @@ -45,6 +45,25 @@ func normalizeRepoName(repoName string) string { return repoOwner() + "/" + repoName } +// normalizeRefPath ensures a ref path is in the correct format for different GitHub API calls. +// For GetRef: expects "heads/main" (no "refs/" prefix) +// For UpdateRef: expects "refs/heads/main" (full ref path) +func normalizeRefPath(branchPath string, fullPath bool) string { + // Strip "refs/" prefix if present + refPath := strings.TrimPrefix(branchPath, "refs/") + + // Ensure "heads/" prefix exists (unless it's a tag) + if !strings.HasPrefix(refPath, "heads/") && !strings.HasPrefix(refPath, "tags/") { + refPath = "heads/" + refPath + } + + // Add "refs/" prefix back if full path is needed + if fullPath { + return "refs/" + refPath + } + return refPath +} + // AddFilesToTargetRepoBranch uploads files to the target repository branch // using the specified commit strategy (direct or via pull request). func AddFilesToTargetRepoBranch() { @@ -344,8 +363,11 @@ func createCommitTree(ctx context.Context, client *github.Client, targetBranch U retryDelay := time.Duration(initialRetryDelay) * time.Millisecond + // GetRef expects "heads/main" format (no "refs/" prefix) + refPath := normalizeRefPath(targetBranch.BranchPath, false) + for attempt := 1; attempt <= maxRetries; attempt++ { - ref, _, err = client.Git.GetRef(ctx, owner, repoName, targetBranch.BranchPath) + ref, _, err = client.Git.GetRef(ctx, owner, repoName, refPath) if err == nil && ref != nil { break // Success } @@ -405,8 +427,10 @@ func createCommit(ctx context.Context, client *github.Client, targetBranch Uploa } // Update branch ref directly (no second GET) + // UpdateRef expects full ref path "refs/heads/main" + fullRefPath := normalizeRefPath(targetBranch.BranchPath, true) ref := &github.Reference{ - Ref: github.String(targetBranch.BranchPath), // e.g., "refs/heads/main" + Ref: github.String(fullRefPath), Object: &github.GitObject{SHA: github.String(newCommit.GetSHA())}, } if _, _, err := client.Git.UpdateRef(ctx, owner, repoName, ref, false); err != nil { diff --git a/services/webhook_handler_new.go b/services/webhook_handler_new.go index dd7ec04..a392677 100644 --- a/services/webhook_handler_new.go +++ b/services/webhook_handler_new.go @@ -45,7 +45,11 @@ func simpleVerifySignature(sigHeader string, body, secret []byte) bool { // RetrieveFileContentsWithConfigAndBranch fetches file contents from a specific branch func RetrieveFileContentsWithConfigAndBranch(ctx context.Context, filePath string, branch string, repoOwner string, repoName string) (*github.RepositoryContent, error) { - client := GetRestClient() + // Use org-specific client to ensure we have the right installation token + client, err := GetRestClientForOrg(repoOwner) + if err != nil { + return nil, fmt.Errorf("failed to get GitHub client for org %s: %w", repoOwner, err) + } fileContent, _, _, err := client.Repositories.GetContents( ctx, diff --git a/services/workflow_processor.go b/services/workflow_processor.go index ee1b9bc..a766907 100644 --- a/services/workflow_processor.go +++ b/services/workflow_processor.go @@ -340,7 +340,7 @@ func (wp *workflowProcessor) addToUploadQueue( // Create upload key key := UploadKey{ RepoName: workflow.Destination.Repo, - BranchPath: "refs/heads/" + workflow.Destination.Branch, + BranchPath: workflow.Destination.Branch, } // Get existing entries from FileStateService From 4eb6e060aa9cbd07ad0bc6120c239a7cddbc5153 Mon Sep 17 00:00:00 2001 From: Cory Bullinger Date: Mon, 2 Feb 2026 14:16:52 -0500 Subject: [PATCH 3/3] Improve error messages for GitHub App authentication failures - Add explicit 'GITHUB APP AUTHENTICATION FAILED' message for 401 errors - Point users to check CODE_COPIER_PEM secret in GCP Secret Manager - Add detection in getInstallationIDForOrg, getInstallationAccessToken - Add detection in config_loader and main_config_loader when fetching configs This makes it immediately obvious when the PEM key is invalid/expired instead of showing misleading 'failed to load config' errors. --- env-cloudrun.yaml | 1 + services/config_loader.go | 6 ++++++ services/github_auth.go | 6 ++++++ services/main_config_loader.go | 5 +++++ 4 files changed, 18 insertions(+) diff --git a/env-cloudrun.yaml b/env-cloudrun.yaml index a255f63..51b13ec 100644 --- a/env-cloudrun.yaml +++ b/env-cloudrun.yaml @@ -13,6 +13,7 @@ CONFIG_REPO_BRANCH: "main" # Secret Manager References GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest" +PEM_NAME: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest" WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" MONGO_URI_SECRET_NAME: "projects/1054147886816/secrets/mongo-uri/versions/latest" diff --git a/services/config_loader.go b/services/config_loader.go index f3567dd..f35d633 100644 --- a/services/config_loader.go +++ b/services/config_loader.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "strings" "github.com/google/go-github/v48/github" "gopkg.in/yaml.v3" @@ -91,6 +92,11 @@ func retrieveConfigFileContent(ctx context.Context, filePath string, config *con }, ) if err != nil { + // Check if this is an authentication error + errStr := err.Error() + if strings.Contains(errStr, "401") || strings.Contains(errStr, "Bad credentials") { + return "", fmt.Errorf("GITHUB APP AUTHENTICATION FAILED: Unable to fetch config file due to authentication error. The GitHub App private key (PEM) may be invalid or expired. Please check the CODE_COPIER_PEM secret in GCP Secret Manager. Original error: %w", err) + } return "", fmt.Errorf("failed to get config file: %w", err) } if fileContent == nil { diff --git a/services/github_auth.go b/services/github_auth.go index b5a5aac..b44a2d3 100644 --- a/services/github_auth.go +++ b/services/github_auth.go @@ -246,6 +246,9 @@ func getInstallationAccessToken(installationId, jwtToken string, hc *http.Client if resp.StatusCode != http.StatusCreated { b, _ := io.ReadAll(resp.Body) + if resp.StatusCode == http.StatusUnauthorized { + return "", fmt.Errorf("GITHUB APP AUTHENTICATION FAILED (401): Failed to get installation access token. The GitHub App private key (PEM) may be invalid or expired. Please check the CODE_COPIER_PEM secret in GCP Secret Manager. Response: %s", string(b)) + } return "", fmt.Errorf("status %d: %s", resp.StatusCode, string(b)) } var out struct { @@ -383,6 +386,9 @@ func getInstallationIDForOrg(org string) (string, error) { if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) + if resp.StatusCode == http.StatusUnauthorized { + return "", fmt.Errorf("GITHUB APP AUTHENTICATION FAILED (401): The GitHub App private key (PEM) may be invalid or expired. Please check the CODE_COPIER_PEM secret in GCP Secret Manager. Response: %s", string(body)) + } return "", fmt.Errorf("GET %s: %d %s %s", url, resp.StatusCode, resp.Status, body) } diff --git a/services/main_config_loader.go b/services/main_config_loader.go index 274663b..c3d4afc 100644 --- a/services/main_config_loader.go +++ b/services/main_config_loader.go @@ -62,6 +62,11 @@ func (mcl *DefaultMainConfigLoader) LoadMainConfig(ctx context.Context, config * // Fall back to fetching from repository content, err = retrieveConfigFileContent(ctx, configFile, config) if err != nil { + // Check if this is an authentication error and make it more prominent + errStr := err.Error() + if strings.Contains(errStr, "GITHUB APP AUTHENTICATION FAILED") || strings.Contains(errStr, "401") || strings.Contains(errStr, "Bad credentials") { + return nil, fmt.Errorf("GITHUB APP AUTHENTICATION FAILED: Unable to retrieve main config file. The GitHub App private key (PEM) may be invalid or expired. Please check the CODE_COPIER_PEM secret in GCP Secret Manager and redeploy the service. Original error: %w", err) + } return nil, fmt.Errorf("failed to retrieve main config file: %w", err) } }