From fe8d9b3eaac8af8dc9c0f7c8afd0d832405460de Mon Sep 17 00:00:00 2001 From: Yoav Date: Mon, 23 Jun 2025 15:34:42 +0200 Subject: [PATCH] start of git clone process --- cmd/project.go | 274 +++++++++++++++ error_codes.yaml | 21 ++ internal/errsystem/errorcodes.go | 28 ++ internal/github/github.go | 566 +++++++++++++++++++++++++++++++ 4 files changed, 889 insertions(+) create mode 100644 internal/github/github.go diff --git a/cmd/project.go b/cmd/project.go index b12becc6..9bc979c8 100644 --- a/cmd/project.go +++ b/cmd/project.go @@ -1,19 +1,26 @@ package cmd import ( + "archive/tar" + "compress/gzip" "context" "encoding/json" "fmt" + "io" + "net/http" "os" "os/exec" "os/signal" "path/filepath" "sort" + "strings" "syscall" + "time" "github.com/agentuity/cli/internal/deployer" "github.com/agentuity/cli/internal/envutil" "github.com/agentuity/cli/internal/errsystem" + "github.com/agentuity/cli/internal/github" "github.com/agentuity/cli/internal/mcp" "github.com/agentuity/cli/internal/organization" "github.com/agentuity/cli/internal/project" @@ -297,6 +304,7 @@ Arguments: Examples: agentuity project create "My Project" "Project description" "My Agent" "Agent description" --auth bearer + agentuity project create "My Project" -e https://github.com/user/repo agentuity create --runtime nodejs --template "OpenAI SDK for Typescript"`, Aliases: []string{"new"}, Args: cobra.MaximumNArgs(4), @@ -309,6 +317,14 @@ Examples: initScreenWithLogo() + // Check if example flag is provided + exampleURL, _ := cmd.Flags().GetString("example") + + if exampleURL != "" { + handleGitHubExample(ctx, logger, cmd, apikey, apiUrl, exampleURL, args) + return + } + cwd, err := os.Getwd() if err != nil { errsystem.New(errsystem.ErrListFilesAndDirectories, err, errsystem.WithContextMessage("Failed to get current working directory")).ShowErrorAndExit() @@ -906,6 +922,7 @@ func init() { projectNewCmd.Flags().String("templates-dir", "", "The directory to load the templates. Defaults to loading them from the github.com/agentuity/templates repository") projectNewCmd.Flags().String("auth", "project", "The authentication type for the agent (project, webhook, or none)") projectNewCmd.Flags().String("action", "github-app", "The action to take for the project (github-action, github-app, none)") + projectNewCmd.Flags().StringP("example", "e", "", "Create project from a GitHub repository example (provide the GitHub URL)") projectImportCmd.Flags().String("name", "", "The name of the project to import") projectImportCmd.Flags().String("description", "", "The description of the project to import") @@ -919,3 +936,260 @@ func init() { projectDeleteCmd.Flags().String("org-id", "", "Only delete the projects in the specified organization") } + +// handleGitHubExample handles downloading a GitHub repository and importing it as a project +func handleGitHubExample(ctx context.Context, logger logger.Logger, cmd *cobra.Command, apikey, apiUrl, githubURL string, args []string) { + // Validate GitHub URL + parsedURL, err := github.ValidateGitHubURL(githubURL) + if err != nil { + logger.Fatal("%s", err) + } + + // Get repository information + repoInfo, err := github.GetRepoInfo(ctx, logger, parsedURL, "") + if err != nil { + logger.Fatal("%s", err) + } + + // Validate that the specified path contains an agentuity.yaml file + if err := github.ValidateAgentuityProjectPath(ctx, logger, repoInfo); err != nil { + logger.Fatal("%s", err) + } + + // Get project name from args or prompt + var name string + if len(args) > 0 { + name = args[0] // project name from args + } + if name == "" { + name = tui.Input(logger, "What should we name this project?", + fmt.Sprintf("This will create a project based on %s/%s", repoInfo.Username, repoInfo.Name)) + } + + // Determine project directory + cwd, err := os.Getwd() + if err != nil { + errsystem.New(errsystem.ErrListFilesAndDirectories, err).ShowErrorAndExit() + } + + projectDir := filepath.Join(cwd, util.SafeProjectFilename(name, false)) + dir, _ := cmd.Flags().GetString("dir") + if dir != "" { + absDir, err := filepath.Abs(dir) + if err != nil { + errsystem.New(errsystem.ErrListFilesAndDirectories, err).ShowErrorAndExit() + } + projectDir = absDir + } + + force, _ := cmd.Flags().GetBool("force") + + // Check if directory exists + if util.Exists(projectDir) { + if !force { + if tui.HasTTY { + fmt.Println(tui.Secondary("The directory ") + tui.Bold(projectDir) + tui.Secondary(" already exists.")) + fmt.Println() + if !tui.Ask(logger, "Delete and continue?", true) { + return + } + } else { + logger.Fatal("The directory %s already exists. Use --force to overwrite.", projectDir) + } + } + os.RemoveAll(projectDir) + } + + // Download and extract repository (without validation) + tui.ShowSpinner(fmt.Sprintf("Downloading repository %s/%s...", repoInfo.Username, repoInfo.Name), func() { + err = downloadAndExtractRepoWithoutValidation(ctx, logger, projectDir, repoInfo) + if err != nil { + logger.Fatal("%s", err) + } + }) + + tui.ShowSuccess("Successfully downloaded repository to %s", projectDir) + + // TODO: Import functionality disabled for now - just clone the repository + tui.ShowBanner("Agentuity project cloned successfully!", + tui.Paragraph("Next steps:", + tui.Secondary("1. Switch into the project directory at ")+tui.Directory(projectDir), + tui.Secondary("2. Review the cloned code and make any necessary adjustments"), + tui.Secondary("3. Run ")+tui.Command("project import")+tui.Secondary(" to import the project into your organization"), + ), + false, + ) +} + +// downloadAndExtractRepoWithoutValidation downloads and extracts a repository without agentuity.yaml validation +func downloadAndExtractRepoWithoutValidation(ctx context.Context, logger logger.Logger, projectDir string, repoInfo *github.RepoInfo) error { + if repoInfo == nil { + return fmt.Errorf("repoInfo cannot be nil") + } + + // Create the project directory if it doesn't exist + if err := os.MkdirAll(projectDir, 0755); err != nil { + return fmt.Errorf("failed to create project directory: %w", err) + } + + // Download the repository as a tar.gz file + tarURL := fmt.Sprintf("https://codeload.github.com/%s/%s/tar.gz/%s", + repoInfo.Username, repoInfo.Name, repoInfo.Branch) + + logger.Debug("Downloading repository from: %s", tarURL) + + req, err := http.NewRequestWithContext(ctx, "GET", tarURL, nil) + if err != nil { + return errsystem.New(errsystem.ErrDownloadGithubRepository, + fmt.Errorf("failed to create download request: %w", err)) + } + + req.Header.Set("User-Agent", util.UserAgent()) + + client := &http.Client{ + Timeout: 5 * time.Minute, // Longer timeout for large repositories + } + + resp, err := client.Do(req) + if err != nil { + return errsystem.New(errsystem.ErrDownloadGithubRepository, + fmt.Errorf("failed to download repository: %w", err), + errsystem.WithUserMessage("Failed to download the repository from GitHub. Please check your internet connection and try again.")) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return errsystem.New(errsystem.ErrDownloadGithubRepository, + fmt.Errorf("failed to download repository: HTTP %d", resp.StatusCode), + errsystem.WithUserMessage("Failed to download the repository. The repository may not exist or the branch '%s' may not be available.", repoInfo.Branch)) + } + + // Extract the tar.gz content using the same extraction logic from github package + return extractTarGzSimple(logger, resp.Body, projectDir, repoInfo.FilePath) +} + +// extractTarGzSimple is a simplified version of the tar extraction without complex path manipulation +func extractTarGzSimple(logger logger.Logger, reader io.Reader, destDir, targetPath string) error { + gzr, err := gzip.NewReader(reader) + if err != nil { + return fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + var rootPath string + fileCount := 0 + + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("failed to read tar entry: %w", err) + } + + logger.Debug("Processing tar entry: %s (type: %d)", header.Name, header.Typeflag) + + // Skip disallowed hidden files + fileName := filepath.Base(header.Name) + if strings.HasPrefix(fileName, ".") && !github.IsHiddenFileAllowed(fileName) { + logger.Debug("Skipping hidden file: %s", header.Name) + continue + } + + // Determine the root path dynamically from the first directory entry + if rootPath == "" && header.Typeflag == tar.TypeDir { + pathParts := strings.Split(header.Name, "/") + if len(pathParts) > 0 && pathParts[0] != "" { + rootPath = pathParts[0] + logger.Debug("Detected root path: %s", rootPath) + } + } + + // Normalize paths for comparison + normalizedHeaderName := strings.ReplaceAll(header.Name, "\\", "/") + + var relativePath string + + if targetPath == "" { + // Extract everything under the root path + if !strings.HasPrefix(normalizedHeaderName, rootPath+"/") && normalizedHeaderName != rootPath { + logger.Debug("Skipping file not in root: %s", normalizedHeaderName) + continue + } + + // Strip the root directory + relativePath = strings.TrimPrefix(normalizedHeaderName, rootPath+"/") + if relativePath == "" && header.Typeflag == tar.TypeDir { + continue // Skip the root directory itself + } + } else { + // Extract only files matching the target path + expectedPrefix := filepath.Join(rootPath, targetPath) + normalizedPrefix := strings.ReplaceAll(expectedPrefix, "\\", "/") + + if !strings.HasPrefix(normalizedHeaderName, normalizedPrefix+"/") && normalizedHeaderName != normalizedPrefix { + logger.Debug("Skipping file due to path filter: %s (expected prefix: %s)", normalizedHeaderName, normalizedPrefix) + continue + } + + // Strip the root directory and target path + relativePath = strings.TrimPrefix(normalizedHeaderName, normalizedPrefix) + if relativePath != "" && relativePath[0] == '/' { + relativePath = relativePath[1:] + } + } + + destPath := filepath.Join(destDir, relativePath) + logger.Debug("Extracting %s to %s", header.Name, destPath) + + // Security check: ensure the destination path is within the target directory + if !strings.HasPrefix(destPath, filepath.Clean(destDir)+string(os.PathSeparator)) { + logger.Warn("Skipping file outside target directory: %s", destPath) + continue + } + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(destPath, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", destPath, err) + } + logger.Debug("Created directory: %s", destPath) + fileCount++ + + case tar.TypeReg: + // Ensure parent directory exists + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + return fmt.Errorf("failed to create parent directory for %s: %w", destPath, err) + } + + // Create and write the file + file, err := os.Create(destPath) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", destPath, err) + } + + _, err = io.Copy(file, tr) + closeErr := file.Close() + + if err != nil { + return fmt.Errorf("failed to write file %s: %w", destPath, err) + } + if closeErr != nil { + return fmt.Errorf("failed to close file %s: %w", destPath, closeErr) + } + + // Set file permissions + if err := os.Chmod(destPath, os.FileMode(header.Mode)); err != nil { + logger.Debug("Failed to set permissions for %s: %v", destPath, err) + } + + logger.Debug("Extracted file: %s", destPath) + fileCount++ + } + } + + logger.Debug("Total files/directories extracted: %d", fileCount) + return nil +} diff --git a/error_codes.yaml b/error_codes.yaml index c6ef769a..4fd30fab 100644 --- a/error_codes.yaml +++ b/error_codes.yaml @@ -88,3 +88,24 @@ errors: - code: CLI-0028 message: Failed to delete API key + + - code: CLI-0029 + message: Failed to process GitHub repository + + - code: CLI-0030 + message: Invalid GitHub URL format + + - code: CLI-0031 + message: GitHub repository not found or not accessible + + - code: CLI-0032 + message: Failed to download GitHub repository + + - code: CLI-0033 + message: Failed to extract GitHub repository + + - code: CLI-0034 + message: GitHub API request failed + + - code: CLI-0035 + message: Not a valid Agentuity project diff --git a/internal/errsystem/errorcodes.go b/internal/errsystem/errorcodes.go index c2e0812c..9ee3ecd2 100644 --- a/internal/errsystem/errorcodes.go +++ b/internal/errsystem/errorcodes.go @@ -114,4 +114,32 @@ var ( Code: "CLI-0028", Message: "Failed to delete API key", } + ErrProcessGithubRepository = errorType{ + Code: "CLI-0029", + Message: "Failed to process GitHub repository", + } + ErrInvalidGithubUrlFormat = errorType{ + Code: "CLI-0030", + Message: "Invalid GitHub URL format", + } + ErrGithubRepositoryNotFoundOrNotAccessible = errorType{ + Code: "CLI-0031", + Message: "GitHub repository not found or not accessible", + } + ErrDownloadGithubRepository = errorType{ + Code: "CLI-0032", + Message: "Failed to download GitHub repository", + } + ErrExtractGithubRepository = errorType{ + Code: "CLI-0033", + Message: "Failed to extract GitHub repository", + } + ErrGithubApiRequest = errorType{ + Code: "CLI-0034", + Message: "GitHub API request failed", + } + ErrNotValidAgentuityProject = errorType{ + Code: "CLI-0035", + Message: "Not a valid Agentuity project", + } ) diff --git a/internal/github/github.go b/internal/github/github.go new file mode 100644 index 00000000..0c9859c2 --- /dev/null +++ b/internal/github/github.go @@ -0,0 +1,566 @@ +package github + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/agentuity/cli/internal/errsystem" + "github.com/agentuity/cli/internal/util" + "github.com/agentuity/go-common/logger" +) + +// allowedHiddenFiles lists hidden files/directories that should be preserved during extraction +var allowedHiddenFiles = []string{ + ".gitignore", ".env", ".env.example", ".env.local", ".env.production", + ".github", ".vscode", ".eslintrc", ".prettierrc", ".editorconfig", ".cursor", ".windsurf", + ".npmrc", ".nvmrc", ".node-version", ".dockerignore", ".gitattributes", +} + +// RepoInfo contains information about a GitHub repository +type RepoInfo struct { + Username string + Name string + Branch string + FilePath string +} + +// GitHubAPIRepo represents the GitHub API response for repository information +type GitHubAPIRepo struct { + DefaultBranch string `json:"default_branch"` + Private bool `json:"private"` + FullName string `json:"full_name"` +} + +// IsURLValid checks if a URL is accessible and returns a 200 status code +func IsURLValid(ctx context.Context, targetURL string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "HEAD", targetURL, nil) + if err != nil { + return false, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("User-Agent", util.UserAgent()) + + client := &http.Client{ + Timeout: 10 * time.Second, + } + + resp, err := client.Do(req) + if err != nil { + return false, fmt.Errorf("failed to execute request: %w", err) + } + defer resp.Body.Close() + + return resp.StatusCode == http.StatusOK, nil +} + +// GetRepoInfo extracts repository information from a GitHub URL +// Supports various GitHub URL formats: +// - https://github.com/username/repo +// - https://github.com/username/repo/ +// - https://github.com/username/repo/tree/branch +// - https://github.com/username/repo/tree/branch/path/to/example +func GetRepoInfo(ctx context.Context, logger logger.Logger, repoURL *url.URL, examplePath string) (*RepoInfo, error) { + if repoURL.Host != "github.com" { + return nil, errsystem.New(errsystem.ErrInvalidGithubUrlFormat, + fmt.Errorf("only GitHub URLs are supported, got: %s", repoURL.Host)) + } + + // Split the path and remove empty elements + pathParts := strings.Split(strings.Trim(repoURL.Path, "/"), "/") + if len(pathParts) < 2 { + return nil, errsystem.New(errsystem.ErrInvalidGithubUrlFormat, + fmt.Errorf("invalid GitHub URL format, expected at least username/repo")) + } + + username := pathParts[0] + name := pathParts[1] + + // Validate username and repository name + if username == "" || name == "" { + return nil, errsystem.New(errsystem.ErrInvalidGithubUrlFormat, + fmt.Errorf("username and repository name cannot be empty")) + } + + // Handle different URL formats + switch { + case len(pathParts) == 2: + // Format: github.com/username/repo + return handleBasicRepoURL(ctx, logger, username, name, examplePath) + + case len(pathParts) == 3 && pathParts[2] == "": + // Format: github.com/username/repo/ (trailing slash) + return handleBasicRepoURL(ctx, logger, username, name, examplePath) + + case len(pathParts) >= 4 && pathParts[2] == "tree": + // Format: github.com/username/repo/tree/branch[/path/to/example] + return handleTreeURL(username, name, pathParts[3:], examplePath) + + default: + return nil, errsystem.New(errsystem.ErrInvalidGithubUrlFormat, + fmt.Errorf("unsupported GitHub URL format")) + } +} + +// handleBasicRepoURL handles URLs without branch specification +func handleBasicRepoURL(ctx context.Context, logger logger.Logger, username, name, examplePath string) (*RepoInfo, error) { + // Get default branch from GitHub API + defaultBranch, err := getDefaultBranch(ctx, logger, username, name) + if err != nil { + return nil, err + } + + return &RepoInfo{ + Username: username, + Name: name, + Branch: defaultBranch, + FilePath: cleanFilePath(examplePath), + }, nil +} + +// handleTreeURL handles URLs with tree/branch specification +func handleTreeURL(username, name string, treeParts []string, examplePath string) (*RepoInfo, error) { + if len(treeParts) == 0 { + return nil, errsystem.New(errsystem.ErrInvalidGithubUrlFormat, + fmt.Errorf("branch name is required after 'tree'")) + } + + branch := treeParts[0] + var filePath string + + if examplePath != "" { + // Use provided example path + filePath = cleanFilePath(examplePath) + } else if len(treeParts) > 1 { + // Use path from URL + filePath = strings.Join(treeParts[1:], "/") + } + + return &RepoInfo{ + Username: username, + Name: name, + Branch: branch, + FilePath: cleanFilePath(filePath), + }, nil +} + +// getDefaultBranch fetches the default branch from GitHub API +func getDefaultBranch(ctx context.Context, logger logger.Logger, username, name string) (string, error) { + apiURL := fmt.Sprintf("https://api.github.com/repos/%s/%s", username, name) + + req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil) + if err != nil { + return "", errsystem.New(errsystem.ErrGithubApiRequest, + fmt.Errorf("failed to create API request: %w", err)) + } + + req.Header.Set("User-Agent", util.UserAgent()) + req.Header.Set("Accept", "application/vnd.github.v3+json") + + client := &http.Client{ + Timeout: 15 * time.Second, + } + + resp, err := client.Do(req) + if err != nil { + return "", errsystem.New(errsystem.ErrGithubApiRequest, + fmt.Errorf("failed to fetch repository info: %w", err)) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + switch resp.StatusCode { + case http.StatusNotFound: + return "", errsystem.New(errsystem.ErrGithubRepositoryNotFoundOrNotAccessible, + fmt.Errorf("repository %s/%s not found or not accessible", username, name), + errsystem.WithUserMessage("The repository '%s/%s' was not found. Please check the repository name and ensure it's publicly accessible.", username, name)) + case http.StatusForbidden: + return "", errsystem.New(errsystem.ErrGithubRepositoryNotFoundOrNotAccessible, + fmt.Errorf("repository %s/%s is private or access denied", username, name), + errsystem.WithUserMessage("Access to repository '%s/%s' is forbidden. The repository may be private or you may not have permission to access it.", username, name)) + default: + return "", errsystem.New(errsystem.ErrGithubApiRequest, + fmt.Errorf("GitHub API returned status %d", resp.StatusCode)) + } + } + + var repoInfo GitHubAPIRepo + if err := json.NewDecoder(resp.Body).Decode(&repoInfo); err != nil { + return "", errsystem.New(errsystem.ErrGithubApiRequest, + fmt.Errorf("failed to decode GitHub API response: %w", err)) + } + + if repoInfo.DefaultBranch == "" { + logger.Warn("No default branch found, using 'main' as fallback") + return "main", nil + } + + return repoInfo.DefaultBranch, nil +} + +// HasRepo checks if a repository exists and is accessible +func HasRepo(ctx context.Context, repoInfo *RepoInfo) (bool, error) { + if repoInfo == nil { + return false, fmt.Errorf("repoInfo cannot be nil") + } + + // Check if repository exists by trying to access its contents + contentsURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/contents", repoInfo.Username, repoInfo.Name) + + // Add file path if specified + if repoInfo.FilePath != "" { + contentsURL = fmt.Sprintf("%s/%s", contentsURL, repoInfo.FilePath) + } + + // Add branch reference + contentsURL = fmt.Sprintf("%s?ref=%s", contentsURL, repoInfo.Branch) + + return IsURLValid(ctx, contentsURL) +} + +// ValidateAgentuityProject checks if the project directory contains a valid agentuity.yaml file +func ValidateAgentuityProject(projectDir string) error { + + agentuityYamlPath := filepath.Join(projectDir, "agentuity.yaml") + + // Check if agentuity.yaml exists + if _, err := os.Stat(agentuityYamlPath); err != nil { + if os.IsNotExist(err) { + return errsystem.New(errsystem.ErrNotValidAgentuityProject, + fmt.Errorf("agentuity.yaml not found in project root"), + errsystem.WithUserMessage("The downloaded repository is not a valid Agentuity project. An 'agentuity.yaml' file is required in the project root.")) + } + return fmt.Errorf("failed to check agentuity.yaml: %w", err) + } + + return nil +} + +// DownloadAndExtractRepo downloads a repository and extracts it to the specified directory +func DownloadAndExtractRepo(ctx context.Context, logger logger.Logger, projectDir string, repoInfo *RepoInfo) error { + if repoInfo == nil { + return fmt.Errorf("repoInfo cannot be nil") + } + + // Create the project directory if it doesn't exist + if err := os.MkdirAll(projectDir, 0755); err != nil { + return fmt.Errorf("failed to create project directory: %w", err) + } + + // Download the repository as a tar.gz file + tarURL := fmt.Sprintf("https://codeload.github.com/%s/%s/tar.gz/%s", + repoInfo.Username, repoInfo.Name, repoInfo.Branch) + + logger.Debug("Downloading repository from: %s", tarURL) + + req, err := http.NewRequestWithContext(ctx, "GET", tarURL, nil) + if err != nil { + return errsystem.New(errsystem.ErrDownloadGithubRepository, + fmt.Errorf("failed to create download request: %w", err)) + } + + req.Header.Set("User-Agent", util.UserAgent()) + + client := &http.Client{ + Timeout: 5 * time.Minute, // Longer timeout for large repositories + } + + resp, err := client.Do(req) + if err != nil { + return errsystem.New(errsystem.ErrDownloadGithubRepository, + fmt.Errorf("failed to download repository: %w", err), + errsystem.WithUserMessage("Failed to download the repository from GitHub. Please check your internet connection and try again.")) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return errsystem.New(errsystem.ErrDownloadGithubRepository, + fmt.Errorf("failed to download repository: HTTP %d", resp.StatusCode), + errsystem.WithUserMessage("Failed to download the repository. The repository may not exist or the branch '%s' may not be available.", repoInfo.Branch)) + } + + // Extract the tar.gz content + if err := extractTarGz(logger, resp.Body, projectDir, repoInfo.FilePath); err != nil { + return errsystem.New(errsystem.ErrExtractGithubRepository, + fmt.Errorf("failed to extract repository: %w", err)) + } + + // Validate that this is a valid Agentuity project + if err := ValidateAgentuityProject(projectDir); err != nil { + return err + } + + logger.Debug("Successfully extracted repository to: %s", projectDir) + return nil +} + +// IsHiddenFileAllowed checks if a hidden file should be preserved +func IsHiddenFileAllowed(fileName string) bool { + for _, allowedFile := range allowedHiddenFiles { + if fileName == allowedFile || strings.HasPrefix(fileName, allowedFile+".") { + return true + } + } + return false +} + +// extractTarGz extracts a tar.gz stream to the specified directory +func extractTarGz(logger logger.Logger, reader io.Reader, destDir, targetPath string) error { + gzr, err := gzip.NewReader(reader) + if err != nil { + return fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + + var rootPath string + stripCount := 1 + + // Calculate strip count based on target path + if targetPath != "" { + stripCount += len(strings.Split(targetPath, "/")) + } + + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("failed to read tar entry: %w", err) + } + + logger.Debug("Processing tar entry: %s (type: %c)", header.Name, header.Typeflag) + + // Skip disallowed hidden files + fileName := filepath.Base(header.Name) + if strings.HasPrefix(fileName, ".") && !IsHiddenFileAllowed(fileName) { + continue + } + + // Determine the root path dynamically from the first entry + if rootPath == "" { + pathParts := strings.Split(header.Name, "/") + if len(pathParts) > 0 { + rootPath = pathParts[0] + } + } + + // Check if this file should be extracted based on target path + expectedPrefix := rootPath + if targetPath != "" { + expectedPrefix = filepath.Join(rootPath, targetPath) + } + + // Normalize paths for comparison + normalizedHeaderName := strings.ReplaceAll(header.Name, "\\", "/") + normalizedPrefix := strings.ReplaceAll(expectedPrefix, "\\", "/") + + // More flexible prefix matching + shouldExtract := false + if targetPath == "" { + // Extract everything under the root path + shouldExtract = strings.HasPrefix(normalizedHeaderName, normalizedPrefix+"/") || + normalizedHeaderName == normalizedPrefix + } else { + // Extract files matching the target path + shouldExtract = strings.HasPrefix(normalizedHeaderName, normalizedPrefix+"/") || + normalizedHeaderName == normalizedPrefix + } + + if !shouldExtract { + logger.Debug("Skipping file due to path filter: %s (expected prefix: %s)", normalizedHeaderName, normalizedPrefix) + continue + } + + // Calculate the relative path by stripping the prefix + relativePath := strings.TrimPrefix(normalizedHeaderName, normalizedPrefix) + if relativePath != "" && relativePath[0] == '/' { + relativePath = relativePath[1:] + } + + // Skip if this would create an empty path for regular files + if relativePath == "" && header.Typeflag == tar.TypeReg { + logger.Debug("Skipping file with empty relative path: %s", header.Name) + continue + } + + destPath := filepath.Join(destDir, relativePath) + logger.Debug("Extracting to: %s (from: %s, relative: %s)", destPath, header.Name, relativePath) + + // Security check: ensure the destination path is within the target directory + if !strings.HasPrefix(destPath, filepath.Clean(destDir)+string(os.PathSeparator)) { + logger.Warn("Skipping file outside target directory: %s", destPath) + continue + } + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(destPath, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", destPath, err) + } + logger.Debug("Created directory: %s", destPath) + + case tar.TypeReg: + // Ensure parent directory exists + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + return fmt.Errorf("failed to create parent directory for %s: %w", destPath, err) + } + + // Create and write the file + file, err := os.Create(destPath) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", destPath, err) + } + + _, err = io.Copy(file, tr) + closeErr := file.Close() + + if err != nil { + return fmt.Errorf("failed to write file %s: %w", destPath, err) + } + if closeErr != nil { + return fmt.Errorf("failed to close file %s: %w", destPath, closeErr) + } + + // Set file permissions + if err := os.Chmod(destPath, os.FileMode(header.Mode)); err != nil { + logger.Debug("Failed to set permissions for %s: %v", destPath, err) + } + + logger.Debug("Extracted file: %s", destPath) + } + } + + return nil +} + +// cleanFilePath cleans and normalizes a file path +func cleanFilePath(path string) string { + if path == "" { + return "" + } + + // Remove leading/trailing slashes and clean the path + cleaned := strings.Trim(path, "/") + return filepath.Clean(cleaned) +} + +// ValidateGitHubURL validates that a URL is a valid GitHub repository URL +func ValidateGitHubURL(rawURL string) (*url.URL, error) { + if rawURL == "" { + return nil, errsystem.New(errsystem.ErrInvalidGithubUrlFormat, + fmt.Errorf("URL cannot be empty"), + errsystem.WithUserMessage("Please provide a valid GitHub repository URL.")) + } + + // Add https:// if no scheme is provided + if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") { + rawURL = "https://" + rawURL + } + + parsedURL, err := url.Parse(rawURL) + if err != nil { + return nil, errsystem.New(errsystem.ErrInvalidGithubUrlFormat, + fmt.Errorf("invalid URL format: %w", err), + errsystem.WithUserMessage("The provided URL format is invalid. Please check the URL and try again.")) + } + + if parsedURL.Host != "github.com" { + return nil, errsystem.New(errsystem.ErrInvalidGithubUrlFormat, + fmt.Errorf("only GitHub URLs are supported, got: %s", parsedURL.Host), + errsystem.WithUserMessage("Only GitHub repositories are supported. Please provide a github.com URL.")) + } + + // Basic validation of the path + pathParts := strings.Split(strings.Trim(parsedURL.Path, "/"), "/") + if len(pathParts) < 2 { + return nil, errsystem.New(errsystem.ErrInvalidGithubUrlFormat, + fmt.Errorf("invalid GitHub URL: missing username or repository name"), + errsystem.WithUserMessage("The GitHub URL must include both a username and repository name (e.g., github.com/username/repository).")) + } + + // Validate username and repo name format (basic GitHub rules) + usernameRegex := regexp.MustCompile(`^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9]|[a-zA-Z0-9]*)$`) + repoRegex := regexp.MustCompile(`^[a-zA-Z0-9._-]+$`) + + username := pathParts[0] + repoName := pathParts[1] + + if !usernameRegex.MatchString(username) { + return nil, errsystem.New(errsystem.ErrInvalidGithubUrlFormat, + fmt.Errorf("invalid GitHub username format: %s", username), + errsystem.WithUserMessage("The GitHub username '%s' contains invalid characters.", username)) + } + + if !repoRegex.MatchString(repoName) { + return nil, errsystem.New(errsystem.ErrInvalidGithubUrlFormat, + fmt.Errorf("invalid GitHub repository name format: %s", repoName), + errsystem.WithUserMessage("The GitHub repository name '%s' contains invalid characters.", repoName)) + } + + return parsedURL, nil +} + +// ValidateAgentuityProjectPath checks if a GitHub repository path contains an agentuity.yaml file +func ValidateAgentuityProjectPath(ctx context.Context, logger logger.Logger, repoInfo *RepoInfo) error { + if repoInfo == nil { + return fmt.Errorf("repoInfo cannot be nil") + } + + // Construct the path to check for agentuity.yaml + var agentuityYamlPath string + if repoInfo.FilePath != "" { + agentuityYamlPath = fmt.Sprintf("%s/agentuity.yaml", strings.Trim(repoInfo.FilePath, "/")) + } else { + agentuityYamlPath = "agentuity.yaml" + } + + // Check if agentuity.yaml exists in the specified path via GitHub API + contentsURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/contents/%s?ref=%s", + repoInfo.Username, repoInfo.Name, agentuityYamlPath, repoInfo.Branch) + + logger.Debug("Checking for agentuity.yaml at: %s", contentsURL) + + req, err := http.NewRequestWithContext(ctx, "HEAD", contentsURL, nil) + if err != nil { + return fmt.Errorf("failed to create validation request: %w", err) + } + + req.Header.Set("User-Agent", util.UserAgent()) + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to check agentuity.yaml: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + pathDesc := "repository root" + if repoInfo.FilePath != "" { + pathDesc = fmt.Sprintf("path '%s'", repoInfo.FilePath) + } + return errsystem.New(errsystem.ErrNotValidAgentuityProject, + fmt.Errorf("agentuity.yaml not found in %s", pathDesc), + errsystem.WithUserMessage("The specified %s does not contain an 'agentuity.yaml' file. Please ensure you're pointing to a valid Agentuity project directory.", pathDesc)) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to validate agentuity.yaml: HTTP %d", resp.StatusCode) + } + + logger.Debug("Found agentuity.yaml in the specified path") + return nil +}