From 8bb3f409afe0fcf4262ecb6350a66b07bdd78559 Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Wed, 25 Jun 2025 19:59:33 +0200 Subject: [PATCH 1/8] FEAT[DXTA-297]: Create workflow for github installation data provisioning --- cmd/internal-api/main.go | 6 +- cmd/onboarding-worker/main.go | 17 +++ go.mod | 5 + go.sum | 11 ++ internal/internal-api/data/tenantDB.go | 7 +- .../handler/add_member_to_team.go | 9 +- .../internal-api/handler/create_member.go | 9 +- internal/internal-api/handler/create_team.go | 26 ++-- .../provision_github_installation_data.go | 58 ++++++++ internal/internal-api/handler/temporal.go | 18 +++ internal/internal-api/handler/users_count.go | 18 +-- internal/internal-api/internal-api.go | 11 +- .../provision-github-installation-data.go | 18 +++ internal/onboarding/data/github.go | 128 ++++++++++++++++++ .../onboarding/data/github_installations.go | 75 ++++++++++ .../provision-github-installation-data.go | 71 ++++++++++ internal/util/auth.go | 32 +---- 17 files changed, 457 insertions(+), 62 deletions(-) create mode 100644 internal/internal-api/handler/provision_github_installation_data.go create mode 100644 internal/internal-api/handler/temporal.go create mode 100644 internal/onboarding/activity/provision-github-installation-data.go create mode 100644 internal/onboarding/data/github.go create mode 100644 internal/onboarding/data/github_installations.go create mode 100644 internal/onboarding/workflows/provision-github-installation-data.go diff --git a/cmd/internal-api/main.go b/cmd/internal-api/main.go index 209d8364..94b1acf1 100644 --- a/cmd/internal-api/main.go +++ b/cmd/internal-api/main.go @@ -129,7 +129,7 @@ func main() { defer temporalClient.Close() - usersHandler := handler.NewUsers(temporalClient, *cfg) + temporalHandler := handler.SetupOnboardingTemporal(temporalClient, *cfg) r.Route("/tenant", func(r chi.Router) { if os.Getenv("ENABLE_JWT_AUTH") == "true" { @@ -147,8 +147,8 @@ func main() { r.Post("/teams", handler.CreateTeam) r.Post("/teams/{team_id}/members/{member_id}", handler.AddMemberToTeam) r.Post("/members", handler.CreateMember) + r.Get("/provision-github-installations/{installation_id}", temporalHandler.ProvisionGithubInstallationData) }) - r.Get("/health", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`OK`)) }) @@ -157,7 +157,7 @@ func main() { w.Write([]byte(`OK`)) }) - r.Get("/users-count", usersHandler.UsersCount) + r.Get("/users-count", temporalHandler.UsersCount) go func() { log.Printf("Listening on %s\n", srv.Addr) diff --git a/cmd/onboarding-worker/main.go b/cmd/onboarding-worker/main.go index ba6d5b09..3d800509 100644 --- a/cmd/onboarding-worker/main.go +++ b/cmd/onboarding-worker/main.go @@ -6,6 +6,7 @@ import ( "github.com/dxta-dev/app/internal/onboarding" "github.com/dxta-dev/app/internal/onboarding/activities" + "github.com/dxta-dev/app/internal/onboarding/data" "github.com/dxta-dev/app/internal/onboarding/workflows" "go.temporal.io/sdk/client" "go.temporal.io/sdk/worker" @@ -17,6 +18,12 @@ func main() { log.Fatalln("Failed to load configuration:", err) } + err = data.LoadGithubConfig() + + if err != nil { + log.Fatalln("Failed to load github configuration:", err) + } + temporalClient, err := client.Dial(client.Options{ HostPort: cfg.TemporalHostPort, Namespace: cfg.TemporalOnboardingNamespace, @@ -26,12 +33,19 @@ func main() { } defer temporalClient.Close() + err = data.InitAppClient() + + if err != nil { + log.Fatalf("Unable to init app client: %v", err) + } + err = onboarding.RegisterNamespace( context.Background(), cfg.TemporalHostPort, cfg.TemporalOnboardingNamespace, 30, ) + if err != nil { log.Fatalln("Failed to register Temporal namespace:", err) } @@ -45,6 +59,9 @@ func main() { w.RegisterWorkflow(workflows.CountUsers) w.RegisterActivity(userActivities) + /* w.RegisterWorkflow(workflow.ProvisionGithubInstallationData) + w.RegisterActivity(activity.GetGithubInstallation) */ + if err := w.Run(worker.InterruptCh()); err != nil { log.Fatalln("Worker failed to start", err) } diff --git a/go.mod b/go.mod index 5a05ba23..e65c786c 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( require ( github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect + github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 // indirect github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect @@ -28,8 +29,12 @@ require ( github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.3 // indirect + github.com/gofri/go-github-ratelimit/v2 v2.0.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/mock v1.6.0 // indirect + github.com/google/go-github/v72 v72.0.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect diff --git a/go.sum b/go.sum index 17940fa9..c0311374 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/XSAM/otelsql v0.39.0/go.mod h1:uMOXLUX+wkuAuP0AR3B45NXX7E9lJS2mERa8gq github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 h1:B91r9bHtXp/+XRgS5aZm6ZzTdz3ahgJYmkt4xZkgDz8= +github.com/bradleyfalzon/ghinstallation/v2 v2.16.0/go.mod h1:OeVe5ggFzoBnmgitZe/A+BqGOnv1DvU/0uiLQi1wutM= github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -54,8 +56,12 @@ github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gofri/go-github-ratelimit/v2 v2.0.2 h1:gS8wAS1jTmlWGdTjAM7KIpsLjwY1S0S/gKK5hthfSXM= +github.com/gofri/go-github-ratelimit/v2 v2.0.2/go.mod h1:YBQt4gTbdcbMjJFT05YFEaECwH78P5b0IwrnbLiHGdE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= @@ -68,9 +74,14 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-github/v72 v72.0.0 h1:FcIO37BLoVPBO9igQQ6tStsv2asG4IPcYFi655PPvBM= +github.com/google/go-github/v72 v72.0.0/go.mod h1:WWtw8GMRiL62mvIquf1kO3onRHeWWKmK01qdCY8c5fg= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/internal/internal-api/data/tenantDB.go b/internal/internal-api/data/tenantDB.go index 3119ac45..6e428bd0 100644 --- a/internal/internal-api/data/tenantDB.go +++ b/internal/internal-api/data/tenantDB.go @@ -1,6 +1,7 @@ package data import ( + "context" "database/sql" "fmt" "os" @@ -12,7 +13,7 @@ type TenantDB struct { DB *sql.DB } -func NewTenantDB(dbUrl string) (TenantDB, error) { +func NewTenantDB(dbUrl string, ctx context.Context) (TenantDB, error) { driverName := otel.GetDriverName() devToken := os.Getenv("DXTA_DEV_GROUP_TOKEN") @@ -30,6 +31,10 @@ func NewTenantDB(dbUrl string) (TenantDB, error) { return TenantDB{}, err } + if err := tenantDB.PingContext(ctx); err != nil { + return TenantDB{}, err + } + return TenantDB{ DB: tenantDB, }, nil diff --git a/internal/internal-api/handler/add_member_to_team.go b/internal/internal-api/handler/add_member_to_team.go index 7b8ed1b5..acd2ad84 100644 --- a/internal/internal-api/handler/add_member_to_team.go +++ b/internal/internal-api/handler/add_member_to_team.go @@ -18,7 +18,14 @@ type AddMemberToTeamResponse struct { func AddMemberToTeam(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - apiState := ctx.Value(util.ApiStateCtxKey).(api.State) + authId := ctx.Value(util.AuthIdCtxKey).(string) + + apiState, err := api.InternalApiState(authId, ctx) + + if err != nil { + util.JSONError(w, util.ErrorParam{Error: "Internal Server Error"}, http.StatusInternalServerError) + return + } teamId, err := strconv.ParseInt(chi.URLParam(r, "team_id"), 10, 64) if err != nil { diff --git a/internal/internal-api/handler/create_member.go b/internal/internal-api/handler/create_member.go index e978ccda..42a8e1d3 100644 --- a/internal/internal-api/handler/create_member.go +++ b/internal/internal-api/handler/create_member.go @@ -34,7 +34,14 @@ func CreateMember(w http.ResponseWriter, r *http.Request) { util.JSONError(w, util.ErrorParam{Error: "Bad Request"}, http.StatusBadRequest) } - apiState := ctx.Value(util.ApiStateCtxKey).(api.State) + authId := ctx.Value(util.AuthIdCtxKey).(string) + + apiState, err := api.InternalApiState(authId, ctx) + + if err != nil { + util.JSONError(w, util.ErrorParam{Error: "Internal Server Error"}, http.StatusInternalServerError) + return + } newMemberRes, err := apiState.DB.CreateMember(body.Name, body.Email, ctx) diff --git a/internal/internal-api/handler/create_team.go b/internal/internal-api/handler/create_team.go index c941881b..6293be5b 100644 --- a/internal/internal-api/handler/create_team.go +++ b/internal/internal-api/handler/create_team.go @@ -28,18 +28,26 @@ func CreateTeam(w http.ResponseWriter, r *http.Request) { return } - organizationId := ctx.Value(util.OrganizationIdCtxKey).(int64) - - if organizationId == 0 || body.TeamName == "" { - fmt.Printf( - "No organization id or team name provided. Organization id: %d Team name: %s", - organizationId, - body.TeamName, - ) + if body.TeamName == "" { + fmt.Printf("No team name provided. Team name: %s", body.TeamName) util.JSONError(w, util.ErrorParam{Error: "Bad Request"}, http.StatusBadRequest) } - apiState := ctx.Value(util.ApiStateCtxKey).(api.State) + authId := ctx.Value(util.AuthIdCtxKey).(string) + + apiState, err := api.InternalApiState(authId, ctx) + + if err != nil { + util.JSONError(w, util.ErrorParam{Error: "Internal Server Error"}, http.StatusInternalServerError) + return + } + + organizationId, err := apiState.DB.GetOrganizationIdByAuthId(authId, ctx) + + if err != nil { + util.JSONError(w, util.ErrorParam{Error: "Bad request"}, http.StatusBadRequest) + return + } newTeamRes, err := apiState.DB.CreateTeam(body.TeamName, organizationId, ctx) diff --git a/internal/internal-api/handler/provision_github_installation_data.go b/internal/internal-api/handler/provision_github_installation_data.go new file mode 100644 index 00000000..60ca8d62 --- /dev/null +++ b/internal/internal-api/handler/provision_github_installation_data.go @@ -0,0 +1,58 @@ +package handler + +import ( + "encoding/json" + "fmt" + "net/http" + "strconv" + + "github.com/dxta-dev/app/internal/onboarding/workflows" + "github.com/dxta-dev/app/internal/util" + "github.com/go-chi/chi/v5" + "github.com/google/go-github/v72/github" +) + +type Response struct { + Installations *github.Installation `json:"installations"` +} + +func (t *OnboardingTemporal) ProvisionGithubInstallationData(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + installationId, err := strconv.ParseInt(chi.URLParam(r, "installation_id"), 10, 64) + if err != nil { + fmt.Printf("Issue while parsing installation id URL param. Error: %s", err.Error()) + util.JSONError(w, util.ErrorParam{Error: "Bad Request"}, http.StatusBadRequest) + return + } + + out, err := workflows.ExecuteGithubInstallationDataProvision( + ctx, + t.temporalClient, + workflows.Args{ + TemporalOnboardingQueueName: t.config.TemporalOnboardingNamespace, + InstallationId: installationId, + }) + + if err != nil { + fmt.Println(err.Error()) + util.JSONError( + w, + util.ErrorParam{Error: "Internal Server Error"}, + http.StatusInternalServerError, + ) + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + if err := json.NewEncoder(w).Encode(Response{Installations: out}); err != nil { + fmt.Printf("Issue while formatting response. Error: %s", err.Error()) + util.JSONError( + w, + util.ErrorParam{Error: "Internal Server Error"}, + http.StatusInternalServerError, + ) + return + } +} diff --git a/internal/internal-api/handler/temporal.go b/internal/internal-api/handler/temporal.go new file mode 100644 index 00000000..181a988c --- /dev/null +++ b/internal/internal-api/handler/temporal.go @@ -0,0 +1,18 @@ +package handler + +import ( + "github.com/dxta-dev/app/internal/onboarding" + "go.temporal.io/sdk/client" +) + +type OnboardingTemporal struct { + temporalClient client.Client + config onboarding.Config +} + +func SetupOnboardingTemporal(temporalClient client.Client, config onboarding.Config) *OnboardingTemporal { + return &OnboardingTemporal{ + temporalClient: temporalClient, + config: config, + } +} diff --git a/internal/internal-api/handler/users_count.go b/internal/internal-api/handler/users_count.go index 53841a4e..f9f51434 100644 --- a/internal/internal-api/handler/users_count.go +++ b/internal/internal-api/handler/users_count.go @@ -7,30 +7,16 @@ import ( "log" "net/http" - "github.com/dxta-dev/app/internal/onboarding" "github.com/dxta-dev/app/internal/onboarding/workflows" "github.com/dxta-dev/app/internal/util" - "go.temporal.io/sdk/client" ) type UsersCountResponse struct { Count int `json:"count"` } -type Users struct { - temporalClient client.Client - config onboarding.Config -} - -func NewUsers(temporalClient client.Client, config onboarding.Config) *Users { - return &Users{ - temporalClient: temporalClient, - config: config, - } -} - -func (u *Users) UsersCount(w http.ResponseWriter, r *http.Request) { - out, err := workflows.ExecuteCountUsersWorkflow(r.Context(), u.temporalClient, u.config) +func (t OnboardingTemporal) UsersCount(w http.ResponseWriter, r *http.Request) { + out, err := workflows.ExecuteCountUsersWorkflow(r.Context(), t.temporalClient, t.config) if err != nil { log.Fatal(errors.Unwrap(err)) } diff --git a/internal/internal-api/internal-api.go b/internal/internal-api/internal-api.go index 9d1bd9be..53583e02 100644 --- a/internal/internal-api/internal-api.go +++ b/internal/internal-api/internal-api.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "fmt" - "net/http" "os" "github.com/dxta-dev/app/internal/internal-api/data" @@ -59,8 +58,14 @@ func GetTenantDBUrlByAuthId(ctx context.Context, authId string) (TenantDBData, e return tenantData, nil } -func InternalApiState(dbUrl string, r *http.Request) (State, error) { - tenantDB, err := data.NewTenantDB(dbUrl) +func InternalApiState(authId string, ctx context.Context) (State, error) { + tenantData, err := GetTenantDBUrlByAuthId(ctx, authId) + + if err != nil { + return State{}, err + } + + tenantDB, err := data.NewTenantDB(tenantData.DBUrl, ctx) if err != nil { return State{}, err diff --git a/internal/onboarding/activity/provision-github-installation-data.go b/internal/onboarding/activity/provision-github-installation-data.go new file mode 100644 index 00000000..ceb10225 --- /dev/null +++ b/internal/onboarding/activity/provision-github-installation-data.go @@ -0,0 +1,18 @@ +package activity + +import ( + "context" + + "github.com/dxta-dev/app/internal/onboarding/data" + "github.com/google/go-github/v72/github" +) + +func GetGithubInstallation(ctx context.Context, installationId int64) (*github.Installation, error) { + installations, err := data.GithubConfig.GetGithubInstallation(installationId, ctx) + + if err != nil { + return nil, err + } + + return installations, nil +} diff --git a/internal/onboarding/data/github.go b/internal/onboarding/data/github.go new file mode 100644 index 00000000..4d125ad5 --- /dev/null +++ b/internal/onboarding/data/github.go @@ -0,0 +1,128 @@ +package data + +import ( + "encoding/base64" + "errors" + "fmt" + "net/http" + "os" + "strconv" + + "github.com/bradleyfalzon/ghinstallation/v2" + "github.com/gofri/go-github-ratelimit/v2/github_ratelimit" + "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" + "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" + "github.com/google/go-github/v72/github" +) + +var GithubConfig *GithubCfg + +type GithubCfg struct { + GithubAppId int64 + GithubAppPrivateKey []byte + GithubAppClient *github.Client + RoundTripper http.RoundTripper +} + +func LoadGithubConfig() error { + appIdStr := os.Getenv("GITHUB_APP_ID") + appPrivateKeyStr := os.Getenv("GITHUB_APP_PRIVATE_KEY") + + if appIdStr == "" { + return errors.New("GITHUB_APP_ID not set") + } + + if appPrivateKeyStr == "" { + return errors.New("GITHUB_APP_PRIVATE_KEY not set") + } + + appId, err := strconv.ParseInt(appIdStr, 10, 64) + + if err != nil { + return errors.New("could not parse app id string to int64") + } + + appPrivateKey, err := base64.StdEncoding.DecodeString(appPrivateKeyStr) + + if err != nil { + return errors.New("failed to decode base64 string") + } + + GithubConfig = &GithubCfg{ + GithubAppId: appId, + GithubAppPrivateKey: appPrivateKey, + GithubAppClient: nil, + RoundTripper: http.DefaultTransport, + } + + return nil +} + +func getInstallationTransport(tr http.RoundTripper, installationId int64) (http.RoundTripper, error) { + itt, err := ghinstallation.New(tr, GithubConfig.GithubAppId, installationId, GithubConfig.GithubAppPrivateKey) + + if err != nil { + return nil, fmt.Errorf("failed to create apps transport: %w", err) + } + + return itt, nil +} + +func getAppTransport(tr http.RoundTripper) (http.RoundTripper, error) { + atr, err := ghinstallation.NewAppsTransport(tr, GithubConfig.GithubAppId, GithubConfig.GithubAppPrivateKey) + + if err != nil { + return nil, fmt.Errorf("failed to create apps transport: %w", err) + } + + return atr, nil +} + +func createLimiter(tr http.RoundTripper) http.RoundTripper { + return github_ratelimit.New(tr, + github_primary_ratelimit.WithLimitDetectedCallback(func(ctx *github_primary_ratelimit.CallbackContext) { + fmt.Printf("Primary rate limit detected: category %s, reset time: %v\n", ctx.Category, ctx.ResetTime) + }), + github_secondary_ratelimit.WithLimitDetectedCallback(func(ctx *github_secondary_ratelimit.CallbackContext) { + fmt.Printf("Secondary rate limit detected: reset time: %v, total sleep time: %v\n", ctx.ResetTime, ctx.TotalSleepTime) + }), + ) +} + +func NewInstallationClient(installationId int64) (*github.Client, error) { + tr := GithubConfig.RoundTripper + tr, err := getInstallationTransport(tr, installationId) + + if err != nil { + return nil, err + } + + tr = createLimiter(tr) + + return github.NewClient(&http.Client{Transport: tr}), nil +} + +func InitAppClient() error { + tr := GithubConfig.RoundTripper + tr, err := getAppTransport(tr) + + if err != nil { + return err + } + + tr = createLimiter(tr) + + GithubConfig.GithubAppClient = github.NewClient(&http.Client{Transport: tr}) + + return nil +} + +type NewAppClient struct { + client *github.Client +} + +func AppClient(client *github.Client) *NewAppClient { + return &NewAppClient{ + client, + } +} diff --git a/internal/onboarding/data/github_installations.go b/internal/onboarding/data/github_installations.go new file mode 100644 index 00000000..2855066b --- /dev/null +++ b/internal/onboarding/data/github_installations.go @@ -0,0 +1,75 @@ +package data + +import ( + "context" + "fmt" + + "github.com/google/go-github/v72/github" +) + +func (cfg GithubCfg) GetGithubInstallation(installationId int64, ctx context.Context) (*github.Installation, error) { + githubAppClient := cfg.GithubAppClient + + installation, _, err := githubAppClient.Apps.GetInstallation(ctx, installationId) + + if err != nil { + fmt.Printf("Could not retrieve installation. Error: %v", err.Error()) + return nil, err + } + + return installation, nil +} + +/* func (d TenantDB) SyncGithubInstallationDataToTenant( + installationId int64, + installationOrgName string, + organizationId string, + ctx context.Context, +) error { + tx, err := d.DB.BeginTx(ctx, nil) + + if err != nil { + return err + } + + _, err = tx.Exec(` + INSERT INTO github_organizations + (github_app_installation_id, name) + VALUES + (?, ?);`, + installationId, installationOrgName) + + if err != nil { + _ = tx.Rollback() + return err + } + + _, err = tx.Exec(` + INSERT INTO organizations + (external_id) + VALUES + (?) + ON CONFLICT + (external_id) + DO NOTHING;`, + organizationId) + + if err != nil { + _ = tx.Rollback() + return err + } + + _, err = tx.Exec(` + INSERT INTO 'organizations_github_organizations' + ('organization_id', 'github_app_installation_id') + VALUES + (?, ?);`, + organizationId, installationId) + + if err != nil { + _ = tx.Rollback() + return err + } + + return nil +} */ diff --git a/internal/onboarding/workflows/provision-github-installation-data.go b/internal/onboarding/workflows/provision-github-installation-data.go new file mode 100644 index 00000000..647681f4 --- /dev/null +++ b/internal/onboarding/workflows/provision-github-installation-data.go @@ -0,0 +1,71 @@ +package workflows + +import ( + "context" + "fmt" + "time" + + api "github.com/dxta-dev/app/internal/internal-api" + "github.com/dxta-dev/app/internal/onboarding/activity" + "github.com/google/go-github/v72/github" + "go.temporal.io/sdk/client" + "go.temporal.io/sdk/workflow" +) + +func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64) (*github.Installation, error) { + ao := workflow.ActivityOptions{ + StartToCloseTimeout: time.Minute, + } + + ctx = workflow.WithActivityOptions(ctx, ao) + + // 1. Get installation + var installation *github.Installation + err := workflow.ExecuteActivity(ctx, activity.GetGithubInstallation, installationId).Get(ctx, &installation) + + if err != nil { + return nil, err + } + + // TO - DO + // 2. Add installation data to tenant + // 3. Add Teams to tenant + + return installation, nil +} + +type Args struct { + TemporalOnboardingQueueName string + InstallationId int64 + ApiState api.State +} + +func ExecuteGithubInstallationDataProvision( + ctx context.Context, + temporalClient client.Client, + args Args, +) (*github.Installation, error) { + + wr, err := temporalClient.ExecuteWorkflow( + ctx, + client.StartWorkflowOptions{ + ID: fmt.Sprintf("onboarding-workflow-%v", time.Now().Format("20060102150405")), + TaskQueue: args.TemporalOnboardingQueueName, + }, + ProvisionGithubInstallationData, + args.InstallationId, + ) + if err != nil { + return nil, err + } + + var installation *github.Installation + + err = wr.Get(ctx, &installation) + + if err != nil { + return nil, err + } + + return installation, nil +} diff --git a/internal/util/auth.go b/internal/util/auth.go index 9101bafa..6eeee77b 100644 --- a/internal/util/auth.go +++ b/internal/util/auth.go @@ -9,7 +9,6 @@ import ( "net/http" "os" - "github.com/dxta-dev/app/internal/internal-api" "github.com/go-chi/jwtauth/v5" ) @@ -92,8 +91,7 @@ type contextKey struct { } var ( - OrganizationIdCtxKey = contextKey{"organizationId"} - ApiStateCtxKey = contextKey{"apiState"} + AuthIdCtxKey = contextKey{"authId"} ) func Authenticator() func(http.Handler) http.Handler { @@ -102,7 +100,7 @@ func Authenticator() func(http.Handler) http.Handler { token, claims, err := jwtauth.FromContext(r.Context()) if err != nil { - fmt.Println("Error extracting token and claims from context") + fmt.Printf("Error extracting token and claims from context. Error: %s", err.Error()) JSONError(w, ErrorParam{Error: "Internal Server Error"}, http.StatusInternalServerError) return } @@ -116,36 +114,14 @@ func Authenticator() func(http.Handler) http.Handler { authId := claims["organizationId"].(string) if authId == "" { - fmt.Println("No organization id found in JWT payload") + fmt.Println("No auth id found in JWT payload") JSONError(w, ErrorParam{Error: "Bad request"}, http.StatusBadRequest) return } ctx := r.Context() - tenantData, err := api.GetTenantDBUrlByAuthId(ctx, authId) - - if err != nil { - JSONError(w, ErrorParam{Error: "Internal Server Error"}, http.StatusInternalServerError) - return - } - - apiState, err := api.InternalApiState(tenantData.DBUrl, r) - - if err != nil { - JSONError(w, ErrorParam{Error: "Internal Server Error"}, http.StatusInternalServerError) - return - } - - organizationId, err := apiState.DB.GetOrganizationIdByAuthId(authId, ctx) - - if err != nil { - JSONError(w, ErrorParam{Error: "Bad request"}, http.StatusBadRequest) - return - } - - ctx = context.WithValue(ctx, OrganizationIdCtxKey, organizationId) - ctx = context.WithValue(ctx, ApiStateCtxKey, apiState) + ctx = context.WithValue(ctx, AuthIdCtxKey, authId) next.ServeHTTP(w, r.WithContext(ctx)) } From 6dde82d89864cc1f9e80cb2bacf1964f4116e21e Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Mon, 30 Jun 2025 17:56:14 +0200 Subject: [PATCH 2/8] feat: Add installation data provision to tenant db activity --- cmd/onboarding-worker/main.go | 13 +++-- .../provision_github_installation_data.go | 15 ++++- internal/onboarding/activities/db.go | 13 +++++ .../onboarding/{data => activities}/github.go | 24 ++++---- .../provision_github_installation_data.go | 55 +++++++++++++++++++ .../provision-github-installation-data.go | 18 ------ .../onboarding/data/github_installations.go | 51 ++++++++++------- ... => provision_github_installation_data.go} | 38 +++++++++---- 8 files changed, 157 insertions(+), 70 deletions(-) create mode 100644 internal/onboarding/activities/db.go rename internal/onboarding/{data => activities}/github.go (85%) create mode 100644 internal/onboarding/activities/provision_github_installation_data.go delete mode 100644 internal/onboarding/activity/provision-github-installation-data.go rename internal/onboarding/workflows/{provision-github-installation-data.go => provision_github_installation_data.go} (51%) diff --git a/cmd/onboarding-worker/main.go b/cmd/onboarding-worker/main.go index 3d800509..2f09a0db 100644 --- a/cmd/onboarding-worker/main.go +++ b/cmd/onboarding-worker/main.go @@ -6,7 +6,6 @@ import ( "github.com/dxta-dev/app/internal/onboarding" "github.com/dxta-dev/app/internal/onboarding/activities" - "github.com/dxta-dev/app/internal/onboarding/data" "github.com/dxta-dev/app/internal/onboarding/workflows" "go.temporal.io/sdk/client" "go.temporal.io/sdk/worker" @@ -18,7 +17,7 @@ func main() { log.Fatalln("Failed to load configuration:", err) } - err = data.LoadGithubConfig() + githubConfig, err := activities.LoadGithubConfig() if err != nil { log.Fatalln("Failed to load github configuration:", err) @@ -33,7 +32,7 @@ func main() { } defer temporalClient.Close() - err = data.InitAppClient() + err = activities.InitAppClient() if err != nil { log.Fatalf("Unable to init app client: %v", err) @@ -55,12 +54,14 @@ func main() { userActivities := activities.NewUserActivites( *cfg, ) + githubActivities := activities.InitGHActivities(*githubConfig) + dbActivities := activities.InitDBActivities() w.RegisterWorkflow(workflows.CountUsers) + w.RegisterWorkflow(workflows.ProvisionGithubInstallationData) w.RegisterActivity(userActivities) - - /* w.RegisterWorkflow(workflow.ProvisionGithubInstallationData) - w.RegisterActivity(activity.GetGithubInstallation) */ + w.RegisterActivity(githubActivities) + w.RegisterActivity(dbActivities) if err := w.Run(worker.InterruptCh()); err != nil { log.Fatalln("Worker failed to start", err) diff --git a/internal/internal-api/handler/provision_github_installation_data.go b/internal/internal-api/handler/provision_github_installation_data.go index 60ca8d62..6165346c 100644 --- a/internal/internal-api/handler/provision_github_installation_data.go +++ b/internal/internal-api/handler/provision_github_installation_data.go @@ -6,6 +6,7 @@ import ( "net/http" "strconv" + api "github.com/dxta-dev/app/internal/internal-api" "github.com/dxta-dev/app/internal/onboarding/workflows" "github.com/dxta-dev/app/internal/util" "github.com/go-chi/chi/v5" @@ -20,18 +21,30 @@ func (t *OnboardingTemporal) ProvisionGithubInstallationData(w http.ResponseWrit ctx := r.Context() installationId, err := strconv.ParseInt(chi.URLParam(r, "installation_id"), 10, 64) + if err != nil { fmt.Printf("Issue while parsing installation id URL param. Error: %s", err.Error()) util.JSONError(w, util.ErrorParam{Error: "Bad Request"}, http.StatusBadRequest) return } + authId := ctx.Value(util.AuthIdCtxKey).(string) + + tenantData, err := api.GetTenantDBUrlByAuthId(ctx, authId) + + if err != nil { + util.JSONError(w, util.ErrorParam{Error: "Internal Server Error"}, http.StatusInternalServerError) + return + } + out, err := workflows.ExecuteGithubInstallationDataProvision( ctx, t.temporalClient, workflows.Args{ TemporalOnboardingQueueName: t.config.TemporalOnboardingNamespace, InstallationId: installationId, + AuthId: authId, + DBUrl: tenantData.DBUrl, }) if err != nil { @@ -46,7 +59,7 @@ func (t *OnboardingTemporal) ProvisionGithubInstallationData(w http.ResponseWrit w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(Response{Installations: out}); err != nil { + if err := json.NewEncoder(w).Encode(Response{Installations: out.Installation}); err != nil { fmt.Printf("Issue while formatting response. Error: %s", err.Error()) util.JSONError( w, diff --git a/internal/onboarding/activities/db.go b/internal/onboarding/activities/db.go new file mode 100644 index 00000000..e83e8cdd --- /dev/null +++ b/internal/onboarding/activities/db.go @@ -0,0 +1,13 @@ +package activities + +import ( + "sync" +) + +type DBActivities struct { + connections sync.Map +} + +func InitDBActivities() *DBActivities { + return &DBActivities{} +} diff --git a/internal/onboarding/data/github.go b/internal/onboarding/activities/github.go similarity index 85% rename from internal/onboarding/data/github.go rename to internal/onboarding/activities/github.go index 4d125ad5..d2a28cc1 100644 --- a/internal/onboarding/data/github.go +++ b/internal/onboarding/activities/github.go @@ -1,4 +1,4 @@ -package data +package activities import ( "encoding/base64" @@ -24,28 +24,28 @@ type GithubCfg struct { RoundTripper http.RoundTripper } -func LoadGithubConfig() error { +func LoadGithubConfig() (*GithubCfg, error) { appIdStr := os.Getenv("GITHUB_APP_ID") appPrivateKeyStr := os.Getenv("GITHUB_APP_PRIVATE_KEY") if appIdStr == "" { - return errors.New("GITHUB_APP_ID not set") + return nil, errors.New("GITHUB_APP_ID not set") } if appPrivateKeyStr == "" { - return errors.New("GITHUB_APP_PRIVATE_KEY not set") + return nil, errors.New("GITHUB_APP_PRIVATE_KEY not set") } appId, err := strconv.ParseInt(appIdStr, 10, 64) if err != nil { - return errors.New("could not parse app id string to int64") + return nil, errors.New("could not parse app id string to int64") } appPrivateKey, err := base64.StdEncoding.DecodeString(appPrivateKeyStr) if err != nil { - return errors.New("failed to decode base64 string") + return nil, errors.New("failed to decode base64 string") } GithubConfig = &GithubCfg{ @@ -55,7 +55,7 @@ func LoadGithubConfig() error { RoundTripper: http.DefaultTransport, } - return nil + return GithubConfig, nil } func getInstallationTransport(tr http.RoundTripper, installationId int64) (http.RoundTripper, error) { @@ -117,12 +117,10 @@ func InitAppClient() error { return nil } -type NewAppClient struct { - client *github.Client +type GithubActivities struct { + githubConfig GithubCfg } -func AppClient(client *github.Client) *NewAppClient { - return &NewAppClient{ - client, - } +func InitGHActivities(githubCfg GithubCfg) *GithubActivities { + return &GithubActivities{githubConfig: githubCfg} } diff --git a/internal/onboarding/activities/provision_github_installation_data.go b/internal/onboarding/activities/provision_github_installation_data.go new file mode 100644 index 00000000..1aa069c5 --- /dev/null +++ b/internal/onboarding/activities/provision_github_installation_data.go @@ -0,0 +1,55 @@ +package activities + +import ( + "context" + "database/sql" + + internal_api_data "github.com/dxta-dev/app/internal/internal-api/data" + "github.com/dxta-dev/app/internal/onboarding/data" + "github.com/google/go-github/v72/github" +) + +func (activity *GithubActivities) GetGithubInstallation(ctx context.Context, installationId int64) (*github.Installation, error) { + installations, err := data.GetGithubInstallation(installationId, activity.githubConfig.GithubAppClient, ctx) + + if err != nil { + return nil, err + } + + return installations, nil +} + +func (activity *DBActivities) SyncGithubInstallationDataToTenant(ctx context.Context, installationId int64, + installationOrgName string, + installationOrgId int64, + organizationId string, + dbUrl string) (bool, error) { + cacheKey := dbUrl + db, ok := activity.connections.Load(cacheKey) + + if !ok { + tenantDB, err := internal_api_data.NewTenantDB(cacheKey, ctx) + db = tenantDB.DB + + if err != nil { + return false, err + } + + activity.connections.Store(cacheKey, db) + } + + err := data.SyncGithubInstallationDataToTenant( + installationId, + installationOrgName, + installationOrgId, + organizationId, + db.(*sql.DB), ctx, + ) + + if err != nil { + return false, err + } + + return true, nil + +} diff --git a/internal/onboarding/activity/provision-github-installation-data.go b/internal/onboarding/activity/provision-github-installation-data.go deleted file mode 100644 index ceb10225..00000000 --- a/internal/onboarding/activity/provision-github-installation-data.go +++ /dev/null @@ -1,18 +0,0 @@ -package activity - -import ( - "context" - - "github.com/dxta-dev/app/internal/onboarding/data" - "github.com/google/go-github/v72/github" -) - -func GetGithubInstallation(ctx context.Context, installationId int64) (*github.Installation, error) { - installations, err := data.GithubConfig.GetGithubInstallation(installationId, ctx) - - if err != nil { - return nil, err - } - - return installations, nil -} diff --git a/internal/onboarding/data/github_installations.go b/internal/onboarding/data/github_installations.go index 2855066b..0db07743 100644 --- a/internal/onboarding/data/github_installations.go +++ b/internal/onboarding/data/github_installations.go @@ -2,14 +2,13 @@ package data import ( "context" + "database/sql" "fmt" "github.com/google/go-github/v72/github" ) -func (cfg GithubCfg) GetGithubInstallation(installationId int64, ctx context.Context) (*github.Installation, error) { - githubAppClient := cfg.GithubAppClient - +func GetGithubInstallation(installationId int64, githubAppClient *github.Client, ctx context.Context) (*github.Installation, error) { installation, _, err := githubAppClient.Apps.GetInstallation(ctx, installationId) if err != nil { @@ -20,56 +19,66 @@ func (cfg GithubCfg) GetGithubInstallation(installationId int64, ctx context.Con return installation, nil } -/* func (d TenantDB) SyncGithubInstallationDataToTenant( +func SyncGithubInstallationDataToTenant( installationId int64, installationOrgName string, + installationOrgId int64, organizationId string, + db *sql.DB, ctx context.Context, ) error { - tx, err := d.DB.BeginTx(ctx, nil) + tx, err := db.BeginTx(ctx, nil) if err != nil { return err } - _, err = tx.Exec(` + rows := tx.QueryRowContext(ctx, ` INSERT INTO github_organizations - (github_app_installation_id, name) + (github_app_installation_id, name, external_id) VALUES - (?, ?);`, - installationId, installationOrgName) + (?, ?, ?) + RETURNING id`, + installationId, installationOrgName, installationOrgId) + + var githubOrganizationId int64 + + err = rows.Scan(&githubOrganizationId) if err != nil { _ = tx.Rollback() return err } - _, err = tx.Exec(` - INSERT INTO organizations - (external_id) - VALUES - (?) - ON CONFLICT - (external_id) - DO NOTHING;`, + rows = tx.QueryRowContext(ctx, ` + SELECT id + FROM organizations + WHERE auth_id = ?;`, organizationId) + var orgId int64 + err = rows.Scan(&orgId) + if err != nil { _ = tx.Rollback() return err } _, err = tx.Exec(` - INSERT INTO 'organizations_github_organizations' - ('organization_id', 'github_app_installation_id') + INSERT INTO 'organizations__github_organizations' + ('organization_id', 'github_organization_id') VALUES (?, ?);`, - organizationId, installationId) + orgId, githubOrganizationId) if err != nil { _ = tx.Rollback() return err } + if err := tx.Commit(); err != nil { + return err + } + return nil -} */ +} diff --git a/internal/onboarding/workflows/provision-github-installation-data.go b/internal/onboarding/workflows/provision_github_installation_data.go similarity index 51% rename from internal/onboarding/workflows/provision-github-installation-data.go rename to internal/onboarding/workflows/provision_github_installation_data.go index 647681f4..f145137d 100644 --- a/internal/onboarding/workflows/provision-github-installation-data.go +++ b/internal/onboarding/workflows/provision_github_installation_data.go @@ -5,14 +5,17 @@ import ( "fmt" "time" - api "github.com/dxta-dev/app/internal/internal-api" - "github.com/dxta-dev/app/internal/onboarding/activity" + "github.com/dxta-dev/app/internal/onboarding/activities" "github.com/google/go-github/v72/github" "go.temporal.io/sdk/client" "go.temporal.io/sdk/workflow" ) -func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64) (*github.Installation, error) { +type GithubDataProvisionResponse struct { + Installation *github.Installation `json:"installation"` +} + +func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, authId string, dbUrl string) (*GithubDataProvisionResponse, error) { ao := workflow.ActivityOptions{ StartToCloseTimeout: time.Minute, } @@ -21,31 +24,42 @@ func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64) // 1. Get installation var installation *github.Installation - err := workflow.ExecuteActivity(ctx, activity.GetGithubInstallation, installationId).Get(ctx, &installation) + err := workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetGithubInstallation, installationId).Get(ctx, &installation) if err != nil { return nil, err } - // TO - DO + fmt.Printf("INSTALLATIONS: %v", installation) + // 2. Add installation data to tenant - // 3. Add Teams to tenant + var syncResult bool + err = workflow.ExecuteActivity(ctx, (*activities.DBActivities).SyncGithubInstallationDataToTenant, installationId, installation.Account.Login, installation.Account.ID, authId, dbUrl).Get(ctx, &syncResult) - return installation, nil + if err != nil { + return nil, err + } + + if installation.TargetType != nil && *installation.TargetType == "Organization" { + // 3. Add Teams to tenant + + } + + return &GithubDataProvisionResponse{Installation: installation}, nil } type Args struct { TemporalOnboardingQueueName string InstallationId int64 - ApiState api.State + AuthId string + DBUrl string } func ExecuteGithubInstallationDataProvision( ctx context.Context, temporalClient client.Client, args Args, -) (*github.Installation, error) { - +) (*GithubDataProvisionResponse, error) { wr, err := temporalClient.ExecuteWorkflow( ctx, client.StartWorkflowOptions{ @@ -54,12 +68,14 @@ func ExecuteGithubInstallationDataProvision( }, ProvisionGithubInstallationData, args.InstallationId, + args.AuthId, + args.DBUrl, ) if err != nil { return nil, err } - var installation *github.Installation + var installation *GithubDataProvisionResponse err = wr.Get(ctx, &installation) From 684aa2d501d139e6f4126b169d43b3aa8380a832 Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Mon, 30 Jun 2025 19:25:22 +0200 Subject: [PATCH 3/8] feat: Add installation teams and its members retrieval --- .../provision_github_installation_data.go | 7 +-- internal/onboarding/activities/github.go | 6 ++- .../provision_github_installation_data.go | 44 +++++++++++++++---- .../data/get_github_installation.go | 19 ++++++++ .../data/get_installation_team_members.go | 36 +++++++++++++++ .../onboarding/data/get_installation_teams.go | 39 ++++++++++++++++ ...ync_github_installation_data_to_tenant.go} | 23 +++------- .../provision_github_installation_data.go | 37 ++++++++++++++-- 8 files changed, 174 insertions(+), 37 deletions(-) create mode 100644 internal/onboarding/data/get_github_installation.go create mode 100644 internal/onboarding/data/get_installation_team_members.go create mode 100644 internal/onboarding/data/get_installation_teams.go rename internal/onboarding/data/{github_installations.go => sync_github_installation_data_to_tenant.go} (70%) diff --git a/internal/internal-api/handler/provision_github_installation_data.go b/internal/internal-api/handler/provision_github_installation_data.go index 6165346c..2dc6a335 100644 --- a/internal/internal-api/handler/provision_github_installation_data.go +++ b/internal/internal-api/handler/provision_github_installation_data.go @@ -10,13 +10,8 @@ import ( "github.com/dxta-dev/app/internal/onboarding/workflows" "github.com/dxta-dev/app/internal/util" "github.com/go-chi/chi/v5" - "github.com/google/go-github/v72/github" ) -type Response struct { - Installations *github.Installation `json:"installations"` -} - func (t *OnboardingTemporal) ProvisionGithubInstallationData(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -59,7 +54,7 @@ func (t *OnboardingTemporal) ProvisionGithubInstallationData(w http.ResponseWrit w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(Response{Installations: out.Installation}); err != nil { + if err := json.NewEncoder(w).Encode(workflows.GithubDataProvisionResponse{Installation: out.Installation, Teams: out.Teams}); err != nil { fmt.Printf("Issue while formatting response. Error: %s", err.Error()) util.JSONError( w, diff --git a/internal/onboarding/activities/github.go b/internal/onboarding/activities/github.go index d2a28cc1..1afcc860 100644 --- a/internal/onboarding/activities/github.go +++ b/internal/onboarding/activities/github.go @@ -117,10 +117,12 @@ func InitAppClient() error { return nil } +type NewInstallationClientFunc func(installationId int64) (*github.Client, error) type GithubActivities struct { - githubConfig GithubCfg + githubConfig GithubCfg + newInstallationClient NewInstallationClientFunc } func InitGHActivities(githubCfg GithubCfg) *GithubActivities { - return &GithubActivities{githubConfig: githubCfg} + return &GithubActivities{githubConfig: githubCfg, newInstallationClient: NewInstallationClient} } diff --git a/internal/onboarding/activities/provision_github_installation_data.go b/internal/onboarding/activities/provision_github_installation_data.go index 1aa069c5..30a32d03 100644 --- a/internal/onboarding/activities/provision_github_installation_data.go +++ b/internal/onboarding/activities/provision_github_installation_data.go @@ -3,20 +3,18 @@ package activities import ( "context" "database/sql" + "fmt" internal_api_data "github.com/dxta-dev/app/internal/internal-api/data" "github.com/dxta-dev/app/internal/onboarding/data" "github.com/google/go-github/v72/github" ) -func (activity *GithubActivities) GetGithubInstallation(ctx context.Context, installationId int64) (*github.Installation, error) { - installations, err := data.GetGithubInstallation(installationId, activity.githubConfig.GithubAppClient, ctx) - - if err != nil { - return nil, err - } - - return installations, nil +func (activity *GithubActivities) GetGithubInstallation( + ctx context.Context, + installationId int64, +) (*github.Installation, error) { + return data.GetGithubInstallation(installationId, activity.githubConfig.GithubAppClient, ctx) } func (activity *DBActivities) SyncGithubInstallationDataToTenant(ctx context.Context, installationId int64, @@ -51,5 +49,35 @@ func (activity *DBActivities) SyncGithubInstallationDataToTenant(ctx context.Con } return true, nil +} + +func (activity *GithubActivities) GetInstallationTeams( + ctx context.Context, + installationOrgName string, + installationId int64, +) ([]*github.Team, error) { + client, err := activity.newInstallationClient(installationId) + + if err != nil { + fmt.Printf("Could not create new installation client. Error: %v", err.Error()) + return nil, err + } + + return data.GetInstallationTeams(ctx, installationOrgName, client) +} + +func (activity *GithubActivities) GetInstallationTeamMembers( + ctx context.Context, + installationId int64, + installationOrgName string, + teamSlug string, +) ([]*github.User, error) { + client, err := activity.newInstallationClient(installationId) + + if err != nil { + fmt.Printf("Could not create new installation client. Error: %v", err.Error()) + return nil, err + } + return data.GetInstallationTeamMembers(ctx, installationOrgName, teamSlug, client /* extendWithEmail= */, false) } diff --git a/internal/onboarding/data/get_github_installation.go b/internal/onboarding/data/get_github_installation.go new file mode 100644 index 00000000..6397df3d --- /dev/null +++ b/internal/onboarding/data/get_github_installation.go @@ -0,0 +1,19 @@ +package data + +import ( + "context" + "fmt" + + "github.com/google/go-github/v72/github" +) + +func GetGithubInstallation(installationId int64, githubAppClient *github.Client, ctx context.Context) (*github.Installation, error) { + installation, _, err := githubAppClient.Apps.GetInstallation(ctx, installationId) + + if err != nil { + fmt.Printf("Could not retrieve installation. Error: %v", err.Error()) + return nil, err + } + + return installation, nil +} diff --git a/internal/onboarding/data/get_installation_team_members.go b/internal/onboarding/data/get_installation_team_members.go new file mode 100644 index 00000000..b5e5031f --- /dev/null +++ b/internal/onboarding/data/get_installation_team_members.go @@ -0,0 +1,36 @@ +package data + +import ( + "context" + "fmt" + + "github.com/google/go-github/v72/github" +) + +func GetInstallationTeamMembers(ctx context.Context, installationOrgName string, teamSlug string, client *github.Client, extendWithEmail bool) ([]*github.User, error) { + opts := &github.TeamListTeamMembersOptions{ListOptions: github.ListOptions{PerPage: 100}} + + var allMembers []*github.User + + for { + members, res, err := client.Teams.ListTeamMembersBySlug(ctx, installationOrgName, teamSlug, opts) + + if err != nil { + fmt.Printf("Could not retrieve installation. Error: %v", err.Error()) + return nil, err + } + + // TO-DO Handle if we need to extend member with email. + // For each member request towards github is needed so it + // makes sense to run each request in its own go routine + + allMembers = append(allMembers, members...) + + if res.NextPage == 0 { + break + } + + opts.Page = res.NextPage + } + return allMembers, nil +} diff --git a/internal/onboarding/data/get_installation_teams.go b/internal/onboarding/data/get_installation_teams.go new file mode 100644 index 00000000..769f7800 --- /dev/null +++ b/internal/onboarding/data/get_installation_teams.go @@ -0,0 +1,39 @@ +package data + +import ( + "context" + "fmt" + + "github.com/google/go-github/v72/github" +) + +func GetInstallationTeams( + ctx context.Context, + installationOrgName string, + client *github.Client, +) ([]*github.Team, error) { + + opt := &github.ListOptions{PerPage: 100} + + var allTeams []*github.Team + + for { + teams, res, err := client.Teams.ListTeams(ctx, installationOrgName, opt) + + if err != nil { + fmt.Printf("Could not retrieve installation. Error: %v", err.Error()) + return nil, err + } + + allTeams = append(allTeams, teams...) + + if res.NextPage == 0 { + break + } + + opt.Page = res.NextPage + } + + return allTeams, nil + +} diff --git a/internal/onboarding/data/github_installations.go b/internal/onboarding/data/sync_github_installation_data_to_tenant.go similarity index 70% rename from internal/onboarding/data/github_installations.go rename to internal/onboarding/data/sync_github_installation_data_to_tenant.go index 0db07743..c16c3a1d 100644 --- a/internal/onboarding/data/github_installations.go +++ b/internal/onboarding/data/sync_github_installation_data_to_tenant.go @@ -3,22 +3,8 @@ package data import ( "context" "database/sql" - "fmt" - - "github.com/google/go-github/v72/github" ) -func GetGithubInstallation(installationId int64, githubAppClient *github.Client, ctx context.Context) (*github.Installation, error) { - installation, _, err := githubAppClient.Apps.GetInstallation(ctx, installationId) - - if err != nil { - fmt.Printf("Could not retrieve installation. Error: %v", err.Error()) - return nil, err - } - - return installation, nil -} - func SyncGithubInstallationDataToTenant( installationId int64, installationOrgName string, @@ -51,9 +37,12 @@ func SyncGithubInstallationDataToTenant( } rows = tx.QueryRowContext(ctx, ` - SELECT id - FROM organizations - WHERE auth_id = ?;`, + SELECT + id + FROM + organizations + WHERE + auth_id = ?;`, organizationId) var orgId int64 diff --git a/internal/onboarding/workflows/provision_github_installation_data.go b/internal/onboarding/workflows/provision_github_installation_data.go index f145137d..2a2baa2d 100644 --- a/internal/onboarding/workflows/provision_github_installation_data.go +++ b/internal/onboarding/workflows/provision_github_installation_data.go @@ -11,8 +11,14 @@ import ( "go.temporal.io/sdk/workflow" ) +type TeamWithMembers struct { + Team *github.Team + Members []*github.User +} + type GithubDataProvisionResponse struct { Installation *github.Installation `json:"installation"` + Teams []TeamWithMembers `json:"teams"` } func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, authId string, dbUrl string) (*GithubDataProvisionResponse, error) { @@ -30,8 +36,6 @@ func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, return nil, err } - fmt.Printf("INSTALLATIONS: %v", installation) - // 2. Add installation data to tenant var syncResult bool err = workflow.ExecuteActivity(ctx, (*activities.DBActivities).SyncGithubInstallationDataToTenant, installationId, installation.Account.Login, installation.Account.ID, authId, dbUrl).Get(ctx, &syncResult) @@ -40,12 +44,37 @@ func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, return nil, err } + // 3. Add Teams to tenant + var teamsWithMembers []TeamWithMembers if installation.TargetType != nil && *installation.TargetType == "Organization" { - // 3. Add Teams to tenant + var teams []*github.Team + // 3.1 Retrieve all teams and its members + err := workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetInstallationTeams, installation.Account.Login, installationId).Get(ctx, &teams) + + if err != nil { + return nil, err + } + + // 3.2 Retrieve members for each team + // TO-DO Run this in go routines instead in sequence + for _, team := range teams { + teamWithMembers := TeamWithMembers{Team: team, Members: []*github.User{}} + + var members []*github.User + + err := workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetInstallationTeamMembers, installationId, installation.Account.Login, *team.Slug).Get(ctx, &members) + + if err != nil { + return nil, err + } + + teamWithMembers.Members = members + teamsWithMembers = append(teamsWithMembers, teamWithMembers) + } } - return &GithubDataProvisionResponse{Installation: installation}, nil + return &GithubDataProvisionResponse{Installation: installation, Teams: teamsWithMembers}, nil } type Args struct { From 47278fa8e01e3e1de09d77c39d841fd24d576e70 Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Tue, 1 Jul 2025 19:29:16 +0200 Subject: [PATCH 4/8] feat: Add teams and members to tenant db --- .../provision_github_installation_data.go | 13 +- internal/onboarding/activities/github.go | 3 +- .../provision_github_installation_data.go | 169 +++++++++++++++++- .../data/get_installation_team_members.go | 57 +++++- .../onboarding/data/get_installation_teams.go | 5 + ...sync_github_installation_data_to_tenant.go | 22 ++- .../provision_github_installation_data.go | 74 ++++---- 7 files changed, 275 insertions(+), 68 deletions(-) diff --git a/internal/internal-api/handler/provision_github_installation_data.go b/internal/internal-api/handler/provision_github_installation_data.go index 2dc6a335..cfe73d5f 100644 --- a/internal/internal-api/handler/provision_github_installation_data.go +++ b/internal/internal-api/handler/provision_github_installation_data.go @@ -32,7 +32,7 @@ func (t *OnboardingTemporal) ProvisionGithubInstallationData(w http.ResponseWrit return } - out, err := workflows.ExecuteGithubInstallationDataProvision( + workflows.ExecuteGithubInstallationDataProvision( ctx, t.temporalClient, workflows.Args{ @@ -42,19 +42,10 @@ func (t *OnboardingTemporal) ProvisionGithubInstallationData(w http.ResponseWrit DBUrl: tenantData.DBUrl, }) - if err != nil { - fmt.Println(err.Error()) - util.JSONError( - w, - util.ErrorParam{Error: "Internal Server Error"}, - http.StatusInternalServerError, - ) - } - w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(workflows.GithubDataProvisionResponse{Installation: out.Installation, Teams: out.Teams}); err != nil { + if err := json.NewEncoder(w).Encode("Success"); err != nil { fmt.Printf("Issue while formatting response. Error: %s", err.Error()) util.JSONError( w, diff --git a/internal/onboarding/activities/github.go b/internal/onboarding/activities/github.go index 1afcc860..bbfda0cc 100644 --- a/internal/onboarding/activities/github.go +++ b/internal/onboarding/activities/github.go @@ -117,10 +117,9 @@ func InitAppClient() error { return nil } -type NewInstallationClientFunc func(installationId int64) (*github.Client, error) type GithubActivities struct { githubConfig GithubCfg - newInstallationClient NewInstallationClientFunc + newInstallationClient func(installationId int64) (*github.Client, error) } func InitGHActivities(githubCfg GithubCfg) *GithubActivities { diff --git a/internal/onboarding/activities/provision_github_installation_data.go b/internal/onboarding/activities/provision_github_installation_data.go index 30a32d03..3753c271 100644 --- a/internal/onboarding/activities/provision_github_installation_data.go +++ b/internal/onboarding/activities/provision_github_installation_data.go @@ -21,7 +21,7 @@ func (activity *DBActivities) SyncGithubInstallationDataToTenant(ctx context.Con installationOrgName string, installationOrgId int64, organizationId string, - dbUrl string) (bool, error) { + dbUrl string) (*data.SyncGithubDataResult, error) { cacheKey := dbUrl db, ok := activity.connections.Load(cacheKey) @@ -30,13 +30,13 @@ func (activity *DBActivities) SyncGithubInstallationDataToTenant(ctx context.Con db = tenantDB.DB if err != nil { - return false, err + return nil, err } activity.connections.Store(cacheKey, db) } - err := data.SyncGithubInstallationDataToTenant( + res, err := data.SyncGithubInstallationDataToTenant( installationId, installationOrgName, installationOrgId, @@ -45,10 +45,10 @@ func (activity *DBActivities) SyncGithubInstallationDataToTenant(ctx context.Con ) if err != nil { - return false, err + return nil, err } - return true, nil + return res, nil } func (activity *GithubActivities) GetInstallationTeams( @@ -78,6 +78,163 @@ func (activity *GithubActivities) GetInstallationTeamMembers( fmt.Printf("Could not create new installation client. Error: %v", err.Error()) return nil, err } + return data.GetInstallationTeamMembers(ctx, installationOrgName, teamSlug, client) +} + +func (activity *GithubActivities) GetInstallationTeamMembersWithEmails(ctx context.Context, installationId int64, members []*github.User) (data.ExtendedMembers, error) { + + client, err := activity.newInstallationClient(installationId) + + if err != nil { + fmt.Printf("Could not create new installation client. Error: %v", err.Error()) + return nil, err + } + + return data.GetInstallationTeamMembersWithEmails(ctx, members, client) +} + +func (activity *DBActivities) SyncTeamsAndMembersToTenant( + ctx context.Context, + teamWithMembers data.TeamWithMembers, + dbUrl string, + githubOrganizationId int64, + organizationId int64, +) (bool, error) { + cacheKey := dbUrl + d, ok := activity.connections.Load(cacheKey) + + if !ok { + tenantDB, err := internal_api_data.NewTenantDB(cacheKey, ctx) + d = tenantDB.DB + + if err != nil { + return false, err + } + + activity.connections.Store(cacheKey, d) + } + db := d.(*sql.DB) + + tx, err := db.BeginTx(ctx, nil) + + if err != nil { + return false, err + } + + rows := tx.QueryRowContext(ctx, ` + INSERT INTO teams + (name, organization_id) + VALUES + (?, ?) + RETURNING + id;`, + teamWithMembers.Team.Name, organizationId) + + var teamId int64 + + err = rows.Scan(&teamId) + + if err != nil { + fmt.Println("Issue creating team") + _ = tx.Rollback() + return false, err + } + + rows = tx.QueryRowContext(ctx, ` + INSERT INTO github_teams + (name, external_id, github_organization_id) + VALUES + (?, ?, ?) + RETURNING + id;`, + teamWithMembers.Team.Name, teamWithMembers.Team.ID, githubOrganizationId) + + var githubTeamId int64 + + err = rows.Scan(&githubTeamId) + + if err != nil { + fmt.Println("Issue creating github team") + + _ = tx.Rollback() + return false, err + } + + for _, member := range teamWithMembers.Members { + name := member.Name + + if name == nil { + defaultName := "DXTA member" + name = &defaultName + } + + rowRes := tx.QueryRowContext(ctx, ` + INSERT INTO members + (name, email) + VALUES + (?, ?) + RETURNING id;`, + name, member.Email) + + var memberId int64 + + err = rowRes.Scan(&memberId) + + if err != nil { + fmt.Println("Issue creating member") + + _ = tx.Rollback() + return false, err + } + + _, err = tx.Exec(` + INSERT INTO teams__members + (team_id, member_id) + VALUES + (?, ?);`, + teamId, memberId) + + if err != nil { + fmt.Println("Issue creating teams__members") + _ = tx.Rollback() + return false, err + } + + rowRes = tx.QueryRowContext(ctx, ` + INSERT INTO github_members + (external_id, username, email, member_id) + VALUES + (?, ?, ?, ?) + RETURNING id;`, + member.ID, member.Login, member.Email, memberId) + + var githubMemberId int64 + + err = rowRes.Scan(&githubMemberId) + + if err != nil { + fmt.Println("Issue creating github member") + _ = tx.Rollback() + return false, err + } + + _, err = tx.Exec(` + INSERT INTO github_teams__github_members + (github_team_id, github_member_id) + VALUES + (?, ?);`, + githubTeamId, githubMemberId) + + if err != nil { + fmt.Println("Issue creating github_teams__github_members") + _ = tx.Rollback() + return false, err + } + } - return data.GetInstallationTeamMembers(ctx, installationOrgName, teamSlug, client /* extendWithEmail= */, false) + if err := tx.Commit(); err != nil { + return false, err + } + + return true, nil } diff --git a/internal/onboarding/data/get_installation_team_members.go b/internal/onboarding/data/get_installation_team_members.go index b5e5031f..d7eb6ca7 100644 --- a/internal/onboarding/data/get_installation_team_members.go +++ b/internal/onboarding/data/get_installation_team_members.go @@ -3,11 +3,21 @@ package data import ( "context" "fmt" + "sync" "github.com/google/go-github/v72/github" + "golang.org/x/sync/errgroup" ) -func GetInstallationTeamMembers(ctx context.Context, installationOrgName string, teamSlug string, client *github.Client, extendWithEmail bool) ([]*github.User, error) { +type ExtendedMember struct { + *github.User + Email *string `json:"email,omitempty"` + Name *string `json:"name,omitempty"` +} + +type ExtendedMembers []ExtendedMember + +func GetInstallationTeamMembers(ctx context.Context, installationOrgName string, teamSlug string, client *github.Client) ([]*github.User, error) { opts := &github.TeamListTeamMembersOptions{ListOptions: github.ListOptions{PerPage: 100}} var allMembers []*github.User @@ -20,10 +30,6 @@ func GetInstallationTeamMembers(ctx context.Context, installationOrgName string, return nil, err } - // TO-DO Handle if we need to extend member with email. - // For each member request towards github is needed so it - // makes sense to run each request in its own go routine - allMembers = append(allMembers, members...) if res.NextPage == 0 { @@ -32,5 +38,46 @@ func GetInstallationTeamMembers(ctx context.Context, installationOrgName string, opts.Page = res.NextPage } + return allMembers, nil } + +type AllMembersContainer struct { + mu sync.Mutex + allMembers ExtendedMembers +} + +func (amc *AllMembersContainer) extendMember(member *github.User, Email *string, Name *string) { + amc.mu.Lock() + defer amc.mu.Unlock() + amc.allMembers = append(amc.allMembers, ExtendedMember{User: member, Email: Email, Name: Name}) +} + +func GetInstallationTeamMembersWithEmails(ctx context.Context, members []*github.User, client *github.Client) (ExtendedMembers, error) { + c := AllMembersContainer{ + allMembers: ExtendedMembers{}, + } + + g := new(errgroup.Group) + + for _, m := range members { + + g.Go(func() error { + user, _, err := client.Users.Get(ctx, *m.Login) + + if err != nil { + return err + } + + c.extendMember(m, user.Email, user.Name) + return nil + }) + + } + + if err := g.Wait(); err != nil { + return nil, err + } + + return c.allMembers, nil +} diff --git a/internal/onboarding/data/get_installation_teams.go b/internal/onboarding/data/get_installation_teams.go index 769f7800..6644c1a6 100644 --- a/internal/onboarding/data/get_installation_teams.go +++ b/internal/onboarding/data/get_installation_teams.go @@ -7,6 +7,11 @@ import ( "github.com/google/go-github/v72/github" ) +type TeamWithMembers struct { + Team *github.Team + Members ExtendedMembers +} + func GetInstallationTeams( ctx context.Context, installationOrgName string, diff --git a/internal/onboarding/data/sync_github_installation_data_to_tenant.go b/internal/onboarding/data/sync_github_installation_data_to_tenant.go index c16c3a1d..d294d090 100644 --- a/internal/onboarding/data/sync_github_installation_data_to_tenant.go +++ b/internal/onboarding/data/sync_github_installation_data_to_tenant.go @@ -5,6 +5,11 @@ import ( "database/sql" ) +type SyncGithubDataResult struct { + OrganizationId int64 + GithubOrganizationId int64 +} + func SyncGithubInstallationDataToTenant( installationId int64, installationOrgName string, @@ -12,11 +17,11 @@ func SyncGithubInstallationDataToTenant( organizationId string, db *sql.DB, ctx context.Context, -) error { +) (*SyncGithubDataResult, error) { tx, err := db.BeginTx(ctx, nil) if err != nil { - return err + return nil, err } rows := tx.QueryRowContext(ctx, ` @@ -33,7 +38,7 @@ func SyncGithubInstallationDataToTenant( if err != nil { _ = tx.Rollback() - return err + return nil, err } rows = tx.QueryRowContext(ctx, ` @@ -50,7 +55,7 @@ func SyncGithubInstallationDataToTenant( if err != nil { _ = tx.Rollback() - return err + return nil, err } _, err = tx.Exec(` @@ -62,12 +67,15 @@ func SyncGithubInstallationDataToTenant( if err != nil { _ = tx.Rollback() - return err + return nil, err } if err := tx.Commit(); err != nil { - return err + return nil, err } - return nil + return &SyncGithubDataResult{ + OrganizationId: orgId, + GithubOrganizationId: githubOrganizationId, + }, nil } diff --git a/internal/onboarding/workflows/provision_github_installation_data.go b/internal/onboarding/workflows/provision_github_installation_data.go index 2a2baa2d..bf574b36 100644 --- a/internal/onboarding/workflows/provision_github_installation_data.go +++ b/internal/onboarding/workflows/provision_github_installation_data.go @@ -6,75 +6,82 @@ import ( "time" "github.com/dxta-dev/app/internal/onboarding/activities" + "github.com/dxta-dev/app/internal/onboarding/data" "github.com/google/go-github/v72/github" "go.temporal.io/sdk/client" "go.temporal.io/sdk/workflow" ) -type TeamWithMembers struct { - Team *github.Team - Members []*github.User -} - type GithubDataProvisionResponse struct { - Installation *github.Installation `json:"installation"` - Teams []TeamWithMembers `json:"teams"` + Installation *github.Installation `json:"installation"` + Teams []data.TeamWithMembers `json:"teams"` } -func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, authId string, dbUrl string) (*GithubDataProvisionResponse, error) { +func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, authId string, dbUrl string) error { ao := workflow.ActivityOptions{ StartToCloseTimeout: time.Minute, } ctx = workflow.WithActivityOptions(ctx, ao) - // 1. Get installation + // 1. Get installation data var installation *github.Installation err := workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetGithubInstallation, installationId).Get(ctx, &installation) if err != nil { - return nil, err + return err } - // 2. Add installation data to tenant - var syncResult bool + // 2. Store installation data to tenant + var syncResult *data.SyncGithubDataResult err = workflow.ExecuteActivity(ctx, (*activities.DBActivities).SyncGithubInstallationDataToTenant, installationId, installation.Account.Login, installation.Account.ID, authId, dbUrl).Get(ctx, &syncResult) if err != nil { - return nil, err + return err } // 3. Add Teams to tenant - var teamsWithMembers []TeamWithMembers if installation.TargetType != nil && *installation.TargetType == "Organization" { + // 3.1 Retrieve all teams var teams []*github.Team - // 3.1 Retrieve all teams and its members err := workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetInstallationTeams, installation.Account.Login, installationId).Get(ctx, &teams) if err != nil { - return nil, err + return err } - // 3.2 Retrieve members for each team - // TO-DO Run this in go routines instead in sequence - for _, team := range teams { - teamWithMembers := TeamWithMembers{Team: team, Members: []*github.User{}} + // 3.2 Retrieve members and store teams and members to tenant db + for _, t := range teams { + team := t + teamWithMembers := data.TeamWithMembers{Team: team, Members: data.ExtendedMembers{}} var members []*github.User + err := workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetInstallationTeamMembers, installationId, installation.Account.Login, team.Slug).Get(ctx, &members) + + if err != nil { + return err + } + + var membersWithEmails *data.ExtendedMembers - err := workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetInstallationTeamMembers, installationId, installation.Account.Login, *team.Slug).Get(ctx, &members) + err = workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetInstallationTeamMembersWithEmails, installationId, members).Get(ctx, &membersWithEmails) if err != nil { - return nil, err + return err } - teamWithMembers.Members = members - teamsWithMembers = append(teamsWithMembers, teamWithMembers) - } + teamWithMembers.Members = *membersWithEmails + + var syncTeamsAndMembersRes *bool + err = workflow.ExecuteActivity(ctx, (*activities.DBActivities).SyncTeamsAndMembersToTenant, teamWithMembers, dbUrl, syncResult.GithubOrganizationId, syncResult.OrganizationId).Get(ctx, &syncTeamsAndMembersRes) + if err != nil { + return err + } + } } - return &GithubDataProvisionResponse{Installation: installation, Teams: teamsWithMembers}, nil + return nil } type Args struct { @@ -88,8 +95,8 @@ func ExecuteGithubInstallationDataProvision( ctx context.Context, temporalClient client.Client, args Args, -) (*GithubDataProvisionResponse, error) { - wr, err := temporalClient.ExecuteWorkflow( +) (string, error) { + _, err := temporalClient.ExecuteWorkflow( ctx, client.StartWorkflowOptions{ ID: fmt.Sprintf("onboarding-workflow-%v", time.Now().Format("20060102150405")), @@ -100,17 +107,10 @@ func ExecuteGithubInstallationDataProvision( args.AuthId, args.DBUrl, ) - if err != nil { - return nil, err - } - - var installation *GithubDataProvisionResponse - - err = wr.Get(ctx, &installation) if err != nil { - return nil, err + return "Unable to execute ", err } - return installation, nil + return "success", nil } From f9b3fed8fb804c344a6fbc2f2d4056ec8d68a230 Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Wed, 2 Jul 2025 14:29:17 +0200 Subject: [PATCH 5/8] fix: Workflow goroutines didn't yield too long. Update tenant db schema. Refactor --- .../tenant/migrations/1750420632_init.up.sql | 8 +- db/migrations/tenant/migrations/schema.sql | 6 +- .../provision_github_installation_data.go | 6 +- internal/onboarding/activities/db.go | 29 ++- .../{github.go => github_client.go} | 6 +- .../provision_github_installation_data.go | 170 ++---------------- ...sync_github_installation_data_to_tenant.go | 4 +- .../data/sync_teams_and_members_to_tenant.go | 159 ++++++++++++++++ .../onboarding/data/sync_teams_to_tenant.go | 56 ++++++ .../provision_github_installation_data.go | 71 ++++---- internal/util/auth.go | 6 +- 11 files changed, 322 insertions(+), 199 deletions(-) rename internal/onboarding/activities/{github.go => github_client.go} (95%) create mode 100644 internal/onboarding/data/sync_teams_and_members_to_tenant.go create mode 100644 internal/onboarding/data/sync_teams_to_tenant.go diff --git a/db/migrations/tenant/migrations/1750420632_init.up.sql b/db/migrations/tenant/migrations/1750420632_init.up.sql index b307836a..2ee5c0ef 100644 --- a/db/migrations/tenant/migrations/1750420632_init.up.sql +++ b/db/migrations/tenant/migrations/1750420632_init.up.sql @@ -24,7 +24,7 @@ CREATE TABLE "teams" ( "created_at" DATETIME NOT NULL DEFAULT (datetime('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime('now')), "deleted_at" DATETIME DEFAULT NULL, - FOREIGN KEY ("organization_id") references "organizations" ("id") + FOREIGN KEY ("organization_id") REFERENCES "organizations" ("id") ); CREATE TABLE "members" ( @@ -49,7 +49,7 @@ CREATE TABLE "teams__members" ( CREATE TABLE "github_organizations" ( "id" INTEGER PRIMARY KEY NOT NULL, - "external_id" INTEGER NOT NULL, + "external_id" INTEGER NOT NULL UNIQUE, "name" TEXT NOT NULL, "github_app_installation_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime('now')), @@ -69,7 +69,7 @@ CREATE TABLE "organizations__github_organizations" ( CREATE TABLE "github_members" ( "id" INTEGER PRIMARY KEY NOT NULL, - "external_id" INTEGER NOT NULL, + "external_id" INTEGER NOT NULL UNIQUE, "username" TEXT NOT NULL, "email" TEXT DEFAULT NULL, "member_id" INTEGER NULL, @@ -81,7 +81,7 @@ CREATE TABLE "github_members" ( CREATE TABLE "github_teams" ( "id" INTEGER PRIMARY KEY NOT NULL, - "external_id" INTEGER NOT NULL, + "external_id" INTEGER NOT NULL UNIQUE, "name" TEXT NOT NULL, "github_organization_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime('now')), diff --git a/db/migrations/tenant/migrations/schema.sql b/db/migrations/tenant/migrations/schema.sql index 3396232a..4c211f2e 100644 --- a/db/migrations/tenant/migrations/schema.sql +++ b/db/migrations/tenant/migrations/schema.sql @@ -9,10 +9,10 @@ CREATE TABLE "organizations" ("id" INTEGER PRIMARY KEY NOT NULL, "name" TEXT NOT CREATE TABLE "teams" ("id" INTEGER PRIMARY KEY NOT NULL, "name" TEXT NOT NULL, "organization_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL, FOREIGN KEY ("organization_id") REFERENCES "organizations" ("id")); CREATE TABLE "members" ("id" INTEGER PRIMARY KEY NOT NULL, "name" TEXT NOT NULL, "email" TEXT DEFAULT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL); CREATE TABLE "teams__members" ("team_id" INTEGER NOT NULL, "member_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), PRIMARY KEY ("team_id", "member_id"), FOREIGN KEY ("team_id") REFERENCES "teams" ("id"), FOREIGN KEY ("member_id") REFERENCES "members" ("id")); -CREATE TABLE "github_organizations" ("id" INTEGER PRIMARY KEY NOT NULL, "external_id" INTEGER NOT NULL, "name" TEXT NOT NULL, "github_app_installation_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL); +CREATE TABLE "github_organizations" ("id" INTEGER PRIMARY KEY NOT NULL, "external_id" INTEGER NOT NULL UNIQUE, "name" TEXT NOT NULL, "github_app_installation_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL); CREATE TABLE "organizations__github_organizations" ("organization_id" INTEGER NOT NULL, "github_organization_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), PRIMARY KEY ("organization_id", "github_organization_id"), FOREIGN KEY ("organization_id") REFERENCES "organizations" ("id"), FOREIGN KEY ("github_organization_id") REFERENCES "github_organizations" ("id")); -CREATE TABLE "github_members" ("id" INTEGER PRIMARY KEY NOT NULL, "external_id" INTEGER NOT NULL, "username" TEXT NOT NULL, "email" TEXT DEFAULT NULL, "member_id" INTEGER NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL, FOREIGN KEY ("member_id") REFERENCES "members" ("id")); -CREATE TABLE "github_teams" ("id" INTEGER PRIMARY KEY NOT NULL, "external_id" INTEGER NOT NULL, "name" TEXT NOT NULL, "github_organization_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL, FOREIGN KEY ("github_organization_id") REFERENCES "github_organizations" ("id")); +CREATE TABLE "github_members" ("id" INTEGER PRIMARY KEY NOT NULL, "external_id" INTEGER NOT NULL UNIQUE, "username" TEXT NOT NULL, "email" TEXT DEFAULT NULL, "member_id" INTEGER NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL, FOREIGN KEY ("member_id") REFERENCES "members" ("id")); +CREATE TABLE "github_teams" ("id" INTEGER PRIMARY KEY NOT NULL, "external_id" INTEGER NOT NULL UNIQUE, "name" TEXT NOT NULL, "github_organization_id" INTEGER NOT NULL, "created_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "updated_at" DATETIME NOT NULL DEFAULT (datetime ('now')), "deleted_at" DATETIME DEFAULT NULL, FOREIGN KEY ("github_organization_id") REFERENCES "github_organizations" ("id")); CREATE TABLE "github_teams__github_members" ("github_team_id" INTEGER NOT NULL, "github_member_id" INTEGER NOT NULL, PRIMARY KEY ("github_team_id", "github_member_id"), FOREIGN KEY ("github_team_id") REFERENCES "github_teams" ("id"), FOREIGN KEY ("github_member_id") REFERENCES "github_members" ("id")); CREATE TRIGGER "settings_set_updated_at" AFTER UPDATE ON "settings" FOR EACH ROW BEGIN UPDATE "settings" SET updated_at = datetime ('now') WHERE id = OLD.id; diff --git a/internal/internal-api/handler/provision_github_installation_data.go b/internal/internal-api/handler/provision_github_installation_data.go index cfe73d5f..fcd67857 100644 --- a/internal/internal-api/handler/provision_github_installation_data.go +++ b/internal/internal-api/handler/provision_github_installation_data.go @@ -12,6 +12,10 @@ import ( "github.com/go-chi/chi/v5" ) +type ProvisionGithubInstallationDataResponse struct { + Message string `json:"message"` +} + func (t *OnboardingTemporal) ProvisionGithubInstallationData(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -45,7 +49,7 @@ func (t *OnboardingTemporal) ProvisionGithubInstallationData(w http.ResponseWrit w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode("Success"); err != nil { + if err := json.NewEncoder(w).Encode(ProvisionGithubInstallationDataResponse{Message: "Success"}); err != nil { fmt.Printf("Issue while formatting response. Error: %s", err.Error()) util.JSONError( w, diff --git a/internal/onboarding/activities/db.go b/internal/onboarding/activities/db.go index e83e8cdd..67d8e638 100644 --- a/internal/onboarding/activities/db.go +++ b/internal/onboarding/activities/db.go @@ -1,13 +1,38 @@ package activities import ( + "context" + "database/sql" "sync" + + internal_api_data "github.com/dxta-dev/app/internal/internal-api/data" ) +func GetCachedTenantDB(store *sync.Map, dbUrl string, ctx context.Context) (*sql.DB, error) { + db, ok := store.Load(dbUrl) + + if !ok { + tenantDB, err := internal_api_data.NewTenantDB(dbUrl, ctx) + db = tenantDB.DB + + if err != nil { + return nil, err + } + + store.Store(dbUrl, db) + } + + return db.(*sql.DB), nil +} + type DBActivities struct { - connections sync.Map + Connections sync.Map + GetCachedTenantDB func(store *sync.Map, dbUrl string, ctx context.Context) (*sql.DB, error) } func InitDBActivities() *DBActivities { - return &DBActivities{} + return &DBActivities{ + Connections: sync.Map{}, + GetCachedTenantDB: GetCachedTenantDB, + } } diff --git a/internal/onboarding/activities/github.go b/internal/onboarding/activities/github_client.go similarity index 95% rename from internal/onboarding/activities/github.go rename to internal/onboarding/activities/github_client.go index bbfda0cc..89272738 100644 --- a/internal/onboarding/activities/github.go +++ b/internal/onboarding/activities/github_client.go @@ -118,10 +118,10 @@ func InitAppClient() error { } type GithubActivities struct { - githubConfig GithubCfg - newInstallationClient func(installationId int64) (*github.Client, error) + GithubConfig GithubCfg + NewInstallationClient func(installationId int64) (*github.Client, error) } func InitGHActivities(githubCfg GithubCfg) *GithubActivities { - return &GithubActivities{githubConfig: githubCfg, newInstallationClient: NewInstallationClient} + return &GithubActivities{GithubConfig: githubCfg, NewInstallationClient: NewInstallationClient} } diff --git a/internal/onboarding/activities/provision_github_installation_data.go b/internal/onboarding/activities/provision_github_installation_data.go index 3753c271..d1e35245 100644 --- a/internal/onboarding/activities/provision_github_installation_data.go +++ b/internal/onboarding/activities/provision_github_installation_data.go @@ -2,10 +2,8 @@ package activities import ( "context" - "database/sql" "fmt" - internal_api_data "github.com/dxta-dev/app/internal/internal-api/data" "github.com/dxta-dev/app/internal/onboarding/data" "github.com/google/go-github/v72/github" ) @@ -14,34 +12,26 @@ func (activity *GithubActivities) GetGithubInstallation( ctx context.Context, installationId int64, ) (*github.Installation, error) { - return data.GetGithubInstallation(installationId, activity.githubConfig.GithubAppClient, ctx) + return data.GetGithubInstallation(installationId, activity.GithubConfig.GithubAppClient, ctx) } func (activity *DBActivities) SyncGithubInstallationDataToTenant(ctx context.Context, installationId int64, installationOrgName string, installationOrgId int64, - organizationId string, + authId string, dbUrl string) (*data.SyncGithubDataResult, error) { - cacheKey := dbUrl - db, ok := activity.connections.Load(cacheKey) + db, err := activity.GetCachedTenantDB(&activity.Connections, dbUrl, ctx) - if !ok { - tenantDB, err := internal_api_data.NewTenantDB(cacheKey, ctx) - db = tenantDB.DB - - if err != nil { - return nil, err - } - - activity.connections.Store(cacheKey, db) + if err != nil { + return nil, err } res, err := data.SyncGithubInstallationDataToTenant( installationId, installationOrgName, installationOrgId, - organizationId, - db.(*sql.DB), ctx, + authId, + db, ctx, ) if err != nil { @@ -56,7 +46,7 @@ func (activity *GithubActivities) GetInstallationTeams( installationOrgName string, installationId int64, ) ([]*github.Team, error) { - client, err := activity.newInstallationClient(installationId) + client, err := activity.NewInstallationClient(installationId) if err != nil { fmt.Printf("Could not create new installation client. Error: %v", err.Error()) @@ -72,7 +62,7 @@ func (activity *GithubActivities) GetInstallationTeamMembers( installationOrgName string, teamSlug string, ) ([]*github.User, error) { - client, err := activity.newInstallationClient(installationId) + client, err := activity.NewInstallationClient(installationId) if err != nil { fmt.Printf("Could not create new installation client. Error: %v", err.Error()) @@ -83,7 +73,7 @@ func (activity *GithubActivities) GetInstallationTeamMembers( func (activity *GithubActivities) GetInstallationTeamMembersWithEmails(ctx context.Context, installationId int64, members []*github.User) (data.ExtendedMembers, error) { - client, err := activity.newInstallationClient(installationId) + client, err := activity.NewInstallationClient(installationId) if err != nil { fmt.Printf("Could not create new installation client. Error: %v", err.Error()) @@ -100,141 +90,17 @@ func (activity *DBActivities) SyncTeamsAndMembersToTenant( githubOrganizationId int64, organizationId int64, ) (bool, error) { - cacheKey := dbUrl - d, ok := activity.connections.Load(cacheKey) - - if !ok { - tenantDB, err := internal_api_data.NewTenantDB(cacheKey, ctx) - d = tenantDB.DB - - if err != nil { - return false, err - } - - activity.connections.Store(cacheKey, d) - } - db := d.(*sql.DB) - - tx, err := db.BeginTx(ctx, nil) + db, err := activity.GetCachedTenantDB(&activity.Connections, dbUrl, ctx) if err != nil { return false, err } - rows := tx.QueryRowContext(ctx, ` - INSERT INTO teams - (name, organization_id) - VALUES - (?, ?) - RETURNING - id;`, - teamWithMembers.Team.Name, organizationId) - - var teamId int64 - - err = rows.Scan(&teamId) - - if err != nil { - fmt.Println("Issue creating team") - _ = tx.Rollback() - return false, err - } - - rows = tx.QueryRowContext(ctx, ` - INSERT INTO github_teams - (name, external_id, github_organization_id) - VALUES - (?, ?, ?) - RETURNING - id;`, - teamWithMembers.Team.Name, teamWithMembers.Team.ID, githubOrganizationId) - - var githubTeamId int64 - - err = rows.Scan(&githubTeamId) - - if err != nil { - fmt.Println("Issue creating github team") - - _ = tx.Rollback() - return false, err - } - - for _, member := range teamWithMembers.Members { - name := member.Name - - if name == nil { - defaultName := "DXTA member" - name = &defaultName - } - - rowRes := tx.QueryRowContext(ctx, ` - INSERT INTO members - (name, email) - VALUES - (?, ?) - RETURNING id;`, - name, member.Email) - - var memberId int64 - - err = rowRes.Scan(&memberId) - - if err != nil { - fmt.Println("Issue creating member") - - _ = tx.Rollback() - return false, err - } - - _, err = tx.Exec(` - INSERT INTO teams__members - (team_id, member_id) - VALUES - (?, ?);`, - teamId, memberId) - - if err != nil { - fmt.Println("Issue creating teams__members") - _ = tx.Rollback() - return false, err - } - - rowRes = tx.QueryRowContext(ctx, ` - INSERT INTO github_members - (external_id, username, email, member_id) - VALUES - (?, ?, ?, ?) - RETURNING id;`, - member.ID, member.Login, member.Email, memberId) - - var githubMemberId int64 - - err = rowRes.Scan(&githubMemberId) - - if err != nil { - fmt.Println("Issue creating github member") - _ = tx.Rollback() - return false, err - } - - _, err = tx.Exec(` - INSERT INTO github_teams__github_members - (github_team_id, github_member_id) - VALUES - (?, ?);`, - githubTeamId, githubMemberId) - - if err != nil { - fmt.Println("Issue creating github_teams__github_members") - _ = tx.Rollback() - return false, err - } - } - - if err := tx.Commit(); err != nil { - return false, err - } - - return true, nil + return data.SyncTeamsAndMembersToTenant( + ctx, + teamWithMembers, + dbUrl, + githubOrganizationId, + organizationId, + db) } diff --git a/internal/onboarding/data/sync_github_installation_data_to_tenant.go b/internal/onboarding/data/sync_github_installation_data_to_tenant.go index d294d090..bead6e24 100644 --- a/internal/onboarding/data/sync_github_installation_data_to_tenant.go +++ b/internal/onboarding/data/sync_github_installation_data_to_tenant.go @@ -14,7 +14,7 @@ func SyncGithubInstallationDataToTenant( installationId int64, installationOrgName string, installationOrgId int64, - organizationId string, + authId string, db *sql.DB, ctx context.Context, ) (*SyncGithubDataResult, error) { @@ -48,7 +48,7 @@ func SyncGithubInstallationDataToTenant( organizations WHERE auth_id = ?;`, - organizationId) + authId) var orgId int64 err = rows.Scan(&orgId) diff --git a/internal/onboarding/data/sync_teams_and_members_to_tenant.go b/internal/onboarding/data/sync_teams_and_members_to_tenant.go new file mode 100644 index 00000000..fbd4f1fc --- /dev/null +++ b/internal/onboarding/data/sync_teams_and_members_to_tenant.go @@ -0,0 +1,159 @@ +package data + +import ( + "context" + "database/sql" + "fmt" +) + +func SyncTeamsAndMembersToTenant( + ctx context.Context, + teamWithMembers TeamWithMembers, + dbUrl string, + githubOrganizationId int64, + organizationId int64, + db *sql.DB, +) (bool, error) { + tx, err := db.BeginTx(ctx, nil) + + if err != nil { + return false, err + } + + teamsData, err := SyncTeamsToTenant(tx, ctx, teamWithMembers.Team.ID, teamWithMembers.Team.Name, organizationId, githubOrganizationId) + + if err != nil { + return false, err + } + + githubTeamId := teamsData.GithubTeamId + teamId := teamsData.TeamId + + for _, member := range teamWithMembers.Members { + + _, err = tx.Exec(` + INSERT INTO github_members + (external_id, username, email) + VALUES + (?, ?, ?) + ON CONFLICT + (external_id) + DO NOTHING;`, + member.ID, member.Login, member.Email) + + if err != nil { + fmt.Println("Issue while creating github member") + + _ = tx.Rollback() + return false, err + } + + rowRes := tx.QueryRowContext(ctx, ` + SELECT + id, member_id + FROM + github_members + WHERE + external_id = ?;`, + member.ID) + + var githubMemberId int64 + var memberRefId *int64 + + err := rowRes.Scan(&githubMemberId, &memberRefId) + + if err != nil { + fmt.Println("Issue while retrieving github member") + _ = tx.Rollback() + return false, err + } + + _, err = tx.Exec(` + INSERT INTO github_teams__github_members + (github_team_id, github_member_id) + VALUES + (?, ?);`, + githubTeamId, githubMemberId) + + if err != nil { + fmt.Println("Issue creating github_teams__github_members") + _ = tx.Rollback() + return false, err + } + + if memberRefId == nil { + name := member.Name + + if name == nil { + defaultName := "DXTA member" + name = &defaultName + } + + rowRes = tx.QueryRowContext(ctx, ` + INSERT INTO members + (name, email) + VALUES + (?, ?) + RETURNING id;`, + name, member.Email) + + var memberId int64 + + err = rowRes.Scan(&memberId) + + if err != nil { + fmt.Println("Issue creating member") + + _ = tx.Rollback() + return false, err + } + + _, err = tx.Exec(` + UPDATE + github_members + SET + member_id = ? + WHERE id = ?`, + memberId, githubMemberId) + + if err != nil { + fmt.Println("Issue while updating member_id in github member") + + _ = tx.Rollback() + return false, err + } + + _, err = tx.Exec(` + INSERT INTO teams__members + (team_id, member_id) + VALUES + (?, ?);`, + teamId, memberId) + + if err != nil { + fmt.Println("Issue creating teams__members") + _ = tx.Rollback() + return false, err + } + } else { + _, err = tx.Exec(` + INSERT INTO teams__members + (team_id, member_id) + VALUES + (?, ?);`, + teamId, memberRefId) + + if err != nil { + fmt.Println("Issue creating teams__members") + _ = tx.Rollback() + return false, err + } + } + } + + if err := tx.Commit(); err != nil { + return false, err + } + + return true, nil +} diff --git a/internal/onboarding/data/sync_teams_to_tenant.go b/internal/onboarding/data/sync_teams_to_tenant.go new file mode 100644 index 00000000..17292223 --- /dev/null +++ b/internal/onboarding/data/sync_teams_to_tenant.go @@ -0,0 +1,56 @@ +package data + +import ( + "context" + "database/sql" + "fmt" +) + +func SyncTeamsToTenant(tx *sql.Tx, ctx context.Context, teamIdPtr *int64, teamNamePtr *string, organizationId, githubOrganizationId int64) (*struct { + GithubTeamId int64 + TeamId int64 +}, error) { + var githubTeamId int64 + + rows := tx.QueryRowContext(ctx, ` + INSERT INTO github_teams + (name, external_id, github_organization_id) + VALUES + (?, ?, ?) + RETURNING + id;`, + teamNamePtr, teamIdPtr, githubOrganizationId) + + err := rows.Scan(&githubTeamId) + + if err != nil { + fmt.Println("Issue creating github team") + + _ = tx.Rollback() + return nil, err + } + + var teamId int64 + + rows = tx.QueryRowContext(ctx, ` + INSERT INTO teams + (name, organization_id) + VALUES + (?, ?) + RETURNING + id;`, + teamNamePtr, organizationId) + + err = rows.Scan(&teamId) + + if err != nil { + fmt.Println("Issue creating team") + _ = tx.Rollback() + return nil, err + } + + return &struct { + GithubTeamId int64 + TeamId int64 + }{GithubTeamId: githubTeamId, TeamId: teamId}, nil +} diff --git a/internal/onboarding/workflows/provision_github_installation_data.go b/internal/onboarding/workflows/provision_github_installation_data.go index bf574b36..a6de3eb1 100644 --- a/internal/onboarding/workflows/provision_github_installation_data.go +++ b/internal/onboarding/workflows/provision_github_installation_data.go @@ -17,7 +17,7 @@ type GithubDataProvisionResponse struct { Teams []data.TeamWithMembers `json:"teams"` } -func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, authId string, dbUrl string) error { +func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, authId string, dbUrl string) (count int, err error) { ao := workflow.ActivityOptions{ StartToCloseTimeout: time.Minute, } @@ -26,10 +26,10 @@ func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, // 1. Get installation data var installation *github.Installation - err := workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetGithubInstallation, installationId).Get(ctx, &installation) + err = workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetGithubInstallation, installationId).Get(ctx, &installation) if err != nil { - return err + return } // 2. Store installation data to tenant @@ -37,51 +37,62 @@ func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, err = workflow.ExecuteActivity(ctx, (*activities.DBActivities).SyncGithubInstallationDataToTenant, installationId, installation.Account.Login, installation.Account.ID, authId, dbUrl).Get(ctx, &syncResult) if err != nil { - return err + return } - // 3. Add Teams to tenant + // 3. Retrieve installation Github Teams and Github Members and store them in tenant db and create copy of them as DXTA Teams and Members if installation.TargetType != nil && *installation.TargetType == "Organization" { - // 3.1 Retrieve all teams + // 3.1 Retrieve all installation Github Teams var teams []*github.Team - err := workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetInstallationTeams, installation.Account.Login, installationId).Get(ctx, &teams) + err = workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetInstallationTeams, installation.Account.Login, installationId).Get(ctx, &teams) if err != nil { - return err + return } - // 3.2 Retrieve members and store teams and members to tenant db - for _, t := range teams { - team := t - teamWithMembers := data.TeamWithMembers{Team: team, Members: data.ExtendedMembers{}} + // 3.2 Retrieve Github Members and store installation Github Teams and Github Members to tenant db + for _, team := range teams { + workflow.Go(ctx, func(gctx workflow.Context) { + teamWithMembers := data.TeamWithMembers{Team: team, Members: data.ExtendedMembers{}} - var members []*github.User - err := workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetInstallationTeamMembers, installationId, installation.Account.Login, team.Slug).Get(ctx, &members) + var members []*github.User - if err != nil { - return err - } + err = workflow.ExecuteActivity(gctx, (*activities.GithubActivities).GetInstallationTeamMembers, installationId, installation.Account.Login, team.Slug).Get(gctx, &members) - var membersWithEmails *data.ExtendedMembers + if err != nil { + return + } - err = workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetInstallationTeamMembersWithEmails, installationId, members).Get(ctx, &membersWithEmails) + var membersWithEmails *data.ExtendedMembers - if err != nil { - return err - } + err = workflow.ExecuteActivity(gctx, (*activities.GithubActivities).GetInstallationTeamMembersWithEmails, installationId, members).Get(gctx, &membersWithEmails) - teamWithMembers.Members = *membersWithEmails + if err != nil { + return + } - var syncTeamsAndMembersRes *bool - err = workflow.ExecuteActivity(ctx, (*activities.DBActivities).SyncTeamsAndMembersToTenant, teamWithMembers, dbUrl, syncResult.GithubOrganizationId, syncResult.OrganizationId).Get(ctx, &syncTeamsAndMembersRes) + teamWithMembers.Members = *membersWithEmails - if err != nil { - return err - } + var syncTeamsAndMembersRes *bool + err = workflow.ExecuteActivity(gctx, (*activities.DBActivities).SyncTeamsAndMembersToTenant, teamWithMembers, dbUrl, syncResult.GithubOrganizationId, syncResult.OrganizationId).Get(gctx, &syncTeamsAndMembersRes) + + if err != nil { + return + } + + // Count number of finished go routines + // so we can unblock calling thread when + // all go routines finish + count += 1 + }) } + + _ = workflow.Await(ctx, func() bool { + return err != nil || count == len(teams) + }) } - return nil + return } type Args struct { @@ -112,5 +123,5 @@ func ExecuteGithubInstallationDataProvision( return "Unable to execute ", err } - return "success", nil + return "Success", nil } diff --git a/internal/util/auth.go b/internal/util/auth.go index 6eeee77b..f3dc5d2b 100644 --- a/internal/util/auth.go +++ b/internal/util/auth.go @@ -111,14 +111,16 @@ func Authenticator() func(http.Handler) http.Handler { return } - authId := claims["organizationId"].(string) + authId := claims["organizationId"] - if authId == "" { + if authId == nil { fmt.Println("No auth id found in JWT payload") JSONError(w, ErrorParam{Error: "Bad request"}, http.StatusBadRequest) return } + authId = authId.(string) + ctx := r.Context() ctx = context.WithValue(ctx, AuthIdCtxKey, authId) From ad552273d66f69eba1664c0f817c5ff7cdbbe823 Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Wed, 2 Jul 2025 14:40:40 +0200 Subject: [PATCH 6/8] chore: cleanup --- .../workflows/provision_github_installation_data.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/internal/onboarding/workflows/provision_github_installation_data.go b/internal/onboarding/workflows/provision_github_installation_data.go index a6de3eb1..58ed94d8 100644 --- a/internal/onboarding/workflows/provision_github_installation_data.go +++ b/internal/onboarding/workflows/provision_github_installation_data.go @@ -24,7 +24,6 @@ func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, ctx = workflow.WithActivityOptions(ctx, ao) - // 1. Get installation data var installation *github.Installation err = workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetGithubInstallation, installationId).Get(ctx, &installation) @@ -32,7 +31,6 @@ func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, return } - // 2. Store installation data to tenant var syncResult *data.SyncGithubDataResult err = workflow.ExecuteActivity(ctx, (*activities.DBActivities).SyncGithubInstallationDataToTenant, installationId, installation.Account.Login, installation.Account.ID, authId, dbUrl).Get(ctx, &syncResult) @@ -40,9 +38,8 @@ func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, return } - // 3. Retrieve installation Github Teams and Github Members and store them in tenant db and create copy of them as DXTA Teams and Members if installation.TargetType != nil && *installation.TargetType == "Organization" { - // 3.1 Retrieve all installation Github Teams + var teams []*github.Team err = workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetInstallationTeams, installation.Account.Login, installationId).Get(ctx, &teams) @@ -50,7 +47,6 @@ func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, return } - // 3.2 Retrieve Github Members and store installation Github Teams and Github Members to tenant db for _, team := range teams { workflow.Go(ctx, func(gctx workflow.Context) { teamWithMembers := data.TeamWithMembers{Team: team, Members: data.ExtendedMembers{}} From 9d1737514ed2b57fc8179ce81512500462e17c7b Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Wed, 2 Jul 2025 18:45:37 +0200 Subject: [PATCH 7/8] fix: pass params as struct to workflow --- .../provision_github_installation_data.go | 2 +- .../provision_github_installation_data.go | 32 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/internal/internal-api/handler/provision_github_installation_data.go b/internal/internal-api/handler/provision_github_installation_data.go index fcd67857..26aa1c71 100644 --- a/internal/internal-api/handler/provision_github_installation_data.go +++ b/internal/internal-api/handler/provision_github_installation_data.go @@ -39,7 +39,7 @@ func (t *OnboardingTemporal) ProvisionGithubInstallationData(w http.ResponseWrit workflows.ExecuteGithubInstallationDataProvision( ctx, t.temporalClient, - workflows.Args{ + workflows.ExecuteGithubInstallationDataProvisionParams{ TemporalOnboardingQueueName: t.config.TemporalOnboardingNamespace, InstallationId: installationId, AuthId: authId, diff --git a/internal/onboarding/workflows/provision_github_installation_data.go b/internal/onboarding/workflows/provision_github_installation_data.go index 58ed94d8..d52af762 100644 --- a/internal/onboarding/workflows/provision_github_installation_data.go +++ b/internal/onboarding/workflows/provision_github_installation_data.go @@ -17,11 +17,25 @@ type GithubDataProvisionResponse struct { Teams []data.TeamWithMembers `json:"teams"` } -func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, authId string, dbUrl string) (count int, err error) { +type ProvisionGithubInstallationDataParams struct { + InstallationId int64 + AuthId string + DBUrl string +} + +func ProvisionGithubInstallationData(ctx workflow.Context, params ProvisionGithubInstallationDataParams) (count int, err error) { + + installationId := params.InstallationId + authId := params.AuthId + dbUrl := params.DBUrl + + fmt.Println(installationId) + fmt.Println(authId) + fmt.Println(dbUrl) + ao := workflow.ActivityOptions{ StartToCloseTimeout: time.Minute, } - ctx = workflow.WithActivityOptions(ctx, ao) var installation *github.Installation @@ -91,7 +105,7 @@ func ProvisionGithubInstallationData(ctx workflow.Context, installationId int64, return } -type Args struct { +type ExecuteGithubInstallationDataProvisionParams struct { TemporalOnboardingQueueName string InstallationId int64 AuthId string @@ -101,18 +115,20 @@ type Args struct { func ExecuteGithubInstallationDataProvision( ctx context.Context, temporalClient client.Client, - args Args, + params ExecuteGithubInstallationDataProvisionParams, ) (string, error) { _, err := temporalClient.ExecuteWorkflow( ctx, client.StartWorkflowOptions{ ID: fmt.Sprintf("onboarding-workflow-%v", time.Now().Format("20060102150405")), - TaskQueue: args.TemporalOnboardingQueueName, + TaskQueue: params.TemporalOnboardingQueueName, }, ProvisionGithubInstallationData, - args.InstallationId, - args.AuthId, - args.DBUrl, + ProvisionGithubInstallationDataParams{ + InstallationId: params.InstallationId, + AuthId: params.AuthId, + DBUrl: params.DBUrl, + }, ) if err != nil { From 49b8afc012a53115036d250a86cc3bd77b66d975 Mon Sep 17 00:00:00 2001 From: stefanskoricdev Date: Wed, 2 Jul 2025 19:28:40 +0200 Subject: [PATCH 8/8] fix: Set proper start to close timeout and max attempts activity options --- .../provision_github_installation_data.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/onboarding/workflows/provision_github_installation_data.go b/internal/onboarding/workflows/provision_github_installation_data.go index d52af762..f5c0c222 100644 --- a/internal/onboarding/workflows/provision_github_installation_data.go +++ b/internal/onboarding/workflows/provision_github_installation_data.go @@ -9,6 +9,7 @@ import ( "github.com/dxta-dev/app/internal/onboarding/data" "github.com/google/go-github/v72/github" "go.temporal.io/sdk/client" + "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" ) @@ -24,20 +25,19 @@ type ProvisionGithubInstallationDataParams struct { } func ProvisionGithubInstallationData(ctx workflow.Context, params ProvisionGithubInstallationDataParams) (count int, err error) { + ao := workflow.ActivityOptions{ + StartToCloseTimeout: time.Second * 30, + RetryPolicy: &temporal.RetryPolicy{ + MaximumAttempts: 10, + }, + } + + ctx = workflow.WithActivityOptions(ctx, ao) installationId := params.InstallationId authId := params.AuthId dbUrl := params.DBUrl - fmt.Println(installationId) - fmt.Println(authId) - fmt.Println(dbUrl) - - ao := workflow.ActivityOptions{ - StartToCloseTimeout: time.Minute, - } - ctx = workflow.WithActivityOptions(ctx, ao) - var installation *github.Installation err = workflow.ExecuteActivity(ctx, (*activities.GithubActivities).GetGithubInstallation, installationId).Get(ctx, &installation)