FEAT: Add github data provision workflow and installation data activity

cmd/onboarding-worker/main.go

@@ -5,8 +5,8 @@ import (
 	"log"
 
 	"github.com/dxta-dev/app/internal/onboarding"
-	"github.com/dxta-dev/app/internal/onboarding/activities"
-	"github.com/dxta-dev/app/internal/onboarding/workflows"
+	"github.com/dxta-dev/app/internal/onboarding/activity"
+	"github.com/dxta-dev/app/internal/onboarding/workflow"
 	"go.temporal.io/sdk/client"
 	"go.temporal.io/sdk/worker"
 )
@@ -17,6 +17,12 @@ func main() {
 		log.Fatalln("Failed to load configuration:", err)
 	}
 
+	githubConfig, err := onboarding.LoadGithubConfig()
+
+	if err != nil {
+		log.Fatalln("Failed to load github configuration:", err)
+	}
+
 	temporalClient, err := client.Dial(client.Options{
 		HostPort:  cfg.TemporalHostPort,
 		Namespace: cfg.TemporalOnboardingNamespace,
@@ -26,6 +32,12 @@ func main() {
 	}
 	defer temporalClient.Close()
 
+	githubAppClient, err := onboarding.NewAppClient(*githubConfig)
+
+	if err != nil {
+		log.Fatalf("Unable to init app client: %v", err)
+	}
+
 	err = onboarding.RegisterNamespace(
 		context.Background(),
 		cfg.TemporalHostPort,
@@ -38,12 +50,15 @@ func main() {
 
 	w := worker.New(temporalClient, cfg.TemporalOnboardingQueueName, worker.Options{})
 
-	userActivities := activities.NewUserActivites(
+	userActivities := activity.NewUserActivites(
 		*cfg,
 	)
+	githubInstallationActivities := activity.NewGithubInstallationActivities(*githubAppClient)
 
-	w.RegisterWorkflow(workflows.CountUsers)
+	w.RegisterWorkflow(workflow.CountUsers)
+	w.RegisterWorkflow(workflow.AfterGithubInstallationWorkflow)
 	w.RegisterActivity(userActivities)
+	w.RegisterActivity(githubInstallationActivities)
 
 	if err := w.Run(worker.InterruptCh()); err != nil {
 		log.Fatalln("Worker failed to start", err)

go.mod

@@ -20,6 +20,7 @@ require (
 
 require (
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect
+	github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
@@ -28,8 +29,13 @@ require (
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/gofri/go-github-ratelimit/v2 v2.0.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 	github.com/golang/mock v1.6.0 // indirect
+	github.com/google/go-github/v72 v72.0.0 // indirect
+	github.com/google/go-github/v73 v73.0.0 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect

go.sum

@@ -5,6 +5,8 @@ github.com/XSAM/otelsql v0.39.0/go.mod h1:uMOXLUX+wkuAuP0AR3B45NXX7E9lJS2mERa8gq
 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk=
 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 h1:B91r9bHtXp/+XRgS5aZm6ZzTdz3ahgJYmkt4xZkgDz8=
+github.com/bradleyfalzon/ghinstallation/v2 v2.16.0/go.mod h1:OeVe5ggFzoBnmgitZe/A+BqGOnv1DvU/0uiLQi1wutM=
 github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
 github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -54,8 +56,12 @@ github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/gofri/go-github-ratelimit/v2 v2.0.2 h1:gS8wAS1jTmlWGdTjAM7KIpsLjwY1S0S/gKK5hthfSXM=
+github.com/gofri/go-github-ratelimit/v2 v2.0.2/go.mod h1:YBQt4gTbdcbMjJFT05YFEaECwH78P5b0IwrnbLiHGdE=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
+github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
@@ -68,9 +74,16 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/go-github/v72 v72.0.0 h1:FcIO37BLoVPBO9igQQ6tStsv2asG4IPcYFi655PPvBM=
+github.com/google/go-github/v72 v72.0.0/go.mod h1:WWtw8GMRiL62mvIquf1kO3onRHeWWKmK01qdCY8c5fg=
+github.com/google/go-github/v73 v73.0.0 h1:aR+Utnh+Y4mMkS+2qLQwcQ/cF9mOTpdwnzlaw//rG24=
+github.com/google/go-github/v73 v73.0.0/go.mod h1:fa6w8+/V+edSU0muqdhCVY7Beh1M8F1IlQPZIANKIYw=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

internal/internal-api/handler/users_count.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 
 	"github.com/dxta-dev/app/internal/onboarding"
-	"github.com/dxta-dev/app/internal/onboarding/workflows"
+	"github.com/dxta-dev/app/internal/onboarding/workflow"
 	"github.com/dxta-dev/app/internal/util"
 	"go.temporal.io/sdk/client"
 )
@@ -30,7 +30,7 @@ func NewUsers(temporalClient client.Client, config onboarding.Config) *Users {
 }
 
 func (u *Users) UsersCount(w http.ResponseWriter, r *http.Request) {
-	out, err := workflows.ExecuteCountUsersWorkflow(r.Context(), u.temporalClient, u.config)
+	out, err := workflow.ExecuteCountUsersWorkflow(r.Context(), u.temporalClient, u.config)
 	if err != nil {
 		log.Fatal(errors.Unwrap(err))
 	}

internal/onboarding/activity/github_installation.go

@@ -0,0 +1,32 @@
+package activity
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/dxta-dev/app/internal/onboarding"
+)
+
+type GithubInstallationActivities struct {
+	githubAppClient onboarding.GithubAppClient
+}
+
+func NewGithubInstallationActivities(GithubAppClient onboarding.GithubAppClient) *GithubInstallationActivities {
+	return &GithubInstallationActivities{
+		githubAppClient: GithubAppClient,
+	}
+}
+
+func (gia *GithubInstallationActivities) GetGithubInstallation(
+	ctx context.Context,
+	installationId int64,
+) (string, error) {
+	login, err := gia.githubAppClient.GetOrganizationLogin(ctx, installationId)
+
+	if err != nil {
+		fmt.Printf("Could not retrieve installation. Error: %v", err.Error())
+		return "", err
+	}
+
+	return login, nil
+}

internal/onboarding/activities/users.go → internal/onboarding/activity/users.go

@@ -1,4 +1,4 @@
-package activities
+package activity
 
 import (
 	"context"

internal/onboarding/github_config.go

@@ -0,0 +1,169 @@
+package onboarding
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"strconv"
+
+	"github.com/bradleyfalzon/ghinstallation/v2"
+	"github.com/gofri/go-github-ratelimit/v2/github_ratelimit"
+	"github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit"
+	"github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit"
+	"github.com/google/go-github/v73/github"
+)
+
+type GithubConfig struct {
+	GithubAppId         int64
+	GithubAppPrivateKey []byte
+	RoundTripper        http.RoundTripper
+}
+
+func LoadGithubConfig() (*GithubConfig, error) {
+	appIdStr := os.Getenv("GITHUB_APP_ID")
+	appPrivateKeyStr := os.Getenv("GITHUB_APP_PRIVATE_KEY")
+
+	if appIdStr == "" {
+		return nil, errors.New("GITHUB_APP_ID not set")
+	}
+
+	if appPrivateKeyStr == "" {
+		return nil, errors.New("GITHUB_APP_PRIVATE_KEY not set")
+	}
+
+	appId, err := strconv.ParseInt(appIdStr, 10, 64)
+
+	if err != nil {
+		return nil, errors.New("could not parse app id string to int64")
+	}
+
+	appPrivateKey, err := base64.StdEncoding.DecodeString(appPrivateKeyStr)
+
+	if err != nil {
+		return nil, errors.New("failed to decode base64 string")
+	}
+
+	return &GithubConfig{
+		GithubAppId:         appId,
+		GithubAppPrivateKey: appPrivateKey,
+		RoundTripper:        http.DefaultTransport,
+	}, nil
+}
+
+func getInstallationTransport(
+	tr http.RoundTripper,
+	installationId int64,
+	appId int64,
+	appPrivateKey []byte,
+) (http.RoundTripper, error) {
+	itt, err := ghinstallation.New(tr, appId, installationId, appPrivateKey)
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to create apps transport: %w", err)
+	}
+
+	return itt, nil
+}
+
+func getAppTransport(
+	tr http.RoundTripper,
+	appId int64,
+	appPrivateKey []byte,
+) (http.RoundTripper, error) {
+	atr, err := ghinstallation.NewAppsTransport(tr, appId, appPrivateKey)
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to create apps transport: %w", err)
+	}
+
+	return atr, nil
+}
+
+func createLimiter(tr http.RoundTripper) http.RoundTripper {
+	return github_ratelimit.New(
+		tr,
+		github_primary_ratelimit.WithLimitDetectedCallback(
+			func(ctx *github_primary_ratelimit.CallbackContext) {
+				fmt.Printf(
+					"Primary rate limit detected: category %s, reset time: %v\n",
+					ctx.Category,
+					ctx.ResetTime,
+				)
+			},
+		),
+		github_secondary_ratelimit.WithLimitDetectedCallback(
+			func(ctx *github_secondary_ratelimit.CallbackContext) {
+				fmt.Printf(
+					"Secondary rate limit detected: reset time: %v, total sleep time: %v\n",
+					ctx.ResetTime,
+					ctx.TotalSleepTime,
+				)
+			},
+		),
+	)
+}
+
+func NewInstallationClient(
+	installationId int64,
+	tr http.RoundTripper,
+	cfg GithubConfig,
+) (*github.Client, error) {
+	tr, err := getInstallationTransport(
+		tr,
+		installationId,
+		cfg.GithubAppId,
+		cfg.GithubAppPrivateKey,
+	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	tr = createLimiter(tr)
+
+	return github.NewClient(&http.Client{Transport: tr}), nil
+}
+
+type GithubAppClient struct {
+	client *github.Client
+}
+
+func (gac *GithubAppClient) GetOrganizationLogin(
+	ctx context.Context,
+	installationID int64,
+) (string, error) {
+	installation, _, error := gac.client.Apps.GetInstallation(ctx, installationID)
+	if error != nil {
+		return "", errors.New("failed to get installation: " + error.Error())
+
+	}
+
+	if installation.Account == nil || installation.Account.Login == nil {
+		return "", errors.New("installation account or login is nil")
+	}
+
+	if installation.TargetType == nil || *installation.TargetType != "organization" {
+		return "", errors.New("installation is not for an organization")
+	}
+
+	return *installation.Account.Login, nil
+}
+
+func NewAppClient(cfg GithubConfig) (*GithubAppClient, error) {
+	tr, err := getAppTransport(cfg.RoundTripper, cfg.GithubAppId, cfg.GithubAppPrivateKey)
+
+	if err != nil {
+		return nil, err
+	}
+
+	tr = createLimiter(tr)
+
+	client := github.NewClient(&http.Client{Transport: tr})
+
+	return &GithubAppClient{
+		client: client,
+	}, nil
+}