From 6353c8c667071e4ed6f232d9fe324b024ce18860 Mon Sep 17 00:00:00 2001 From: whywaita Date: Mon, 23 Aug 2021 11:38:24 +0900 Subject: [PATCH 1/2] check to register in GitHub before deleting job --- pkg/gh/github.go | 21 ++++++++ pkg/starter/starter.go | 107 +++++++++++++++++++++++++++++++++-------- pkg/web/http.go | 1 + 3 files changed, 110 insertions(+), 19 deletions(-) diff --git a/pkg/gh/github.go b/pkg/gh/github.go index bbd147d..9e22124 100644 --- a/pkg/gh/github.go +++ b/pkg/gh/github.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "path" + "strings" "github.com/bradleyfalzon/ghinstallation" "github.com/google/go-github/v35/github" @@ -15,6 +16,10 @@ import ( "golang.org/x/oauth2" ) +var ( + ErrNotFound = fmt.Errorf("not found") +) + // NewClient create a client of GitHub func NewClient(ctx context.Context, personalToken, gheDomain string) (*github.Client, error) { ts := oauth2.StaticTokenSource( @@ -72,6 +77,22 @@ func ExistGitHubRepository(scope, gheDomain string, githubPersonalToken string) return fmt.Errorf("invalid response code (%d)", resp.StatusCode) } +// ExistGitHubRunner check exist registered of github runner +func ExistGitHubRunner(ctx context.Context, client *github.Client, owner, repo, runnerName string) (*github.Runner, error) { + runners, err := ListRunners(ctx, client, owner, repo) + if err != nil { + return nil, fmt.Errorf("failed to get list of runners: %w", err) + } + + for _, r := range runners { + if strings.EqualFold(r.GetName(), runnerName) { + return r, nil + } + } + + return nil, ErrNotFound +} + // ListRunners get runners that registered repository or org func ListRunners(ctx context.Context, client *github.Client, owner, repo string) ([]*github.Runner, error) { var rs []*github.Runner diff --git a/pkg/starter/starter.go b/pkg/starter/starter.go index 5b16b43..7a62489 100644 --- a/pkg/starter/starter.go +++ b/pkg/starter/starter.go @@ -2,15 +2,15 @@ package starter import ( "context" + "errors" "fmt" "sync" "time" - "github.com/whywaita/myshoes/pkg/runner" - - "github.com/whywaita/myshoes/pkg/logger" - "github.com/whywaita/myshoes/pkg/datastore" + "github.com/whywaita/myshoes/pkg/gh" + "github.com/whywaita/myshoes/pkg/logger" + "github.com/whywaita/myshoes/pkg/runner" "github.com/whywaita/myshoes/pkg/shoes" "github.com/whywaita/myshoes/pkg/starter/safety" ) @@ -87,8 +87,43 @@ func (s *Starter) do(ctx context.Context) error { return } - if err := s.bung(ctx, job); err != nil { - logger.Logf(false, "failed to bung: %+v\n", err) + cloudID, ipAddress, shoesType, err := s.bung(ctx, job) + if err != nil { + logger.Logf(false, "failed to bung (target ID: %s, job ID: %s): %+v\n", job.TargetID, job.UUID, err) + + if err := datastore.UpdateTargetStatus(ctx, s.ds, job.TargetID, datastore.TargetStatusErr, fmt.Sprintf("failed to create an instance (job ID: %s)", job.UUID)); err != nil { + logger.Logf(false, "failed to update target status (target ID: %s, job ID: %s): %+v\n", job.TargetID, job.UUID, err) + return + } + + return + } + + if err := s.checkRegisteredRunner(ctx, job, cloudID); err != nil { + logger.Logf(false, "failed to check to register runner (target ID: %s, job ID: %s): %+v\n", job.TargetID, job.UUID, err) + + if err := deleteInstance(ctx, cloudID); err != nil { + logger.Logf(false, "failed to delete an instance that not registered instance (target ID: %s, cloud ID: %s): %+v\n", job.TargetID, cloudID, err) + // not return, need to update target status if err. + } + + if err := datastore.UpdateTargetStatus(ctx, s.ds, job.TargetID, datastore.TargetStatusErr, fmt.Sprintf("cannot register runner to GitHub (job ID: %s)", job.UUID)); err != nil { + logger.Logf(false, "failed to update target status (target ID: %s, job ID: %s): %+v\n", job.TargetID, job.UUID, err) + return + } + + return + } + + r := datastore.Runner{ + UUID: job.UUID, + ShoesType: shoesType, + IPAddress: ipAddress, + TargetID: job.TargetID, + CloudID: cloudID, + } + if err := s.ds.CreateRunner(ctx, r); err != nil { + logger.Logf(false, "failed to save runner to datastore (target ID: %s, job ID: %s): %+v\n", job.TargetID, job.UUID, err) if err := datastore.UpdateTargetStatus(ctx, s.ds, job.TargetID, datastore.TargetStatusErr, fmt.Sprintf("job id: %s", job.UUID)); err != nil { logger.Logf(false, "failed to update target status (target ID: %s, job ID: %s): %+v\n", job.TargetID, job.UUID, err) @@ -97,6 +132,7 @@ func (s *Starter) do(ctx context.Context) error { return } + if err := s.ds.DeleteJob(ctx, job.UUID); err != nil { logger.Logf(false, "failed to delete job: %+v\n", err) @@ -116,42 +152,75 @@ func (s *Starter) do(ctx context.Context) error { } // bung is start runner, like a pistol! :) -func (s *Starter) bung(ctx context.Context, job datastore.Job) error { +func (s *Starter) bung(ctx context.Context, job datastore.Job) (string, string, string, error) { logger.Logf(false, "start create instance (job: %s)", job.UUID) client, teardown, err := shoes.GetClient() if err != nil { - return fmt.Errorf("failed to get plugin client: %w", err) + return "", "", "", fmt.Errorf("failed to get plugin client: %w", err) } defer teardown() target, err := s.ds.GetTarget(ctx, job.TargetID) if err != nil { - return fmt.Errorf("failed to retrieve relational target (job: %s, target: %s): %w", job.UUID, job.TargetID, err) + return "", "", "", fmt.Errorf("failed to retrieve relational target: %w", err) } script, err := s.getSetupScript(*target) if err != nil { - return fmt.Errorf("failed to get setup scripts: %w", err) + return "", "", "", fmt.Errorf("failed to get setup scripts: %w", err) } runnerName := runner.ToName(job.UUID.String()) cloudID, ipAddress, shoesType, err := client.AddInstance(ctx, runnerName, script, target.ResourceType) if err != nil { - return fmt.Errorf("failed to add instance: %w", err) + return "", "", "", fmt.Errorf("failed to add instance: %w", err) } logger.Logf(false, "instance create successfully! (job: %s, cloud ID: %s)", job.UUID, cloudID) - r := datastore.Runner{ - UUID: job.UUID, - ShoesType: shoesType, - IPAddress: ipAddress, - TargetID: target.UUID, - CloudID: cloudID, + return cloudID, ipAddress, shoesType, nil +} + +func deleteInstance(ctx context.Context, cloudID string) error { + client, teardown, err := shoes.GetClient() + if err != nil { + return fmt.Errorf("failed to get plugin client: %w", err) } - if err := s.ds.CreateRunner(ctx, r); err != nil { - return fmt.Errorf("failed to create runner: %w", err) + defer teardown() + + if err := client.DeleteInstance(ctx, cloudID); err != nil { + return fmt.Errorf("failed to delete instance: %w", err) } return nil } + +// checkRegisteredRunner check to register runner to GitHub +func (s *Starter) checkRegisteredRunner(ctx context.Context, job datastore.Job, cloudID string) error { + target, err := s.ds.GetTarget(ctx, job.TargetID) + if err != nil { + return fmt.Errorf("failed to retrieve relational target: %w", err) + } + + client, err := gh.NewClient(ctx, target.GitHubToken, target.GHEDomain.String) + if err != nil { + return fmt.Errorf("failed to create github client: %w", err) + } + + owner, repo := gh.DivideScope(target.Scope) + + timeoutSeconds := 60 + for i := 0; i > timeoutSeconds; i++ { // 60 second is timeout + if _, err := gh.ExistGitHubRunner(ctx, client, owner, repo, cloudID); err == nil { + // success to register runner to GitHub + return nil + } else if !errors.Is(err, gh.ErrNotFound) { + // not retryable error + return fmt.Errorf("failed to check existing runner in GitHub: %w", err) + } + + time.Sleep(1 * time.Second) + } + + return fmt.Errorf("faied to to check existing runner in GitHub: timeout in %ds", timeoutSeconds) +} diff --git a/pkg/web/http.go b/pkg/web/http.go index b49134f..726cebb 100644 --- a/pkg/web/http.go +++ b/pkg/web/http.go @@ -63,6 +63,7 @@ func NewMux(ds datastore.Datastore) *goji.Mux { handleTargetDelete(w, r, ds) }) + // Config endpoints mux.HandleFunc(pat.Post("/config/debug"), func(w http.ResponseWriter, r *http.Request) { apacheLogging(r) handleConfigDebug(w, r) From 4700df30a500a10a05d0979270a16e2b3a7c6067 Mon Sep 17 00:00:00 2001 From: whywaita Date: Mon, 23 Aug 2021 11:40:45 +0900 Subject: [PATCH 2/2] add comments for golint --- pkg/gh/github.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/gh/github.go b/pkg/gh/github.go index 9e22124..1d458bf 100644 --- a/pkg/gh/github.go +++ b/pkg/gh/github.go @@ -17,6 +17,7 @@ import ( ) var ( + // ErrNotFound is error for not found ErrNotFound = fmt.Errorf("not found") )