From 76f2dd03fcf407ed23905ad411660b5641437928 Mon Sep 17 00:00:00 2001 From: Wilfried OLLIVIER Date: Thu, 10 Feb 2022 07:38:01 +0100 Subject: [PATCH 1/8] Clean up duplicate code --- application/task.go | 3 +- volumes/volumes.go | 75 ----------------------------------------- volumes/volumes_test.go | 37 -------------------- 3 files changed, 2 insertions(+), 113 deletions(-) diff --git a/application/task.go b/application/task.go index 1442399..d4c0810 100644 --- a/application/task.go +++ b/application/task.go @@ -119,11 +119,12 @@ func (a *Application) TaskHandler(w http.ResponseWriter, r *http.Request) { zap.String("branch", chi.URLParam(r, "branch")), zap.String("commit", chi.URLParam(r, "commit")), ) - t, err := a.volumes.Get( + t, err := a.storage.GetByCommit( chi.URLParam(r, "serviceID"), chi.URLParam(r, "project"), chi.URLParam(r, "branch"), chi.URLParam(r, "commit"), + false, ) if err != nil { l.Warn("Task get error", zap.Error(err)) diff --git a/volumes/volumes.go b/volumes/volumes.go index 9e8c3a1..4e44916 100644 --- a/volumes/volumes.go +++ b/volumes/volumes.go @@ -1,12 +1,9 @@ package volumes import ( - "encoding/json" "io/fs" - "io/ioutil" "os" "path/filepath" - "sort" "github.com/factorysh/microdensity/task" "go.uber.org/zap" @@ -60,79 +57,7 @@ func (v *Volumes) Create(t *task.Task) error { return err } -func (v *Volumes) Get(service, project, branch, commit string) (*task.Task, error) { - p := v.Path(service, project, branch) - l := v.logger.With( - zap.String("root", v.root), - zap.String("service", service), - zap.String("project", project), - zap.String("branch", branch), - zap.String("commit", commit), - zap.String("path", p), - ) - _, err := os.Stat(p) - if err != nil { - if os.IsNotExist(err) { - l.Warn("Volume not found") - } else { - l.Error("Stat error", zap.Error(err)) - } - return nil, err - } - - commits, err := ioutil.ReadDir(p) - if err != nil { - l.Error("Readall error", zap.Error(err)) - return nil, err - } - for _, com := range commits { - l = l.With(zap.String("commit", com.Name())) - f, err := os.OpenFile(v.Path(service, project, branch, com.Name(), "task.json"), os.O_RDONLY, 0) - if err != nil { - l.Error("open commit file", zap.Error(err)) - return nil, err - } - var t task.Task - err = json.NewDecoder(f).Decode(&t) - if err != nil { - l.Error("JSON decode", zap.Error(err)) - return nil, err - } - if t.Commit == commit { - return &t, nil - } - } - return nil, nil -} - // Path will return the full path for this volume func (v *Volumes) Path(elem ...string) string { return filepath.Join(v.root, filepath.Join(elem...)) } - -// ByProjectByBranch returns all subvolumes for a specific branch of a project -func (v *Volumes) ByProjectByBranch(project string, branch string) ([]string, error) { - vols, err := ioutil.ReadDir(v.Path(project, branch)) - if err != nil || !containsFiles(vols) { - v.logger.Error("by project by branch", - zap.String("project", project), - zap.String("brnach", branch), - zap.Error(err), - ) - return nil, err - } - - var res []string - for _, run := range vols { - if run.IsDir() { - res = append(res, v.Path(project, branch, run.Name())) - } - } - sort.Strings(res) - - return res, nil -} - -func containsFiles(files []fs.FileInfo) bool { - return len(files) > 0 -} diff --git a/volumes/volumes_test.go b/volumes/volumes_test.go index f6af081..0c0ede4 100644 --- a/volumes/volumes_test.go +++ b/volumes/volumes_test.go @@ -2,12 +2,8 @@ package volumes import ( "os" - "strings" "testing" - "time" - "github.com/factorysh/microdensity/task" - "github.com/google/uuid" "github.com/stretchr/testify/assert" ) @@ -42,36 +38,3 @@ func TestNewVolumes(t *testing.T) { }) } } - -func TestListByProjectByBranch(t *testing.T) { - defer cleanDir() - - v, err := New(testRootDir) - assert.NoError(t, err) - var first uuid.UUID - var last uuid.UUID - for i := 0; i < 10; i++ { - id, err := uuid.NewUUID() - assert.NoError(t, err) - c := time.Now() - if first == uuid.Nil { - first = id - } - last = id - err = v.Create(&task.Task{ - Project: "group%2Fproject", - Branch: "master", - Id: id, - Creation: c, - Commit: "70ea687225ea4869311c500a954e8cc5e687e608", - }) - assert.NoError(t, err) - } - - dirs, err := v.ByProjectByBranch("group%2Fproject", "master") - assert.NoError(t, err) - assert.Len(t, dirs, 10, "one folder should be found") - assert.True(t, strings.HasSuffix(dirs[0], first.String()), dirs[0]) - assert.True(t, strings.HasSuffix(dirs[9], last.String())) - cleanDir() -} From 4f58a91e98d027a2762b9118473b76239e9983db Mon Sep 17 00:00:00 2001 From: Wilfried OLLIVIER Date: Thu, 10 Feb 2022 07:47:35 +0100 Subject: [PATCH 2/8] Add dedicated handler to get a task by ID --- application/application.go | 2 +- application/task.go | 54 +++++++++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/application/application.go b/application/application.go index 0a364ea..055647f 100644 --- a/application/application.go +++ b/application/application.go @@ -126,7 +126,7 @@ func New(cfg *conf.Conf) (*Application, error) { r.Use(authMiddleware.Middleware()) r.Use(a.ServiceMiddleware) r.Get("/", a.ReadmeHandler) - r.Get("/-{taskID}", a.TaskHandler) + r.Get("/-{taskID}", a.TaskIDHandler) }) r.Route("/service/{serviceID}/{project}", func(r chi.Router) { r.Use(authMiddleware.Middleware()) diff --git a/application/task.go b/application/task.go index d4c0810..ca9edaf 100644 --- a/application/task.go +++ b/application/task.go @@ -111,21 +111,48 @@ func (a *Application) PostTaskHandler(w http.ResponseWriter, r *http.Request) { } // TaskHandler show a Task -func (a *Application) TaskHandler(w http.ResponseWriter, r *http.Request) { +func (a *Application) TaskHandler(latest bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + l := a.logger.With( + zap.String("url", r.URL.String()), + zap.String("service", chi.URLParam(r, "serviceID")), + zap.String("project", chi.URLParam(r, "project")), + zap.String("branch", chi.URLParam(r, "branch")), + zap.String("commit", chi.URLParam(r, "commit")), + ) + t, err := a.storage.GetByCommit( + chi.URLParam(r, "serviceID"), + chi.URLParam(r, "project"), + chi.URLParam(r, "branch"), + chi.URLParam(r, "commit"), + latest, + ) + if err != nil { + l.Warn("Task get error", zap.Error(err)) + if os.IsNotExist(err) { + w.WriteHeader(http.StatusNotFound) + } else { + w.WriteHeader(http.StatusBadRequest) + } + return + } + err = json.NewEncoder(w).Encode(t) + if err != nil { + l.Error("Json encoding error", zap.Error(err)) + panic(err) + } + } +} + +// TaskIDHandler get a task using it's ID +func (a *Application) TaskIDHandler(w http.ResponseWriter, r *http.Request) { l := a.logger.With( zap.String("url", r.URL.String()), - zap.String("service", chi.URLParam(r, "serviceID")), - zap.String("project", chi.URLParam(r, "project")), - zap.String("branch", chi.URLParam(r, "branch")), - zap.String("commit", chi.URLParam(r, "commit")), - ) - t, err := a.storage.GetByCommit( - chi.URLParam(r, "serviceID"), - chi.URLParam(r, "project"), - chi.URLParam(r, "branch"), - chi.URLParam(r, "commit"), - false, + zap.String("task ID", chi.URLParam(r, "taskID")), ) + + t, err := a.storage.Get(chi.URLParam(r, "taskID")) if err != nil { l.Warn("Task get error", zap.Error(err)) if os.IsNotExist(err) { @@ -135,9 +162,10 @@ func (a *Application) TaskHandler(w http.ResponseWriter, r *http.Request) { } return } + err = json.NewEncoder(w).Encode(t) if err != nil { l.Error("Json encoding error", zap.Error(err)) - panic(err) + return } } From 3ad215594f833e920eb4263f7a81a60f89942470 Mon Sep 17 00:00:00 2001 From: Wilfried OLLIVIER Date: Thu, 10 Feb 2022 07:49:21 +0100 Subject: [PATCH 3/8] More routes for latest path --- application/application.go | 11 +++++++---- application/volume.go | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/application/application.go b/application/application.go index 055647f..ab94a7c 100644 --- a/application/application.go +++ b/application/application.go @@ -135,13 +135,16 @@ func New(cfg *conf.Conf) (*Application, error) { r.Route("/{branch}", func(r chi.Router) { r.Route("/{commit}", func(r chi.Router) { r.Post("/", a.PostTaskHandler) - r.Get("/", a.TaskHandler) + r.Get("/", a.TaskHandler(false)) r.Get("/status", badge.StatusBadge(a.storage, false)) r.Get("/badge/{badge}", a.TaskMyBadgeHandler) - r.Get("/volumes/*", a.VolumesHandler(6)) + r.Get("/volumes/*", a.VolumesHandler(6, false)) + }) + r.Route("/latest", func(r chi.Router) { + r.Get("/", a.TaskHandler(true)) + r.Get("/status", badge.StatusBadge(a.storage, true)) + r.Get("/volumes/*", a.VolumesHandler(6, true)) }) - r.Get("/latest", nil) - r.Get("/latest/status", badge.StatusBadge(a.storage, true)) }) }) }) diff --git a/application/volume.go b/application/volume.go index bf2baf8..c98beaf 100644 --- a/application/volume.go +++ b/application/volume.go @@ -13,7 +13,7 @@ import ( ) // VolumesHandler expose volumes of a task -func (a *Application) VolumesHandler(basePathLen int) http.HandlerFunc { +func (a *Application) VolumesHandler(basePathLen int, latest bool) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { l := a.logger.With( zap.String("url", r.URL.String()), @@ -28,7 +28,7 @@ func (a *Application) VolumesHandler(basePathLen int) http.HandlerFunc { branch := chi.URLParam(r, "branch") commit := chi.URLParam(r, "commit") - t, err := a.storage.GetByCommit(service, project, branch, commit, false) + t, err := a.storage.GetByCommit(service, project, branch, commit, latest) if err != nil { l.Error("Get task", zap.Error(err)) w.WriteHeader(http.StatusNotFound) From c5beb5378b509c92bb7f5153295e122134c5d13a Mon Sep 17 00:00:00 2001 From: Wilfried OLLIVIER Date: Thu, 10 Feb 2022 07:54:12 +0100 Subject: [PATCH 4/8] Add badge route for specific commit and latest --- application/application.go | 3 +- application/badge.go | 91 ++++++++++++++++++++------------------ 2 files changed, 49 insertions(+), 45 deletions(-) diff --git a/application/application.go b/application/application.go index ab94a7c..e198e85 100644 --- a/application/application.go +++ b/application/application.go @@ -137,12 +137,13 @@ func New(cfg *conf.Conf) (*Application, error) { r.Post("/", a.PostTaskHandler) r.Get("/", a.TaskHandler(false)) r.Get("/status", badge.StatusBadge(a.storage, false)) - r.Get("/badge/{badge}", a.TaskMyBadgeHandler) + r.Get("/badge/{badge}", a.BadgeMyTaskHandler(false)) r.Get("/volumes/*", a.VolumesHandler(6, false)) }) r.Route("/latest", func(r chi.Router) { r.Get("/", a.TaskHandler(true)) r.Get("/status", badge.StatusBadge(a.storage, true)) + r.Get("/badge/{badge}", a.BadgeMyTaskHandler(true)) r.Get("/volumes/*", a.VolumesHandler(6, true)) }) }) diff --git a/application/badge.go b/application/badge.go index b21eb4a..721b939 100644 --- a/application/badge.go +++ b/application/badge.go @@ -13,53 +13,56 @@ import ( "go.uber.org/zap" ) -func (a *Application) TaskMyBadgeHandler(w http.ResponseWriter, r *http.Request) { - l := a.logger.With( - zap.String("url", r.URL.String()), - zap.String("service", chi.URLParam(r, "serviceID")), - zap.String("project", chi.URLParam(r, "project")), - zap.String("branch", chi.URLParam(r, "branch")), - zap.String("commit", chi.URLParam(r, "commit")), - zap.String("branch", chi.URLParam(r, "branch")), - ) - service := chi.URLParam(r, "serviceID") - project := chi.URLParam(r, "project") - branch := chi.URLParam(r, "branch") - commit := chi.URLParam(r, "commit") - bdg := chi.URLParam(r, "badge") +// BadgeMyTaskHandler generates a badge for a given task +func (a *Application) BadgeMyTaskHandler(latest bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := a.logger.With( + zap.String("url", r.URL.String()), + zap.String("service", chi.URLParam(r, "serviceID")), + zap.String("project", chi.URLParam(r, "project")), + zap.String("branch", chi.URLParam(r, "branch")), + zap.String("commit", chi.URLParam(r, "commit")), + zap.String("branch", chi.URLParam(r, "branch")), + ) + service := chi.URLParam(r, "serviceID") + project := chi.URLParam(r, "project") + branch := chi.URLParam(r, "branch") + commit := chi.URLParam(r, "commit") + bdg := chi.URLParam(r, "badge") - t, err := a.storage.GetByCommit(service, project, branch, commit, false) - if err != nil { - l.Warn("Task get error", zap.Error(err)) - w.WriteHeader(http.StatusNotFound) - return - } + t, err := a.storage.GetByCommit(service, project, branch, commit, latest) + if err != nil { + l.Warn("Task get error", zap.Error(err)) + w.WriteHeader(http.StatusNotFound) + return + } - p := filepath.Join(a.storage.GetVolumePath(t), "/data", fmt.Sprintf("%s.badge", bdg)) + p := filepath.Join(a.storage.GetVolumePath(t), "/data", fmt.Sprintf("%s.badge", bdg)) - _, err = os.Stat(p) - if err != nil { - l.Warn("Task get error", zap.Error(err)) - if os.IsNotExist(err) { - w.WriteHeader(http.StatusNotFound) - } else { - w.WriteHeader(http.StatusBadRequest) + _, err = os.Stat(p) + if err != nil { + l.Warn("Task get error", zap.Error(err)) + if os.IsNotExist(err) { + w.WriteHeader(http.StatusNotFound) + } else { + w.WriteHeader(http.StatusBadRequest) + } + return } - return - } - l = l.With(zap.String("path", p)) - b, err := ioutil.ReadFile(p) - if err != nil { - l.Error("reading file", zap.Error(err)) - w.WriteHeader(http.StatusInternalServerError) - return - } - var badge _badge.Badge - err = json.Unmarshal(b, &badge) - if err != nil { - l.Error("JSON unmarshal", zap.Error(err)) - w.WriteHeader(http.StatusInternalServerError) - return + l = l.With(zap.String("path", p)) + b, err := ioutil.ReadFile(p) + if err != nil { + l.Error("reading file", zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + var badge _badge.Badge + err = json.Unmarshal(b, &badge) + if err != nil { + l.Error("JSON unmarshal", zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + badge.Render(w, r) } - badge.Render(w, r) } From bdd81d3782fd562866fef832736a11d1685e84b4 Mon Sep 17 00:00:00 2001 From: Wilfried OLLIVIER Date: Thu, 10 Feb 2022 08:01:48 +0100 Subject: [PATCH 5/8] Fix: harmonize green colors with other badges --- badge/badge.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/badge/badge.go b/badge/badge.go index cf919f0..4777df8 100644 --- a/badge/badge.go +++ b/badge/badge.go @@ -20,8 +20,8 @@ var colors = statusToColors{ task.Running: "#DD571C", // red - ruby task.Failed: "#900603", - // green - emerald - task.Done: "#028A0F", + // green + task.Done: "#4ec820", }, // blue Default: "#527284", From 95619090251e8a2d5fbdc90f4647293e2e5ad2fd Mon Sep 17 00:00:00 2001 From: Wilfried OLLIVIER Date: Thu, 10 Feb 2022 08:02:05 +0100 Subject: [PATCH 6/8] Make Colors public --- badge/badge.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/badge/badge.go b/badge/badge.go index 4777df8..07046c6 100644 --- a/badge/badge.go +++ b/badge/badge.go @@ -10,7 +10,8 @@ import ( "github.com/narqo/go-badge" ) -var colors = statusToColors{ +// Colors is used to harmonize status colors for all badges +var Colors = statusToColors{ c: map[task.State]badge.Color{ // blue - lapis task.Ready: "#2832C2", @@ -67,7 +68,7 @@ func StatusBadge(s storage.Storage, latest bool) func(http.ResponseWriter, *http t, err := s.GetByCommit(service, project, branch, commit, latest) if t == nil || err != nil { - err = writeBadge("status", "?!", colors.Default, w) + err = writeBadge("status", "?!", Colors.Default, w) if err != nil { panic(err) } @@ -79,7 +80,7 @@ func StatusBadge(s storage.Storage, latest bool) func(http.ResponseWriter, *http return } - writeBadge(fmt.Sprintf("status : %s", service), t.State.String(), colors.Get(t.State), w) + writeBadge(fmt.Sprintf("status : %s", service), t.State.String(), Colors.Get(t.State), w) if err != nil { panic(err) } From d3a471b1113bb707cb5dd83827f948d101684cfb Mon Sep 17 00:00:00 2001 From: Wilfried OLLIVIER Date: Thu, 10 Feb 2022 08:08:21 +0100 Subject: [PATCH 7/8] Write badge goes public --- badge/badge.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/badge/badge.go b/badge/badge.go index 07046c6..236988d 100644 --- a/badge/badge.go +++ b/badge/badge.go @@ -68,7 +68,7 @@ func StatusBadge(s storage.Storage, latest bool) func(http.ResponseWriter, *http t, err := s.GetByCommit(service, project, branch, commit, latest) if t == nil || err != nil { - err = writeBadge("status", "?!", Colors.Default, w) + err = WriteBadge("status", "?!", Colors.Default, w) if err != nil { panic(err) } @@ -80,15 +80,15 @@ func StatusBadge(s storage.Storage, latest bool) func(http.ResponseWriter, *http return } - writeBadge(fmt.Sprintf("status : %s", service), t.State.String(), Colors.Get(t.State), w) + WriteBadge(fmt.Sprintf("status : %s", service), t.State.String(), Colors.Get(t.State), w) if err != nil { panic(err) } } } -// writeBadge is a wrapper use to write a badge into an http response -func writeBadge(label string, content string, color badge.Color, w http.ResponseWriter) error { +// WriteBadge is a wrapper use to write a badge into an http response +func WriteBadge(label string, content string, color badge.Color, w http.ResponseWriter) error { w.Header().Set("content-type", "image/svg+xml") return badge.Render(label, content, color, w) } From 014c298ed6e17ef6b90f7e1f02b33f486d291423 Mon Sep 17 00:00:00 2001 From: Wilfried OLLIVIER Date: Thu, 10 Feb 2022 08:09:38 +0100 Subject: [PATCH 8/8] Add support for status badge in BadgeMyTask --- application/badge.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/application/badge.go b/application/badge.go index 721b939..21ff17b 100644 --- a/application/badge.go +++ b/application/badge.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" + "github.com/factorysh/microdensity/badge" _badge "github.com/factorysh/microdensity/badge" "github.com/go-chi/chi/v5" "go.uber.org/zap" @@ -30,6 +31,7 @@ func (a *Application) BadgeMyTaskHandler(latest bool) http.HandlerFunc { commit := chi.URLParam(r, "commit") bdg := chi.URLParam(r, "badge") + // get the task t, err := a.storage.GetByCommit(service, project, branch, commit, latest) if err != nil { l.Warn("Task get error", zap.Error(err)) @@ -37,16 +39,19 @@ func (a *Application) BadgeMyTaskHandler(latest bool) http.HandlerFunc { return } + // try to get the output badge for this task in this service p := filepath.Join(a.storage.GetVolumePath(t), "/data", fmt.Sprintf("%s.badge", bdg)) - _, err = os.Stat(p) + + // if not found if err != nil { - l.Warn("Task get error", zap.Error(err)) + // fallback to status badge if os.IsNotExist(err) { - w.WriteHeader(http.StatusNotFound) - } else { - w.WriteHeader(http.StatusBadRequest) + // use the service name, task status and colors from badge pkg + badge.WriteBadge(service, t.State.String(), _badge.Colors.Get(t.State), w) + return } + w.WriteHeader(http.StatusBadRequest) return } l = l.With(zap.String("path", p))