From f6f2025fa7828fd08862a0fab466db907645f6e4 Mon Sep 17 00:00:00 2001 From: Abhishek Rai Date: Thu, 7 Nov 2024 19:03:27 -0800 Subject: [PATCH 1/4] chore: Create schema for github events Signed-off-by: Abhishek Rai --- src/github.go | 157 ++++++++++++++------------ src/github_test.go | 21 +++- src/metrics_test.go | 217 ++++++++++++++++++++++++++++-------- test_data/commit.json | 15 +++ test_data/pull_request.json | 21 ++++ test_data/workflow_job.json | 18 +++ test_data/workflow_run.json | 17 +++ 7 files changed, 342 insertions(+), 124 deletions(-) create mode 100644 test_data/commit.json create mode 100644 test_data/pull_request.json create mode 100644 test_data/workflow_job.json create mode 100644 test_data/workflow_run.json diff --git a/src/github.go b/src/github.go index 1a76f23..6aeadc7 100644 --- a/src/github.go +++ b/src/github.go @@ -14,6 +14,72 @@ import ( "go.uber.org/zap" ) +type GithubRepo struct { + FullName string `json:"full_name"` +} + +type GithubWorkflow struct { + Workflow struct { + ID int `json:"id"` + Status string `json:"status"` + RunID int `json:"run_id"` + Name string `json:"name"` + Branch string `json:"head_branch"` + Repository GithubRepo `json:"repository"` + Conclusion string `json:"conclusion"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + HTMLURL string `json:"html_url"` + } `json:"workflow_run"` +} +type GithubJob struct { + Job struct { + ID int `json:"id"` + Status string `json:"status"` + Name string `json:"name"` + Branch string `json:"head_branch"` + Repository GithubRepo `json:"repository"` + RunnerName string `json:"runner_name"` + Conclusion string `json:"conclusion"` + StartedAt string `json:"started_at"` + CompletedAt string `json:"completed_at"` + WorkflowName string `json:"workflow_name"` + HTMLURL string `json:"html_url"` + } `json:"workflow_job"` +} + +type GithubCommit struct { + Repository GithubRepo `json:"repository"` + Commits []struct { + ID string `json:"id"` + Author struct { + Name string `json:"name"` + Email string `json:"email"` + } `json:"author"` + } `json:"commits"` + Ref string `json:"ref"` +} + +type GithubPullRequest struct { + Action string `json:"action"` + PullRequest struct { + ID int `json:"id"` + State string `json:"state"` + Title string `json:"title"` + Base struct { + Ref string `json:"ref"` + } `json:"base"` + Head struct { + Ref string `json:"ref"` + } `json:"head"` + User struct { + Login string `json:"login"` + Email string `json:"email"` + } `json:"user"` + } `json:"pull_request"` + Repository GithubRepo `json:"repository"` +} + func validateHMAC(body []byte, signature string, secret []byte) bool { h := hmac.New(sha256.New, secret) h.Write(body) @@ -54,22 +120,8 @@ func githubEventsHandler(w http.ResponseWriter, r *http.Request) { } func updateWorkflowMetrics(body []byte) { - var payload struct { - Workflow struct { - ID int `json:"id"` - Status string `json:"status"` - RunID int `json:"run_id"` - Name string `json:"name"` - Branch string `json:"head_branch"` - Repository struct { - FullName string `json:"full_name"` - } `json:"repository"` - Conclusion string `json:"conclusion"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - HTMLURL string `json:"html_url"` - } `json:"workflow_run"` - } + var payload GithubWorkflow + if err := json.Unmarshal(body, &payload); err != nil { logger.Error("Failed to unmarshal workflow_run payload", zap.Error(err)) return @@ -132,22 +184,9 @@ func updateWorkflowMetrics(body []byte) { } func updateJobMetrics(body []byte) { - var payload struct { - Job struct { - ID int `json:"id"` - Status string `json:"status"` - Name string `json:"name"` - Branch string `json:"head_branch"` - Repository struct { - FullName string `json:"full_name"` - } `json:"repository"` - RunnerName string `json:"runner_name"` - Conclusion string `json:"conclusion"` - StartedAt string `json:"started_at"` - CompletedAt string `json:"completed_at"` - HTMLURL string `json:"html_url"` - } `json:"workflow_job"` - } + + var payload GithubJob + if err := json.Unmarshal(body, &payload); err != nil { logger.Error("Failed to unmarshal workflow_job payload", zap.Error(err)) return @@ -157,7 +196,7 @@ func updateJobMetrics(body []byte) { "runner": payload.Job.RunnerName, "repository": payload.Job.Repository.FullName, "branch": payload.Job.Branch, - "workflow_name": payload.Job.Name, + "workflow_name": payload.Job.WorkflowName, "job_name": payload.Job.Name, "job_status": payload.Job.Status, "job_conclusion": payload.Job.Conclusion, @@ -171,7 +210,7 @@ func updateJobMetrics(body []byte) { "runner": payload.Job.RunnerName, "repository": payload.Job.Repository.FullName, "branch": payload.Job.Branch, - "workflow_name": payload.Job.Name, + "workflow_name": payload.Job.WorkflowName, "job_name": payload.Job.Name, }).Inc() case "in_progress": @@ -179,14 +218,14 @@ func updateJobMetrics(body []byte) { "runner": payload.Job.RunnerName, "repository": payload.Job.Repository.FullName, "branch": payload.Job.Branch, - "workflow_name": payload.Job.Name, + "workflow_name": payload.Job.WorkflowName, "job_name": payload.Job.Name, }).Inc() jobQueuedGauge.With(prometheus.Labels{ "runner": payload.Job.RunnerName, "repository": payload.Job.Repository.FullName, "branch": payload.Job.Branch, - "workflow_name": payload.Job.Name, + "workflow_name": payload.Job.WorkflowName, "job_name": payload.Job.Name, }).Dec() case "completed": @@ -194,14 +233,14 @@ func updateJobMetrics(body []byte) { "runner": payload.Job.RunnerName, "repository": payload.Job.Repository.FullName, "branch": payload.Job.Branch, - "workflow_name": payload.Job.Name, + "workflow_name": payload.Job.WorkflowName, "job_name": payload.Job.Name, }).Inc() jobInProgressGauge.With(prometheus.Labels{ "runner": payload.Job.RunnerName, "repository": payload.Job.Repository.FullName, "branch": payload.Job.Branch, - "workflow_name": payload.Job.Name, + "workflow_name": payload.Job.WorkflowName, "job_name": payload.Job.Name, }).Dec() @@ -214,7 +253,7 @@ func updateJobMetrics(body []byte) { "runner": payload.Job.RunnerName, "repository": payload.Job.Repository.FullName, "branch": payload.Job.Branch, - "workflow_name": payload.Job.Name, + "workflow_name": payload.Job.WorkflowName, "job_name": payload.Job.Name, "job_status": payload.Job.Status, "job_conclusion": payload.Job.Conclusion, @@ -224,19 +263,9 @@ func updateJobMetrics(body []byte) { } func updateCommitMetrics(body []byte) { - var payload struct { - Repository struct { - FullName string `json:"full_name"` - } `json:"repository"` - Commits []struct { - ID string `json:"id"` - Author struct { - Name string `json:"name"` - Email string `json:"email"` - } `json:"author"` - } `json:"commits"` - Ref string `json:"ref"` - } + + var payload GithubCommit + if err := json.Unmarshal(body, &payload); err != nil { logger.Error("Failed to unmarshal push payload", zap.Error(err)) return @@ -253,27 +282,9 @@ func updateCommitMetrics(body []byte) { } func updatePullRequestMetrics(body []byte) { - var payload struct { - Action string `json:"action"` - PullRequest struct { - ID int `json:"id"` - State string `json:"state"` - Title string `json:"title"` - Base struct { - Ref string `json:"ref"` - } `json:"base"` - Head struct { - Ref string `json:"ref"` - } `json:"head"` - User struct { - Login string `json:"login"` - Email string `json:"email"` - } `json:"user"` - } `json:"pull_request"` - Repository struct { - FullName string `json:"full_name"` - } `json:"repository"` - } + + var payload GithubPullRequest + if err := json.Unmarshal(body, &payload); err != nil { logger.Error("Failed to unmarshal pull_request payload", zap.Error(err)) return diff --git a/src/github_test.go b/src/github_test.go index 00150c0..94ea3dd 100644 --- a/src/github_test.go +++ b/src/github_test.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "net/http" "net/http/httptest" + "os" "testing" "github.com/stretchr/testify/assert" @@ -40,13 +41,19 @@ func TestValidateHMAC(t *testing.T) { } func TestValidPayload(t *testing.T) { - body := []byte(`{"workflow_run": {"id": 1, "status": "completed", "run_id": 1001, "name": "CI", "head_branch": "main", "repository": {"full_name": "user/repo"}, "conclusion": "success", "created_at": "2023-01-01T00:00:00Z", "updated_at": "2023-01-01T01:00:00Z"}}`) + body, err := os.ReadFile("../test_data/workflow_run.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } recorder := sendTestRequest(body, "workflow_run") assert.Equal(t, http.StatusOK, recorder.Code) } func TestInvalidSignature(t *testing.T) { - body := []byte(`{"workflow_run": {"id": 1, "status": "completed", "run_id": 1001, "name": "CI", "head_branch": "main", "repository": {"full_name": "user/repo"}, "conclusion": "success", "created_at": "2023-01-01T00:00:00Z", "updated_at": "2023-01-01T01:00:00Z"}}`) + body, err := os.ReadFile("../test_data/workflow_run.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } recorder := httptest.NewRecorder() req := httptest.NewRequest(http.MethodPost, "/webhook", bytes.NewBuffer(body)) req.Header.Set("X-Hub-Signature-256", "invalid_signature") @@ -59,7 +66,10 @@ func TestInvalidSignature(t *testing.T) { } func TestMissingSignature(t *testing.T) { - body := []byte(`{"workflow_run": {"id": 1, "status": "completed", "run_id": 1001, "name": "CI", "head_branch": "main", "repository": {"full_name": "user/repo"}, "conclusion": "success", "created_at": "2023-01-01T00:00:00Z", "updated_at": "2023-01-01T01:00:00Z"}}`) + body, err := os.ReadFile("../test_data/workflow_run.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } recorder := httptest.NewRecorder() req := httptest.NewRequest(http.MethodPost, "/webhook", bytes.NewBuffer(body)) req.Header.Set("X-GitHub-Event", "workflow_run") @@ -71,7 +81,10 @@ func TestMissingSignature(t *testing.T) { } func TestUnknownEvent(t *testing.T) { - body := []byte(`{"workflow_run": {"id": 1, "status": "completed", "run_id": 1001, "name": "CI", "head_branch": "main", "repository": {"full_name": "user/repo"}, "conclusion": "success", "created_at": "2023-01-01T00:00:00Z", "updated_at": "2023-01-01T01:00:00Z"}}`) + body, err := os.ReadFile("../test_data/workflow_run.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } recorder := sendTestRequest(body, "unknown_event") assert.Equal(t, http.StatusOK, recorder.Code) } diff --git a/src/metrics_test.go b/src/metrics_test.go index 83b1bbc..935a238 100644 --- a/src/metrics_test.go +++ b/src/metrics_test.go @@ -1,18 +1,33 @@ package main import ( + "encoding/json" + "os" "strings" "testing" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" + "go.uber.org/zap" ) +var ( + reg *prometheus.Registry +) + +func init() { + // Disable logging + logger = zap.NewNop() + reg = prometheus.NewRegistry() +} + func TestWorkflowStatusCounter(t *testing.T) { - reg := prometheus.NewRegistry() workflowStatusCounter.Reset() reg.MustRegister(workflowStatusCounter) - body := []byte(`{"workflow_run": {"id": 1, "status": "completed", "run_id": 1001, "name": "CI", "head_branch": "main", "repository": {"full_name": "user/repo"}, "conclusion": "success", "html_url": "https://github.com/user/repo/actions/runs/1001", "created_at": "2023-01-01T00:00:00Z", "updated_at": "2023-01-01T01:00:00Z"}}`) + body, err := os.ReadFile("../test_data/workflow_run.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } updateWorkflowMetrics(body) // Test counter @@ -26,27 +41,31 @@ func TestWorkflowStatusCounter(t *testing.T) { } func TestJobStatusCounter(t *testing.T) { - reg := prometheus.NewRegistry() jobStatusCounter.Reset() reg.MustRegister(jobStatusCounter) - body := []byte(`{"workflow_job": {"id": 1, "status": "completed", "name": "Job1", "head_branch": "main", "repository": {"full_name": "user/repo"}, "runner_name": "runner1", "conclusion": "success", "html_url": "https://github.com/user/repo/actions/jobs/1", "started_at": "2023-01-01T00:00:00Z", "completed_at": "2023-01-01T01:00:00Z"}}`) + body, err := os.ReadFile("../test_data/workflow_job.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } updateJobMetrics(body) // Test counter if err := testutil.CollectAndCompare(jobStatusCounter, strings.NewReader(` # HELP promgithub_job_status Total number of jobs with status # TYPE promgithub_job_status counter - promgithub_job_status{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",job_url="https://github.com/user/repo/actions/jobs/1",repository="user/repo",runner="runner1",workflow_name="Job1"} 1 + promgithub_job_status{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",job_url="https://github.com/user/repo/actions/jobs/1",repository="user/repo",runner="runner1",workflow_name="CI"} 1 `)); err != nil { t.Errorf("unexpected metrics: %v", err) } } func TestCommitsPushedCounter(t *testing.T) { - reg := prometheus.NewRegistry() commitPushedCounter.Reset() reg.MustRegister(commitPushedCounter) - body := []byte(`{"repository": {"full_name": "user/repo"}, "commits": [{"id": "commit1", "author": {"name": "Author1", "email": "author1@example.com"}}], "ref": "refs/heads/main"}`) + body, err := os.ReadFile("../test_data/commit.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } updateCommitMetrics(body) // Test counter @@ -60,10 +79,12 @@ func TestCommitsPushedCounter(t *testing.T) { } func TestPullRequestsCounter(t *testing.T) { - reg := prometheus.NewRegistry() pullRequestCounter.Reset() reg.MustRegister(pullRequestCounter) - body := []byte(`{"action": "opened", "pull_request": {"id": 1, "state": "open", "title": "PR title", "base": {"ref": "main"}, "head": {"ref": "feature-branch"}, "user": {"login": "user1", "email": "user1@example.com"}}, "repository": {"full_name": "user/repo"}}`) + body, err := os.ReadFile("../test_data/pull_request.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } updatePullRequestMetrics(body) // Test counter @@ -77,10 +98,12 @@ func TestPullRequestsCounter(t *testing.T) { } func TestWorkflowDurationHistogram(t *testing.T) { - reg := prometheus.NewRegistry() workflowDurationHistogram.Reset() reg.MustRegister(workflowDurationHistogram) - body := []byte(`{"workflow_run": {"id": 1, "status": "completed", "run_id": 1001, "name": "CI", "head_branch": "main", "repository": {"full_name": "user/repo"}, "conclusion": "success", "created_at": "2023-01-01T00:00:00Z", "updated_at": "2023-01-01T01:00:00Z"}}`) + body, err := os.ReadFile("../test_data/workflow_run.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } updateWorkflowMetrics(body) // Test histogram @@ -107,41 +130,62 @@ func TestWorkflowDurationHistogram(t *testing.T) { } func TestJobDurationHistogram(t *testing.T) { - reg := prometheus.NewRegistry() jobDurationHistogram.Reset() reg.MustRegister(jobDurationHistogram) - body := []byte(`{"workflow_job": {"id": 1, "status": "completed", "name": "Job1", "head_branch": "main", "repository": {"full_name": "user/repo"}, "runner_name": "runner1", "conclusion": "success", "started_at": "2023-01-01T00:00:00Z", "completed_at": "2023-01-01T01:00:00Z"}}`) + body, err := os.ReadFile("../test_data/workflow_job.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } updateJobMetrics(body) // Test histogram if err := testutil.CollectAndCompare(jobDurationHistogram, strings.NewReader(` # HELP promgithub_job_duration Duration of jobs runs in seconds # TYPE promgithub_job_duration histogram - promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1",le="0.005"} 0 - promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1",le="0.01"} 0 - promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1",le="0.025"} 0 - promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1",le="0.05"} 0 - promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1",le="0.1"} 0 - promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1",le="0.25"} 0 - promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1",le="0.5"} 0 - promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1",le="1"} 0 - promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1",le="2.5"} 0 - promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1",le="5"} 0 - promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1",le="10"} 0 - promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1",le="+Inf"} 1 - promgithub_job_duration_sum{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1"} 3600 - promgithub_job_duration_count{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="Job1"} 1 + promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI",le="0.005"} 0 + promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI",le="0.01"} 0 + promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI",le="0.025"} 0 + promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI",le="0.05"} 0 + promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI",le="0.1"} 0 + promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI",le="0.25"} 0 + promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI",le="0.5"} 0 + promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI",le="1"} 0 + promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI",le="2.5"} 0 + promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI",le="5"} 0 + promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI",le="10"} 0 + promgithub_job_duration_bucket{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI",le="+Inf"} 1 + promgithub_job_duration_sum{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI"} 3600 + promgithub_job_duration_count{branch="main",job_conclusion="success",job_name="Job1",job_status="completed",repository="user/repo",runner="runner1",workflow_name="CI"} 1 `)); err != nil { t.Errorf("unexpected metrics: %v", err) } } func TestWorkflowQueuedGauge(t *testing.T) { - reg := prometheus.NewRegistry() workflowQueuedGauge.Reset() reg.MustRegister(workflowQueuedGauge) - body := []byte(`{"workflow_run": {"id": 1, "status": "queued", "run_id": 1001, "name": "CI", "head_branch": "main", "repository": {"full_name": "user/repo"}}}`) - updateWorkflowMetrics(body) + body, err := os.ReadFile("../test_data/workflow_run.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } + + var payload GithubWorkflow + + // Unmarshal the JSON data into the struct + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("Failed to unmarshal JSON data: %v", err) + } + + // Modify the status field + payload.Workflow.Status = "queued" + + // Marshal the modified struct back to JSON if needed + modifiedBody, err := json.Marshal(payload) + if err != nil { + t.Fatalf("Failed to marshal modified JSON data: %v", err) + } + + updateWorkflowMetrics(modifiedBody) // Test gauge if err := testutil.CollectAndCompare(workflowQueuedGauge, strings.NewReader(` @@ -154,11 +198,30 @@ func TestWorkflowQueuedGauge(t *testing.T) { } func TestWorkflowInProgressGauge(t *testing.T) { - reg := prometheus.NewRegistry() workflowInProgressGauge.Reset() reg.MustRegister(workflowInProgressGauge) - body := []byte(`{"workflow_run": {"id": 1, "status": "in_progress", "run_id": 1001, "name": "CI", "head_branch": "main", "repository": {"full_name": "user/repo"}}}`) - updateWorkflowMetrics(body) + body, err := os.ReadFile("../test_data/workflow_run.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } + + var payload GithubWorkflow + + // Unmarshal the JSON data into the struct + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("Failed to unmarshal JSON data: %v", err) + } + + // Modify the status field + payload.Workflow.Status = "in_progress" + + // Marshal the modified struct back to JSON if needed + modifiedBody, err := json.Marshal(payload) + if err != nil { + t.Fatalf("Failed to marshal modified JSON data: %v", err) + } + + updateWorkflowMetrics(modifiedBody) // Test gauge if err := testutil.CollectAndCompare(workflowInProgressGauge, strings.NewReader(` @@ -171,10 +234,12 @@ func TestWorkflowInProgressGauge(t *testing.T) { } func TestWorkflowCompletedGauge(t *testing.T) { - reg := prometheus.NewRegistry() workflowCompletedGauge.Reset() reg.MustRegister(workflowCompletedGauge) - body := []byte(`{"workflow_run": {"id": 1, "status": "completed", "run_id": 1001, "name": "CI", "head_branch": "main", "repository": {"full_name": "user/repo"}}}`) + body, err := os.ReadFile("../test_data/workflow_run.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } updateWorkflowMetrics(body) // Test gauge @@ -188,51 +253,109 @@ func TestWorkflowCompletedGauge(t *testing.T) { } func TestJobQueuedGauge(t *testing.T) { - reg := prometheus.NewRegistry() jobQueuedGauge.Reset() reg.MustRegister(jobQueuedGauge) - body := []byte(`{"workflow_job": {"id": 1, "status": "queued", "name": "Job1", "head_branch": "main", "repository": {"full_name": "user/repo"}, "runner_name": "runner1"}}`) - updateJobMetrics(body) + body, err := os.ReadFile("../test_data/workflow_job.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } + + var payload GithubJob + + // Unmarshal the JSON data into the struct + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("Failed to unmarshal JSON data: %v", err) + } + + // Modify the status field + payload.Job.Status = "queued" + + // Marshal the modified struct back to JSON + modifiedBody, err := json.Marshal(payload) + if err != nil { + t.Fatalf("Failed to marshal modified JSON data: %v", err) + } + + updateJobMetrics(modifiedBody) // Test gauge if err := testutil.CollectAndCompare(jobQueuedGauge, strings.NewReader(` # HELP promgithub_job_queued Number of jobs queued # TYPE promgithub_job_queued gauge - promgithub_job_queued{branch="main",job_name="Job1",repository="user/repo",runner="runner1",workflow_name="Job1"} 1 + promgithub_job_queued{branch="main",job_name="Job1",repository="user/repo",runner="runner1",workflow_name="CI"} 1 `)); err != nil { t.Errorf("unexpected metrics: %v", err) } } func TestJobInProgressGauge(t *testing.T) { - reg := prometheus.NewRegistry() jobInProgressGauge.Reset() reg.MustRegister(jobInProgressGauge) - body := []byte(`{"workflow_job": {"id": 1, "status": "in_progress", "name": "Job1", "head_branch": "main", "repository": {"full_name": "user/repo"}, "runner_name": "runner1"}}`) - updateJobMetrics(body) + body, err := os.ReadFile("../test_data/workflow_job.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } + + var payload GithubJob + + // Unmarshal the JSON data into the struct + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("Failed to unmarshal JSON data: %v", err) + } + + // Modify the status field + payload.Job.Status = "in_progress" + + // Marshal the modified struct back to JSON + modifiedBody, err := json.Marshal(payload) + if err != nil { + t.Fatalf("Failed to marshal modified JSON data: %v", err) + } + + updateJobMetrics(modifiedBody) // Test gauge if err := testutil.CollectAndCompare(jobInProgressGauge, strings.NewReader(` # HELP promgithub_job_in_progress Number of jobs in progress # TYPE promgithub_job_in_progress gauge - promgithub_job_in_progress{branch="main",job_name="Job1",repository="user/repo",runner="runner1",workflow_name="Job1"} 1 + promgithub_job_in_progress{branch="main",job_name="Job1",repository="user/repo",runner="runner1",workflow_name="CI"} 1 `)); err != nil { t.Errorf("unexpected metrics: %v", err) } } func TestJobCompletedGauge(t *testing.T) { - reg := prometheus.NewRegistry() jobCompletedGauge.Reset() reg.MustRegister(jobCompletedGauge) - body := []byte(`{"workflow_job": {"id": 1, "status": "completed", "name": "Job1", "head_branch": "main", "repository": {"full_name": "user/repo"}, "runner_name": "runner1"}}`) - updateJobMetrics(body) + + body, err := os.ReadFile("../test_data/workflow_job.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } + + var payload GithubJob + + // Unmarshal the JSON data into the struct + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("Failed to unmarshal JSON data: %v", err) + } + + // Modify the status field + payload.Job.Status = "completed" + + // Marshal the modified struct back to JSON + modifiedBody, err := json.Marshal(payload) + if err != nil { + t.Fatalf("Failed to marshal modified JSON data: %v", err) + } + + updateJobMetrics(modifiedBody) // Test gauge if err := testutil.CollectAndCompare(jobCompletedGauge, strings.NewReader(` # HELP promgithub_job_completed Number of jobs completed # TYPE promgithub_job_completed gauge - promgithub_job_completed{branch="main",job_name="Job1",repository="user/repo",runner="runner1",workflow_name="Job1"} 1 + promgithub_job_completed{branch="main",job_name="Job1",repository="user/repo",runner="runner1",workflow_name="CI"} 1 `)); err != nil { t.Errorf("unexpected metrics: %v", err) } diff --git a/test_data/commit.json b/test_data/commit.json new file mode 100644 index 0000000..b514684 --- /dev/null +++ b/test_data/commit.json @@ -0,0 +1,15 @@ +{ + "repository": { + "full_name": "user/repo" + }, + "commits": [ + { + "id": "commit1", + "author": { + "name": "Author1", + "email": "author1@example.com" + } + } + ], + "ref": "refs/heads/main" +} diff --git a/test_data/pull_request.json b/test_data/pull_request.json new file mode 100644 index 0000000..a63ac04 --- /dev/null +++ b/test_data/pull_request.json @@ -0,0 +1,21 @@ +{ + "action": "opened", + "pull_request": { + "id": 1, + "state": "open", + "title": "PR title", + "base": { + "ref": "main" + }, + "head": { + "ref": "feature-branch" + }, + "user": { + "login": "user1", + "email": "user1@example.com" + } + }, + "repository": { + "full_name": "user/repo" + } +} diff --git a/test_data/workflow_job.json b/test_data/workflow_job.json new file mode 100644 index 0000000..bba24d6 --- /dev/null +++ b/test_data/workflow_job.json @@ -0,0 +1,18 @@ +{ + "action": "completed", + "workflow_job": { + "id": 1, + "status": "completed", + "name": "Job1", + "workflow_name": "CI", + "head_branch": "main", + "repository": { + "full_name": "user/repo" + }, + "runner_name": "runner1", + "conclusion": "success", + "html_url": "https://github.com/user/repo/actions/jobs/1", + "started_at": "2023-01-01T00:00:00Z", + "completed_at": "2023-01-01T01:00:00Z" + } +} diff --git a/test_data/workflow_run.json b/test_data/workflow_run.json new file mode 100644 index 0000000..79b27a6 --- /dev/null +++ b/test_data/workflow_run.json @@ -0,0 +1,17 @@ +{ + "action": "completed", + "workflow_run": { + "id": 1, + "status": "completed", + "run_id": 1001, + "name": "CI", + "head_branch": "main", + "repository": { + "full_name": "user/repo" + }, + "conclusion": "success", + "html_url": "https://github.com/user/repo/actions/runs/1001", + "created_at": "2023-01-01T00:00:00Z", + "updated_at": "2023-01-01T01:00:00Z" + } +} From 8874d0ad95c652f65f0ec77f6ecc535fbbbd69c9 Mon Sep 17 00:00:00 2001 From: Abhishek Rai Date: Thu, 7 Nov 2024 19:29:13 -0800 Subject: [PATCH 2/4] add tests for main.go and improve coverage Signed-off-by: Abhishek Rai --- src/github_test.go | 21 ++++-- src/main_test.go | 109 +++++++++++++++++++++++++++ src/metrics_test.go | 2 +- test_data/{commit.json => push.json} | 0 4 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 src/main_test.go rename test_data/{commit.json => push.json} (100%) diff --git a/src/github_test.go b/src/github_test.go index 94ea3dd..4b1ab8f 100644 --- a/src/github_test.go +++ b/src/github_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -40,13 +41,23 @@ func TestValidateHMAC(t *testing.T) { assert.True(t, valid) } -func TestValidPayload(t *testing.T) { - body, err := os.ReadFile("../test_data/workflow_run.json") +func TestValidWorkflowPayload(t *testing.T) { + dir, err := os.ReadDir("../test_data") if err != nil { - t.Fatalf("Failed to read test data file: %v", err) + t.Fatalf("Failed to read test data directory: %v", err) + } + for _, file := range dir { + if file.IsDir() { + continue + } + body, err := os.ReadFile("../test_data/" + file.Name()) + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } + eventType := strings.TrimSuffix(file.Name(), ".json") + recorder := sendTestRequest(body, eventType) + assert.Equal(t, http.StatusOK, recorder.Code) } - recorder := sendTestRequest(body, "workflow_run") - assert.Equal(t, http.StatusOK, recorder.Code) } func TestInvalidSignature(t *testing.T) { diff --git a/src/main_test.go b/src/main_test.go new file mode 100644 index 0000000..b1c5bba --- /dev/null +++ b/src/main_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "go.uber.org/zap" +) + +func TestHealthCheck(t *testing.T) { + // Set the Version variable for the test + Version = "1.0.0" + + // Create a test HTTP request + req, err := http.NewRequest("GET", "/health", nil) + if err != nil { + t.Fatalf("Failed to create HTTP request: %v", err) + } + + // Create a test HTTP response recorder + rr := httptest.NewRecorder() + + // Call the healthCheck handler + handler := http.HandlerFunc(healthCheck) + handler.ServeHTTP(rr, req) + + // Verify the response status code + if status := rr.Code; status != http.StatusOK { + t.Errorf("Expected status code %d, got %d", http.StatusOK, status) + } + + // Verify the response body + expectedResponse := HealthCheckResposne{Status: "ok", Version: Version} + var actualResponse HealthCheckResposne + if err := json.NewDecoder(rr.Body).Decode(&actualResponse); err != nil { + t.Fatalf("Failed to decode response body: %v", err) + } + + if actualResponse != expectedResponse { + t.Errorf("Expected response body %+v, got %+v", expectedResponse, actualResponse) + } +} + +func TestAPIHandler(t *testing.T) { + // Initialize the logger + logger := zap.NewNop() + + // Initialize the Prometheus metrics + apiCallsCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "promgithub_api_calls_total", + Help: "Number of API calls", + }, + []string{"status", "method", "path"}, + ) + requestDurationHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "promgithub_request_duration_seconds", + Help: "Request duration in seconds", + Buckets: prometheus.DefBuckets, + }, + []string{"path", "method"}, + ) + + // Create a test HTTP handler + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + if _, err := w.Write([]byte("OK")); err != nil { + t.Errorf("Failed to write response: %v", err) + } + }) + + // Wrap the test handler with the APIHandler middleware + handler := APIHandler(logger)(testHandler) + + // Create a test HTTP server + server := httptest.NewServer(handler) + defer server.Close() + + // Create a test HTTP client + client := &http.Client{Timeout: 10 * time.Second} + + // Send a test HTTP request + resp, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Failed to send HTTP request: %v", err) + } + defer resp.Body.Close() + + // Verify the response status code + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status code %d, got %d", http.StatusOK, resp.StatusCode) + } + + // Verify the Prometheus metrics + if err := testutil.CollectAndCompare(apiCallsCounter, strings.NewReader(` + # HELP promgithub_api_calls_total Number of API calls + # TYPE promgithub_api_calls_total counter + promgithub_api_calls_total{method="GET",path="/",status="OK"} 1 + `)); err != nil { + t.Errorf("unexpected metrics: %v", err) + } +} diff --git a/src/metrics_test.go b/src/metrics_test.go index 935a238..9391c9a 100644 --- a/src/metrics_test.go +++ b/src/metrics_test.go @@ -62,7 +62,7 @@ func TestJobStatusCounter(t *testing.T) { func TestCommitsPushedCounter(t *testing.T) { commitPushedCounter.Reset() reg.MustRegister(commitPushedCounter) - body, err := os.ReadFile("../test_data/commit.json") + body, err := os.ReadFile("../test_data/push.json") if err != nil { t.Fatalf("Failed to read test data file: %v", err) } diff --git a/test_data/commit.json b/test_data/push.json similarity index 100% rename from test_data/commit.json rename to test_data/push.json From e2218f71d5a333750acba56cdb53f7d01438a0c7 Mon Sep 17 00:00:00 2001 From: Abhishek Rai Date: Thu, 7 Nov 2024 19:39:21 -0800 Subject: [PATCH 3/4] re-organize Signed-off-by: Abhishek Rai --- src/main_test.go | 31 ++++++++++++------------------- src/metrics_test.go | 12 ------------ 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/src/main_test.go b/src/main_test.go index b1c5bba..4976cf5 100644 --- a/src/main_test.go +++ b/src/main_test.go @@ -13,6 +13,16 @@ import ( "go.uber.org/zap" ) +var ( + reg *prometheus.Registry +) + +func init() { + // Disable logging + logger = zap.NewNop() + reg = prometheus.NewRegistry() +} + func TestHealthCheck(t *testing.T) { // Set the Version variable for the test Version = "1.0.0" @@ -48,25 +58,8 @@ func TestHealthCheck(t *testing.T) { } func TestAPIHandler(t *testing.T) { - // Initialize the logger - logger := zap.NewNop() - - // Initialize the Prometheus metrics - apiCallsCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "promgithub_api_calls_total", - Help: "Number of API calls", - }, - []string{"status", "method", "path"}, - ) - requestDurationHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "promgithub_request_duration_seconds", - Help: "Request duration in seconds", - Buckets: prometheus.DefBuckets, - }, - []string{"path", "method"}, - ) + apiCallsCounter.Reset() + reg.MustRegister(apiCallsCounter) // Create a test HTTP handler testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/src/metrics_test.go b/src/metrics_test.go index 9391c9a..1ac39ed 100644 --- a/src/metrics_test.go +++ b/src/metrics_test.go @@ -6,21 +6,9 @@ import ( "strings" "testing" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" - "go.uber.org/zap" ) -var ( - reg *prometheus.Registry -) - -func init() { - // Disable logging - logger = zap.NewNop() - reg = prometheus.NewRegistry() -} - func TestWorkflowStatusCounter(t *testing.T) { workflowStatusCounter.Reset() reg.MustRegister(workflowStatusCounter) From 444b18059956597a776d79da60f486ea10687fce Mon Sep 17 00:00:00 2001 From: Abhishek Rai Date: Thu, 7 Nov 2024 19:51:16 -0800 Subject: [PATCH 4/4] add tests for routes Signed-off-by: Abhishek Rai --- src/main.go | 32 ++++++++++++++++++-------------- src/main_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/src/main.go b/src/main.go index 8fd8b77..b36193a 100644 --- a/src/main.go +++ b/src/main.go @@ -108,25 +108,12 @@ func init() { defer logger.Sync() } -func main() { - port := strings.TrimSpace(os.Getenv("PROMGITHUB_SERVICE_PORT")) - if port == "" { - port = "8080" - } - - ghWebhookSecretEnv := strings.TrimSpace(os.Getenv("PROMGITHUB_WEBHOOK_SECRET")) - if ghWebhookSecretEnv == "" { - logger.Fatal("Environment variable PROMGITHUB_WEBHOOK_SECRET is not set") - } - githubWebhookSecret = []byte(ghWebhookSecretEnv) - +func setupRouter(logger *zap.Logger) *mux.Router { r := mux.NewRouter() r.Use(APIHandler(logger)) r.HandleFunc("/health", healthCheck).Methods("GET") - r.Handle("/metrics", promhttp.Handler()) - r.HandleFunc("/webhook", githubEventsHandler).Methods("POST") // Profiling endpoints @@ -138,6 +125,23 @@ func main() { r.HandleFunc("/debug/pprof/trace", pprof.Trace) } + return r +} + +func main() { + port := strings.TrimSpace(os.Getenv("PROMGITHUB_SERVICE_PORT")) + if port == "" { + port = "8080" + } + + ghWebhookSecretEnv := strings.TrimSpace(os.Getenv("PROMGITHUB_WEBHOOK_SECRET")) + if ghWebhookSecretEnv == "" { + logger.Fatal("Environment variable PROMGITHUB_WEBHOOK_SECRET is not set") + } + githubWebhookSecret = []byte(ghWebhookSecretEnv) + + r := setupRouter(logger) + logger.Info("Starting server", zap.String("port", port)) if err := http.ListenAndServe(":"+port, r); err != nil { logger.Fatal("Error starting server", zap.Error(err)) diff --git a/src/main_test.go b/src/main_test.go index 4976cf5..f0c2f64 100644 --- a/src/main_test.go +++ b/src/main_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "os" "strings" "testing" "time" @@ -57,6 +58,44 @@ func TestHealthCheck(t *testing.T) { } } +func TestSetupRouter(t *testing.T) { + // Set environment variables for the test + os.Setenv("PROMGITHUB_WEBHOOK_SECRET", "testsecret") + defer os.Unsetenv("PROMGITHUB_WEBHOOK_SECRET") + + // Initialize the logger + logger := zap.NewNop() + + // Set up the router + r := setupRouter(logger) + + // Create a test HTTP server + server := httptest.NewServer(r) + defer server.Close() + + // Test the /health endpoint + resp, err := http.Get(server.URL + "/health") + if err != nil { + t.Fatalf("Failed to send HTTP request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status code %d, got %d", http.StatusOK, resp.StatusCode) + } + + // Test the /metrics endpoint + resp, err = http.Get(server.URL + "/metrics") + if err != nil { + t.Fatalf("Failed to send HTTP request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status code %d, got %d", http.StatusOK, resp.StatusCode) + } +} + func TestAPIHandler(t *testing.T) { apiCallsCounter.Reset() reg.MustRegister(apiCallsCounter)