diff --git a/.github/workflows/ci-doctor.lock.yml b/.github/workflows/ci-doctor.lock.yml index b18336e9e89..c2504a29096 100644 --- a/.github/workflows/ci-doctor.lock.yml +++ b/.github/workflows/ci-doctor.lock.yml @@ -5,7 +5,7 @@ # # Source: githubnext/agentics/workflows/ci-doctor.md@09e77ed2e49f0612e258db12839e86e8e2a6c692 # -# Effective stop-time: 2025-11-16 13:10:44 +# Effective stop-time: 2025-11-16 13:48:24 # # Job Dependency Graph: # ```mermaid @@ -4914,7 +4914,7 @@ jobs: id: check_stop_time uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: - GH_AW_STOP_TIME: 2025-11-16 13:10:44 + GH_AW_STOP_TIME: 2025-11-16 13:48:24 GH_AW_WORKFLOW_NAME: "CI Failure Doctor" with: script: | diff --git a/.github/workflows/daily-team-status.lock.yml b/.github/workflows/daily-team-status.lock.yml index 3e6cc7c83a6..2d028e14f85 100644 --- a/.github/workflows/daily-team-status.lock.yml +++ b/.github/workflows/daily-team-status.lock.yml @@ -5,7 +5,7 @@ # # Source: githubnext/agentics/workflows/daily-team-status.md@1e366aa4518cf83d25defd84e454b9a41e87cf7c # -# Effective stop-time: 2025-12-14 13:10:46 +# Effective stop-time: 2025-12-14 13:48:26 # # Job Dependency Graph: # ```mermaid @@ -4205,7 +4205,7 @@ jobs: id: check_stop_time uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: - GH_AW_STOP_TIME: 2025-12-14 13:10:46 + GH_AW_STOP_TIME: 2025-12-14 13:48:26 GH_AW_WORKFLOW_NAME: "Daily Team Status" with: script: | diff --git a/.github/workflows/super-linter.lock.yml b/.github/workflows/super-linter.lock.yml index 255d1dc1430..f9d8f5faec9 100644 --- a/.github/workflows/super-linter.lock.yml +++ b/.github/workflows/super-linter.lock.yml @@ -4569,7 +4569,7 @@ jobs: fi - name: Upload super-linter log if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 with: name: super-linter-log path: super-linter.log diff --git a/pkg/workflow/action_pins_test.go b/pkg/workflow/action_pins_test.go index 7af2d7a7377..4d14564dc15 100644 --- a/pkg/workflow/action_pins_test.go +++ b/pkg/workflow/action_pins_test.go @@ -345,9 +345,9 @@ func TestApplyActionPinToStep(t *testing.T) { func TestGetActionPinsSorting(t *testing.T) { pins := getActionPins() - // Verify we got all the pins (should be 18) - if len(pins) != 18 { - t.Errorf("getActionPins() returned %d pins, expected 18", len(pins)) + // Verify we got all the pins (should be 20) + if len(pins) != 20 { + t.Errorf("getActionPins() returned %d pins, expected 20", len(pins)) } // Verify they are sorted by version (descending) then by repository name (ascending) diff --git a/pkg/workflow/action_resolver.go b/pkg/workflow/action_resolver.go index b5c9d49c4b1..89d88ed262d 100644 --- a/pkg/workflow/action_resolver.go +++ b/pkg/workflow/action_resolver.go @@ -50,6 +50,7 @@ func (r *ActionResolver) ResolveSHA(repo, version string) (string, error) { } // resolveFromGitHub uses gh CLI to resolve the SHA for an action@version +// Falls back to unauthenticated REST API if gh CLI fails due to authentication func (r *ActionResolver) resolveFromGitHub(repo, version string) (string, error) { // Extract base repository (for actions like "github/codeql-action/upload-sarif") baseRepo := extractBaseRepo(repo) @@ -60,13 +61,16 @@ func (r *ActionResolver) resolveFromGitHub(repo, version string) (string, error) apiPath := fmt.Sprintf("/repos/%s/git/ref/tags/%s", baseRepo, version) resolverLog.Printf("Querying GitHub API: %s", apiPath) - cmd := ExecGH("api", apiPath, "--jq", ".object.sha") - output, err := cmd.Output() + output, fromREST, err := ExecGHAPIWithRESTFallback(apiPath, ".object.sha") if err != nil { // Try without "refs/tags/" prefix in case version is already a ref return "", fmt.Errorf("failed to resolve %s@%s: %w", repo, version, err) } + if fromREST { + resolverLog.Printf("Resolved using REST API fallback") + } + sha := strings.TrimSpace(string(output)) if sha == "" { return "", fmt.Errorf("empty SHA returned for %s@%s", repo, version) diff --git a/pkg/workflow/gh_helper.go b/pkg/workflow/gh_helper.go index 3f3421a98c4..1678a283522 100644 --- a/pkg/workflow/gh_helper.go +++ b/pkg/workflow/gh_helper.go @@ -1,8 +1,14 @@ package workflow import ( + "encoding/json" + "fmt" + "io" + "net/http" "os" "os/exec" + "strings" + "time" "github.com/githubnext/gh-aw/pkg/logger" ) @@ -39,3 +45,189 @@ func ExecGH(args ...string) *exec.Cmd { return cmd } + +// ExecGHAPIWithRESTFallback executes a "gh api" command with fallback to unauthenticated REST API. +// This function is specialized for GitHub API calls only. +// +// When gh CLI fails due to missing/invalid authentication, this function will attempt +// to make the same API call using direct HTTP REST API without authentication. +// +// Args: +// - apiPath: GitHub API path (e.g., "/repos/owner/repo/git/ref/tags/v1.0") +// - jqFilter: Optional jq filter for JSON extraction (e.g., ".object.sha"), empty string if not needed +// +// Returns: +// - output: The command output (either from gh CLI or REST API) +// - fromREST: true if the output came from REST API fallback, false if from gh CLI +// - err: Error if both gh CLI and REST API failed +// +// Usage: +// +// output, fromREST, err := ExecGHAPIWithRESTFallback("/repos/actions/checkout/git/ref/tags/v4", ".object.sha") +func ExecGHAPIWithRESTFallback(apiPath, jqFilter string) ([]byte, bool, error) { + // Build gh api command arguments + args := []string{"api", apiPath} + if jqFilter != "" { + args = append(args, "--jq", jqFilter) + } + + // First try with gh CLI + cmd := ExecGH(args...) + output, err := cmd.CombinedOutput() + + if err == nil { + ghHelperLog.Printf("gh api succeeded") + return output, false, nil + } + + // Check if error is authentication-related + exitErr, ok := err.(*exec.ExitError) + if !ok { + ghHelperLog.Printf("Not an exit error, returning original error") + return nil, false, err + } + + // Common authentication error exit codes: + // - exit status 4: authentication error in gh CLI + // - exit status 1: general error (could be auth-related) + // Check both combined output and stderr for error messages + combinedOutput := string(output) + stderr := string(exitErr.Stderr) + isAuthError := exitErr.ExitCode() == 4 || + strings.Contains(combinedOutput, "GH_TOKEN") || + strings.Contains(combinedOutput, "authentication") || + strings.Contains(stderr, "authentication") || + strings.Contains(combinedOutput, "HTTP 401") || + strings.Contains(stderr, "HTTP 401") || + strings.Contains(combinedOutput, "HTTP 403") || + strings.Contains(stderr, "HTTP 403") || + strings.Contains(strings.ToLower(combinedOutput), "unauthorized") || + strings.Contains(strings.ToLower(stderr), "unauthorized") + + if !isAuthError { + ghHelperLog.Printf("Error is not authentication-related, returning original error: %v", combinedOutput) + return nil, false, exitErr + } + + ghHelperLog.Printf("Authentication error detected, attempting REST API fallback") + + // Attempt REST API call + restOutput, err := callGitHubRESTAPI(apiPath, jqFilter) + if err != nil { + ghHelperLog.Printf("REST API fallback failed: %v", err) + // Return original gh CLI error since REST API also failed + return nil, false, exitErr + } + + ghHelperLog.Printf("REST API fallback succeeded") + return restOutput, true, nil +} + +// callGitHubRESTAPI makes a direct HTTP call to the GitHub REST API without authentication. +// It handles jq filtering by parsing the JSON response and extracting the specified field. +func callGitHubRESTAPI(apiPath, jqFilter string) ([]byte, error) { + // Normalize API path (remove leading slash if present) + apiPath = strings.TrimPrefix(apiPath, "/") + + // Build API URL + baseURL := "https://api.github.com" + url := fmt.Sprintf("%s/%s", baseURL, apiPath) + + ghHelperLog.Printf("Making REST API call to: %s", url) + + // Create HTTP client with timeout + client := &http.Client{ + Timeout: 30 * time.Second, + } + + // Create request + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + // Set headers + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "gh-aw") + + // Make request + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + // Check status code + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(body)) + } + + // Read response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response: %w", err) + } + + // If jq filter is specified, extract the field + if jqFilter != "" { + filtered, err := applyJQFilter(body, jqFilter) + if err != nil { + return nil, fmt.Errorf("failed to apply jq filter %s: %w", jqFilter, err) + } + return filtered, nil + } + + return body, nil +} + +// applyJQFilter applies a simple jq-like filter to JSON data. +// It only supports simple field extraction like ".object.sha" or ".name" +func applyJQFilter(jsonData []byte, filter string) ([]byte, error) { + // Parse filter (e.g., ".object.sha" -> ["object", "sha"]) + filter = strings.TrimPrefix(filter, ".") + fields := strings.Split(filter, ".") + + // Parse JSON + var data interface{} + if err := json.Unmarshal(jsonData, &data); err != nil { + return nil, fmt.Errorf("invalid JSON: %w", err) + } + + // Navigate through fields + current := data + for _, field := range fields { + if field == "" { + continue + } + + m, ok := current.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("cannot access field %s: not an object", field) + } + + value, exists := m[field] + if !exists { + return nil, fmt.Errorf("field %s not found", field) + } + + current = value + } + + // Convert result to string + switch v := current.(type) { + case string: + return []byte(v + "\n"), nil + case float64: + return []byte(fmt.Sprintf("%v\n", v)), nil + case bool: + return []byte(fmt.Sprintf("%v\n", v)), nil + default: + // For complex types, return JSON + result, err := json.Marshal(v) + if err != nil { + return nil, fmt.Errorf("failed to marshal result: %w", err) + } + return append(result, '\n'), nil + } +} diff --git a/pkg/workflow/gh_helper_integration_test.go b/pkg/workflow/gh_helper_integration_test.go new file mode 100644 index 00000000000..f4813099f40 --- /dev/null +++ b/pkg/workflow/gh_helper_integration_test.go @@ -0,0 +1,218 @@ +// go:build integration +//go:build integration + +package workflow + +import ( + "os" + "strings" + "testing" +) + +func TestExecGHAPIWithRESTFallback_RealAPI(t *testing.T) { + // Save original environment + originalGHToken := os.Getenv("GH_TOKEN") + originalGitHubToken := os.Getenv("GITHUB_TOKEN") + defer func() { + os.Setenv("GH_TOKEN", originalGHToken) + os.Setenv("GITHUB_TOKEN", originalGitHubToken) + }() + + // Clear tokens to force REST API fallback + os.Unsetenv("GH_TOKEN") + os.Unsetenv("GITHUB_TOKEN") + + t.Run("fallback to REST API for public repository tag", func(t *testing.T) { + // Test with a known public repository and tag + output, fromREST, err := ExecGHAPIWithRESTFallback( + "/repos/actions/checkout/git/ref/tags/v4", + ".object.sha", + ) + + if err != nil { + t.Fatalf("Expected fallback to succeed, got error: %v", err) + } + + if !fromREST { + t.Logf("gh CLI succeeded (gh is configured), but we expected REST fallback") + // This is OK - if gh CLI is configured, it will succeed before trying REST + // The important thing is that the command succeeded + } + + // Verify we got a valid SHA (40 hex characters) + sha := strings.TrimSpace(string(output)) + if len(sha) != 40 { + t.Errorf("Expected 40 character SHA, got: %s (length %d)", sha, len(sha)) + } + + // Verify it's all hex characters + for _, c := range sha { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + t.Errorf("Expected hex SHA, got character %c in: %s", c, sha) + break + } + } + + t.Logf("Successfully resolved SHA: %s (fromREST: %v)", sha, fromREST) + }) + + t.Run("fallback handles nested field extraction", func(t *testing.T) { + // Test with a repository info endpoint + output, fromREST, err := ExecGHAPIWithRESTFallback( + "/repos/actions/checkout", + ".name", + ) + + if err != nil { + t.Fatalf("Expected fallback to succeed, got error: %v", err) + } + + name := strings.TrimSpace(string(output)) + if name != "checkout" { + t.Errorf("Expected repository name 'checkout', got: %s", name) + } + + t.Logf("Successfully extracted name: %s (fromREST: %v)", name, fromREST) + }) + + t.Run("fallback returns error for non-existent repository", func(t *testing.T) { + output, fromREST, err := ExecGHAPIWithRESTFallback( + "/repos/nonexistent-owner-12345/nonexistent-repo-67890/git/ref/tags/v1.0", + ".object.sha", + ) + + // This should fail (either gh CLI or REST API) + if err == nil { + t.Errorf("Expected error for non-existent repository, but got output: %s", string(output)) + } + + t.Logf("Correctly failed for non-existent repository (fromREST: %v): %v", fromREST, err) + }) +} + +func TestActionResolver_WithRESTFallback(t *testing.T) { + // Save original environment + originalGHToken := os.Getenv("GH_TOKEN") + originalGitHubToken := os.Getenv("GITHUB_TOKEN") + defer func() { + os.Setenv("GH_TOKEN", originalGHToken) + os.Setenv("GITHUB_TOKEN", originalGitHubToken) + }() + + // Clear tokens to force REST API fallback + os.Unsetenv("GH_TOKEN") + os.Unsetenv("GITHUB_TOKEN") + + // Create a temporary directory for cache + tmpDir := t.TempDir() + cache := NewActionCache(tmpDir) + resolver := NewActionResolver(cache) + + t.Run("resolve action SHA using REST API fallback", func(t *testing.T) { + // Test resolving a real action + sha, err := resolver.ResolveSHA("actions/checkout", "v4") + + if err != nil { + t.Fatalf("Expected resolution to succeed via REST API fallback, got error: %v", err) + } + + // Verify we got a valid SHA + if len(sha) != 40 { + t.Errorf("Expected 40 character SHA, got: %s (length %d)", sha, len(sha)) + } + + t.Logf("Successfully resolved actions/checkout@v4 to SHA: %s", sha) + + // Verify caching works + cachedSHA, found := cache.Get("actions/checkout", "v4") + if !found { + t.Errorf("Expected SHA to be cached after resolution") + } + if cachedSHA != sha { + t.Errorf("Cached SHA %s doesn't match resolved SHA %s", cachedSHA, sha) + } + }) + + t.Run("resolve complex action path", func(t *testing.T) { + // Test with a complex action path (has subdirectory) + sha, err := resolver.ResolveSHA("github/codeql-action/upload-sarif", "v3") + + if err != nil { + t.Fatalf("Expected resolution to succeed, got error: %v", err) + } + + if len(sha) != 40 { + t.Errorf("Expected 40 character SHA, got: %s (length %d)", sha, len(sha)) + } + + t.Logf("Successfully resolved github/codeql-action/upload-sarif@v3 to SHA: %s", sha) + }) +} + +func TestCallGitHubRESTAPI_RealEndpoint(t *testing.T) { + t.Run("fetch repository info without authentication", func(t *testing.T) { + // Test direct REST API call to a public repository + output, err := callGitHubRESTAPI("/repos/actions/checkout", "") + + if err != nil { + t.Fatalf("Expected REST API call to succeed, got error: %v", err) + } + + // Verify we got JSON response + if len(output) == 0 { + t.Errorf("Expected non-empty response") + } + + // The response should contain repository information + if !strings.Contains(string(output), "checkout") { + t.Errorf("Expected response to contain 'checkout', got: %s", string(output[:100])) + } + + t.Logf("Successfully fetched repository info (%d bytes)", len(output)) + }) + + t.Run("fetch with jq filter", func(t *testing.T) { + output, err := callGitHubRESTAPI("/repos/actions/checkout", ".name") + + if err != nil { + t.Fatalf("Expected REST API call to succeed, got error: %v", err) + } + + name := strings.TrimSpace(string(output)) + if name != "checkout" { + t.Errorf("Expected 'checkout', got: %s", name) + } + + t.Logf("Successfully extracted name: %s", name) + }) + + t.Run("fetch nested field with jq filter", func(t *testing.T) { + output, err := callGitHubRESTAPI("/repos/actions/checkout/git/ref/tags/v4", ".object.sha") + + if err != nil { + t.Fatalf("Expected REST API call to succeed, got error: %v", err) + } + + sha := strings.TrimSpace(string(output)) + if len(sha) != 40 { + t.Errorf("Expected 40 character SHA, got: %s (length %d)", sha, len(sha)) + } + + t.Logf("Successfully extracted SHA: %s", sha) + }) + + t.Run("handle 404 error gracefully", func(t *testing.T) { + _, err := callGitHubRESTAPI("/repos/nonexistent-owner/nonexistent-repo", "") + + if err == nil { + t.Errorf("Expected error for non-existent repository") + } + + // Verify error message contains status code + if !strings.Contains(err.Error(), "404") { + t.Errorf("Expected error to mention 404, got: %v", err) + } + + t.Logf("Correctly handled 404 error: %v", err) + }) +} diff --git a/pkg/workflow/gh_helper_test.go b/pkg/workflow/gh_helper_test.go index 7a59fe8babe..1c4e842f243 100644 --- a/pkg/workflow/gh_helper_test.go +++ b/pkg/workflow/gh_helper_test.go @@ -147,3 +147,139 @@ func TestExecGHWithMultipleArgs(t *testing.T) { t.Errorf("Expected environment to contain GH_TOKEN=test-token") } } + +func TestApplyJQFilter(t *testing.T) { + tests := []struct { + name string + jsonData string + filter string + expected string + expectError bool + }{ + { + name: "simple field extraction", + jsonData: `{"name": "test"}`, + filter: ".name", + expected: "test\n", + }, + { + name: "nested field extraction", + jsonData: `{"object": {"sha": "abc123"}}`, + filter: ".object.sha", + expected: "abc123\n", + }, + { + name: "number field", + jsonData: `{"count": 42}`, + filter: ".count", + expected: "42\n", + }, + { + name: "boolean field", + jsonData: `{"enabled": true}`, + filter: ".enabled", + expected: "true\n", + }, + { + name: "missing field", + jsonData: `{"name": "test"}`, + filter: ".missing", + expectError: true, + }, + { + name: "invalid JSON", + jsonData: `{invalid}`, + filter: ".name", + expectError: true, + }, + { + name: "complex object", + jsonData: `{"meta": {"nested": {"value": "deep"}}}`, + filter: ".meta.nested.value", + expected: "deep\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := applyJQFilter([]byte(tt.jsonData), tt.filter) + + if tt.expectError { + if err == nil { + t.Errorf("Expected error but got none") + } + return + } + + if err != nil { + t.Errorf("Unexpected error: %v", err) + return + } + + if string(result) != tt.expected { + t.Errorf("Expected %q, got %q", tt.expected, string(result)) + } + }) + } +} + +func TestCallGitHubRESTAPI(t *testing.T) { + // This is a unit test that verifies the function exists and has the right signature + // Integration tests would require network access + + t.Run("function exists and accepts correct parameters", func(t *testing.T) { + // Test that the function can be called (will fail due to network, but that's expected) + _, err := callGitHubRESTAPI("/repos/actions/checkout/git/ref/tags/v4", ".object.sha") + + // We expect an error (network error in test environment), but we're just verifying + // the function signature is correct + if err == nil { + // If somehow this succeeds (network is available), that's fine too + t.Log("REST API call succeeded (network available)") + } else { + // Expected: network error or API error + t.Logf("Expected network error in test environment: %v", err) + } + }) +} + +func TestExecGHAPIWithRESTFallback_BasicFunctionality(t *testing.T) { + // Save original environment + originalGHToken := os.Getenv("GH_TOKEN") + originalGitHubToken := os.Getenv("GITHUB_TOKEN") + defer func() { + os.Setenv("GH_TOKEN", originalGHToken) + os.Setenv("GITHUB_TOKEN", originalGitHubToken) + }() + + // Clear tokens to ensure gh CLI will fail + os.Unsetenv("GH_TOKEN") + os.Unsetenv("GITHUB_TOKEN") + + t.Run("function accepts API path and jq filter", func(t *testing.T) { + // The function should accept the proper parameters + // Network will likely fail in test environment, but we're testing the signature + _, fromREST, err := ExecGHAPIWithRESTFallback("/repos/actions/checkout", ".name") + + if err == nil { + t.Logf("Command succeeded (gh is configured or network is available)") + return + } + + // We expect either gh CLI error or REST API error, but the function call should work + t.Logf("Expected error in test environment (fromREST: %v): %v", fromREST, err) + }) + + t.Run("function works without jq filter", func(t *testing.T) { + // Test with empty jq filter + _, fromREST, err := ExecGHAPIWithRESTFallback("/repos/actions/checkout", "") + + if err == nil { + t.Logf("Command succeeded (gh is configured or network is available)") + return + } + + // We expect either gh CLI error or REST API error + t.Logf("Expected error in test environment (fromREST: %v): %v", fromREST, err) + }) +}