Generate init/upgrade dispatcher file lists from github/gh-aw with embedded fallback

pkg/cli/copilot_agents.go

@@ -1,8 +1,13 @@
 package cli
 
 import (
+	"context"
 	_ "embed"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"net"
+	"net/http"
 	"os"
 	"path/filepath"
 	"sort"
@@ -17,13 +22,19 @@ import (
 var copilotAgentsLog = logger.New("cli:copilot_agents")
 
 const agenticWorkflowsSkillFileListPlaceholder = "{{AW_FILE_LIST}}"
+const ghAWMarkdownFilesAPIURL = "https://api.github.com/repos/github/gh-aw/contents/.github/aw?ref=main"
 
 //go:embed data/agentic_workflows_agent.md
 var agenticWorkflowsAgentTemplate string
 
 //go:embed data/agentic_workflows_skill.md
 var agenticWorkflowsSkillTemplate string
 
+//go:embed data/agentic_workflows_fallback_aw_files.json
+var agenticWorkflowsFallbackAWFiles string
+
+var listAgenticWorkflowsMarkdownFiles = fetchAgenticWorkflowsMarkdownFiles
+
 // ensureAgenticWorkflowsDispatcher ensures that .github/skills/agentic-workflows/SKILL.md
 // exists and contains the routing instructions loaded by the Agentic Workflows agent.
 func ensureAgenticWorkflowsDispatcher(verbose bool, skipInstructions bool) error {
@@ -47,7 +58,7 @@ func ensureAgenticWorkflowsDispatcher(verbose bool, skipInstructions bool) error
 		return fmt.Errorf("failed to create .github/skills/agentic-workflows directory: %w", err)
 	}
 
-	skillContent, err := buildAgenticWorkflowsSkillContent(gitRoot)
+	skillContent, err := buildAgenticWorkflowsSkillContent()
 	if err != nil {
 		copilotAgentsLog.Printf("Failed to build dispatcher skill: %v", err)
 		return fmt.Errorf("failed to build dispatcher skill: %w", err)
@@ -148,28 +159,15 @@ func buildAgenticWorkflowsAgentContent(gitRoot string) (string, error) {
 	return agenticWorkflowsAgentTemplate, nil
 }
 
-func buildAgenticWorkflowsSkillContent(gitRoot string) (string, error) {
-	awRoot := filepath.Join(gitRoot, ".github", "aw")
-	entries, err := os.ReadDir(awRoot)
+func buildAgenticWorkflowsSkillContent() (string, error) {
+	awFiles, err := listAgenticWorkflowsMarkdownFiles(context.Background())
 	if err != nil {
-		if os.IsNotExist(err) {
-			// No .github/aw directory yet — emit a minimal skill without the file list.
-			return strings.Replace(agenticWorkflowsSkillTemplate, agenticWorkflowsSkillFileListPlaceholder, "", 1), nil
-		}
-		return "", fmt.Errorf("failed to read .github/aw directory for skill generation (%s): %w", awRoot, err)
-	}
-
-	awFiles := make([]string, 0, len(entries))
-	for _, entry := range entries {
-		if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".md") {
-			continue
-		}
-		awFiles = append(awFiles, entry.Name())
+		fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to fetch .github/aw markdown file list from github/gh-aw: %v. Falling back to embedded list.", err)))
+		awFiles = embeddedFallbackAWMarkdownFiles()
 	}
 	sort.Strings(awFiles)
-
 	if len(awFiles) == 0 {
-		return "", fmt.Errorf("no markdown files found in %s - ensure .github/aw contains workflow documentation files", awRoot)
+		return "", errors.New("no .github/aw markdown files available from remote or embedded fallback")
 	}
 
 	var fileList strings.Builder
@@ -184,6 +182,65 @@ func buildAgenticWorkflowsSkillContent(gitRoot string) (string, error) {
 	return strings.Replace(agenticWorkflowsSkillTemplate, agenticWorkflowsSkillFileListPlaceholder, fileList.String(), 1), nil
 }
 
+type gitHubRepositoryContentEntry struct {
+	Name string `json:"name"`
+	Type string `json:"type"`
+}
+
+func fetchAgenticWorkflowsMarkdownFiles(ctx context.Context) ([]string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ghAWMarkdownFilesAPIURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build github API request: %w", err)
+	}
+	req.Header.Set("Accept", "application/vnd.github+json")
+	req.Header.Set("User-Agent", "gh-aw")
+
+	client := &http.Client{Timeout: constants.DefaultHTTPClientTimeout}
+	resp, err := client.Do(req)
+	if err != nil {
+		var netErr net.Error
+		if errors.As(err, &netErr) && netErr.Timeout() {
+			return nil, fmt.Errorf("github API request timed out after %s: %w", constants.DefaultHTTPClientTimeout, err)
+		}
+		return nil, fmt.Errorf("github API request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("github API returned %s", resp.Status)
+	}
+
+	var entries []gitHubRepositoryContentEntry
+	if err := json.NewDecoder(resp.Body).Decode(&entries); err != nil {
+		return nil, fmt.Errorf("failed to decode github API response: %w", err)
+	}
+
+	awFiles := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		if entry.Type != "file" || !strings.HasSuffix(entry.Name, ".md") {
+			continue
+		}
+		awFiles = append(awFiles, entry.Name)
+	}
+
+	if len(awFiles) == 0 {
+		return nil, errors.New("github API returned no markdown files")
+	}
+
+	sort.Strings(awFiles)
+	return awFiles, nil
+}
+
+func embeddedFallbackAWMarkdownFiles() []string {
+	var awFiles []string
+	if err := json.Unmarshal([]byte(agenticWorkflowsFallbackAWFiles), &awFiles); err != nil {
+		fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse embedded .github/aw fallback markdown file list: %v", err)))
+		return nil
+	}
+	sort.Strings(awFiles)
+	return awFiles
+}
+
 // cleanupOldPromptFile removes an old prompt file from .github/prompts/ if it exists
 func cleanupOldPromptFile(promptFileName string, verbose bool) error {
 	gitRoot, err := gitutil.FindGitRoot()

pkg/cli/copilot_agents_test.go

@@ -3,14 +3,18 @@
 package cli
 
 import (
+	"context"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
 	"testing"
 
 	"github.com/github/gh-aw/pkg/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // TestDeleteLegacyAgentFiles tests deletion of old agent files.
@@ -316,19 +320,9 @@ func TestCheckedInAgenticWorkflowsAgentMatchesGeneratedContent(t *testing.T) {
 }
 
 func TestBuildAgenticWorkflowsSkillContent(t *testing.T) {
-	tempDir := testutil.TempDir(t, "test-*")
-	awDir := filepath.Join(tempDir, ".github", "aw")
-	if err := os.MkdirAll(awDir, 0o755); err != nil {
-		t.Fatalf("Failed to create .github/aw directory: %v", err)
-	}
-
-	for _, name := range []string{"workflow-z.md", "workflow-a.md", "ignore.txt"} {
-		if err := os.WriteFile(filepath.Join(awDir, name), []byte("# test"), 0o644); err != nil {
-			t.Fatalf("Failed to create %s: %v", name, err)
-		}
-	}
+	withMockAWMarkdownFileList(t, []string{"workflow-z.md", "workflow-a.md"}, nil)
 
-	content, err := buildAgenticWorkflowsSkillContent(tempDir)
+	content, err := buildAgenticWorkflowsSkillContent()
 	if err != nil {
 		t.Fatalf("buildAgenticWorkflowsSkillContent() returned error: %v", err)
 	}
@@ -342,29 +336,39 @@ func TestBuildAgenticWorkflowsSkillContent(t *testing.T) {
 	if content != expected {
 		t.Fatalf("Expected exact skill content:\n%s\ngot:\n%s", expected, content)
 	}
-	if strings.Contains(content, "ignore.txt") {
-		t.Fatalf("expected non-markdown files to be excluded from generated skill content:\n%s", content)
-	}
 	if strings.Contains(content, ".github/agents/agentic-workflows") {
 		t.Fatalf("expected generated skill content to avoid agent cross-references:\n%s", content)
 	}
 }
 
 func TestBuildAgenticWorkflowsSkillContentWithoutAWDirectory(t *testing.T) {
-	tempDir := testutil.TempDir(t, "test-*")
+	withMockAWMarkdownFileList(t, []string{"workflow-a.md"}, nil)
 
-	content, err := buildAgenticWorkflowsSkillContent(tempDir)
+	content, err := buildAgenticWorkflowsSkillContent()
 	if err != nil {
 		t.Fatalf("buildAgenticWorkflowsSkillContent() returned error: %v", err)
 	}
 
-	expected := strings.Replace(agenticWorkflowsSkillTemplate, agenticWorkflowsSkillFileListPlaceholder, "", 1)
+	expected := strings.Replace(agenticWorkflowsSkillTemplate, agenticWorkflowsSkillFileListPlaceholder, "- `.github/aw/workflow-a.md`\n", 1)
 	if content != expected {
 		t.Fatalf("Expected exact skill content without .github/aw directory:\n%s\ngot:\n%s", expected, content)
 	}
 	if strings.Contains(content, agenticWorkflowsSkillFileListPlaceholder) {
 		t.Fatalf("expected generated skill content to replace the file-list placeholder:\n%s", content)
 	}
+	if !strings.Contains(content, "- `.github/aw/workflow-a.md`") {
+		t.Fatalf("expected generated skill content to include remotely sourced markdown files:\n%s", content)
+	}
+}
+
+func TestBuildAgenticWorkflowsSkillContentFallsBackToEmbeddedFileList(t *testing.T) {
+	withMockAWMarkdownFileList(t, nil, assert.AnError)
+
+	content, err := buildAgenticWorkflowsSkillContent()
+	require.NoError(t, err, "buildAgenticWorkflowsSkillContent() returned error")
+
+	assert.NotContains(t, content, agenticWorkflowsSkillFileListPlaceholder, "expected generated skill content to replace the file-list placeholder")
+	assert.Contains(t, content, "- `.github/aw/create-agentic-workflow.md`\n", "expected embedded fallback markdown file list to be used")
 }
 
 func TestCheckedInAgenticWorkflowsSkillMatchesGeneratedContent(t *testing.T) {
@@ -374,7 +378,19 @@ func TestCheckedInAgenticWorkflowsSkillMatchesGeneratedContent(t *testing.T) {
 	}
 
 	gitRoot := filepath.Clean(filepath.Join(filepath.Dir(file), "..", ".."))
-	expected, err := buildAgenticWorkflowsSkillContent(gitRoot)
+	awEntries, err := os.ReadDir(filepath.Join(gitRoot, ".github", "aw"))
+	require.NoError(t, err, "failed to read .github/aw for test fixture")
+	awFiles := make([]string, 0, len(awEntries))
+	for _, entry := range awEntries {
+		if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".md") {
+			continue
+		}
+		awFiles = append(awFiles, entry.Name())
+	}
+	sort.Strings(awFiles)
+	withMockAWMarkdownFileList(t, awFiles, nil)
+
+	expected, err := buildAgenticWorkflowsSkillContent()
 	if err != nil {
 		t.Fatalf("buildAgenticWorkflowsSkillContent() returned error: %v", err)
 	}
@@ -388,3 +404,15 @@ func TestCheckedInAgenticWorkflowsSkillMatchesGeneratedContent(t *testing.T) {
 		t.Fatalf("Checked-in skill file is out of sync with generated content\nexpected:\n%s\nactual:\n%s", expected, string(actual))
 	}
 }
+
+func withMockAWMarkdownFileList(t *testing.T, files []string, err error) {
+	t.Helper()
+	previous := listAgenticWorkflowsMarkdownFiles
+	listAgenticWorkflowsMarkdownFiles = func(context.Context) ([]string, error) {
+		// Return a copy so tests can't mutate shared backing arrays across invocations.
+		return append([]string(nil), files...), err
+	}
+	t.Cleanup(func() {
+		listAgenticWorkflowsMarkdownFiles = previous
+	})
+}

pkg/cli/data/agentic_workflows_fallback_aw_files.json

@@ -0,0 +1,48 @@
+[
+  "agentic-chat.md",
+  "asciicharts.md",
+  "campaign.md",
+  "charts-trending.md",
+  "charts.md",
+  "cli-commands.md",
+  "context.md",
+  "create-agentic-workflow.md",
+  "create-shared-agentic-workflow.md",
+  "debug-agentic-workflow.md",
+  "dependabot.md",
+  "deployment-status.md",
+  "experiments.md",
+  "github-agentic-workflows.md",
+  "github-mcp-server.md",
+  "llms.md",
+  "memory.md",
+  "messages.md",
+  "network.md",
+  "patterns.md",
+  "pr-reviewer.md",
+  "report.md",
+  "reuse.md",
+  "safe-outputs-automation.md",
+  "safe-outputs-content.md",
+  "safe-outputs-management.md",
+  "safe-outputs-runtime.md",
+  "safe-outputs.md",
+  "serena-tool.md",
+  "shared-safe-jobs.md",
+  "skills.md",
+  "subagents.md",
+  "syntax-agentic.md",
+  "syntax-core.md",
+  "syntax-tools-imports.md",
+  "syntax.md",
+  "test-coverage.md",
+  "test-expression.md",
+  "token-optimization.md",
+  "triggers.md",
+  "update-agentic-workflow.md",
+  "upgrade-agentic-workflows.md",
+  "visual-regression.md",
+  "workflow-constraints.md",
+  "workflow-editing.md",
+  "workflow-patterns.md"
+]

pkg/cli/upgrade_integration_test.go

@@ -3,7 +3,9 @@
 package cli
 
 import (
+	"os"
 	"os/exec"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -24,3 +26,57 @@ func TestUpgradeCommand_OnExistingRepository(t *testing.T) {
 	require.NoError(t, err, "upgrade command should succeed on existing repository, output: %s", outputStr)
 	assert.Contains(t, outputStr, "Upgrade complete", "Should report upgrade complete")
 }
+
+func TestInitAndUpgradeWithEmptyAWDirectory(t *testing.T) {
+	setup := setupIntegrationTest(t)
+	defer setup.cleanup()
+
+	initGit := exec.Command("git", "init", "--quiet")
+	initGit.Dir = setup.tempDir
+	require.NoError(t, initGit.Run(), "git init should succeed")
+
+	awDir := filepath.Join(setup.tempDir, ".github", "aw")
+	require.NoError(t, os.MkdirAll(filepath.Join(awDir, "logs"), 0o755), "should create .github/aw/logs")
+	require.NoError(t, os.WriteFile(filepath.Join(awDir, "actions-lock.json"), []byte("{}\n"), 0o644), "should create actions-lock.json")
+
+	initCmd := exec.Command(setup.binaryPath, "init")
+	initCmd.Dir = setup.tempDir
+	initOutput, initErr := initCmd.CombinedOutput()
+	require.NoError(t, initErr, "init command should succeed with empty .github/aw directory, output: %s", string(initOutput))
+	_, err := os.Stat(filepath.Join(awDir, "actions-lock.json"))
+	require.NoError(t, err, "expected actions-lock.json to be preserved after init")
+	_, err = os.Stat(filepath.Join(awDir, "logs"))
+	require.NoError(t, err, "expected .github/aw/logs to be preserved after init")
+
+	skillPath := filepath.Join(setup.tempDir, ".github", "skills", "agentic-workflows", "SKILL.md")
+	_, err = os.Stat(skillPath)
+	require.NoError(t, err, "expected dispatcher skill file to exist after init")
+
+	workflowPath := filepath.Join(setup.tempDir, ".github", "workflows", "example.md")
+	workflowContent := `---
+name: Example Agentic Workflow
+on:
+  workflow_dispatch:
+permissions:
+  contents: read
+  actions: read
+engine: copilot
+strict: true
+timeout-minutes: 5
+---
+
+Say hello.
+`
+	require.NoError(t, os.WriteFile(workflowPath, []byte(workflowContent), 0o644), "should create sample workflow")
+
+	upgradeCmd := exec.Command(setup.binaryPath, "upgrade", "--no-fix", "--skip-extension-upgrade")
+	upgradeCmd.Dir = setup.tempDir
+	upgradeOutput, upgradeErr := upgradeCmd.CombinedOutput()
+	upgradeOutputStr := string(upgradeOutput)
+	require.NoError(t, upgradeErr, "upgrade command should succeed with empty .github/aw directory, output: %s", upgradeOutputStr)
+	assert.Contains(t, upgradeOutputStr, "Upgrade complete", "Should report upgrade complete")
+	_, err = os.Stat(filepath.Join(awDir, "actions-lock.json"))
+	require.NoError(t, err, "expected actions-lock.json to be preserved after upgrade")
+	_, err = os.Stat(filepath.Join(awDir, "logs"))
+	require.NoError(t, err, "expected .github/aw/logs to be preserved after upgrade")
+}