diff --git a/.copier/workflows/main.yaml b/.copier/main.yaml similarity index 95% rename from .copier/workflows/main.yaml rename to .copier/main.yaml index 22f307d..0d201b4 100644 --- a/.copier/workflows/main.yaml +++ b/.copier/main.yaml @@ -29,7 +29,7 @@ workflow_configs: - source: "repo" repo: "mongodb/docs-sample-apps" branch: "main" # optional, defaults to main - path: ".copier/workflows.yaml" + path: ".copier/config.yaml" enabled: true # -------------------------------------------------------------------------- @@ -38,7 +38,7 @@ workflow_configs: - source: "repo" repo: "10gen/docs-mongodb-internal" branch: "main" - path: ".copier/workflows.yaml" + path: ".copier/config.yaml" enabled: true # -------------------------------------------------------------------------- @@ -47,7 +47,7 @@ workflow_configs: - source: "repo" repo: "mongodb/docs-code-examples" branch: "main" - path: ".copier/workflows.yaml" + path: ".copier/config.yaml" enabled: false # -------------------------------------------------------------------------- diff --git a/examples-copier/Dockerfile b/examples-copier/Dockerfile index c1e0ac5..2bedd4f 100644 --- a/examples-copier/Dockerfile +++ b/examples-copier/Dockerfile @@ -1,5 +1,5 @@ # Build stage -FROM golang:1.23.4-alpine AS builder +FROM golang:1.24.0-alpine AS builder # Install build dependencies RUN apk add --no-cache git ca-certificates diff --git a/examples-copier/QUICK-REFERENCE.md b/examples-copier/QUICK-REFERENCE.md index 8e8d289..eabfc2b 100644 --- a/examples-copier/QUICK-REFERENCE.md +++ b/examples-copier/QUICK-REFERENCE.md @@ -42,37 +42,43 @@ ## Configuration Patterns -### Prefix Pattern +### Move Transformation ```yaml -source_pattern: - type: "prefix" - pattern: "examples/go/" +transformations: + - move: + from: "examples/go" + to: "code/go" ``` -### Glob Pattern +### Glob Transformation ```yaml -source_pattern: - type: "glob" - pattern: "examples/*/main.go" +transformations: + - glob: + pattern: "examples/*/main.go" + transform: "code/${relative_path}" ``` -### Regex Pattern +### Regex Transformation ```yaml -source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P.+)$" +transformations: + - regex: + pattern: "^examples/(?P[^/]+)/(?P.+)$" + transform: "code/${lang}/${file}" ``` -### Pattern with Exclusions +### Workflow with Exclusions ```yaml -source_pattern: - type: "prefix" - pattern: "examples/" - exclude_patterns: - - "\.gitignore$" - - "node_modules/" - - "\.env$" - - "/dist/" +workflows: + - name: "Copy examples" + transformations: + - move: + from: "examples" + to: "code" + exclude: + - "**/.gitignore" + - "**/node_modules/**" + - "**/.env" + - "**/dist/**" ``` ## Path Transformations @@ -118,36 +124,22 @@ commit_strategy: auto_merge: true ``` -### Batch PRs by Repository -```yaml -batch_by_repo: true - -batch_pr_config: - pr_title: "Update from ${source_repo}" - pr_body: | - 🤖 Automated update - Files: ${file_count} - use_pr_template: true - commit_message: "Update from ${source_repo} PR #${pr_number}" -``` - ## Advanced Features ### Exclude Patterns -Exclude unwanted files from being copied: +Exclude unwanted files from being copied at the workflow level: ```yaml -source_pattern: - type: "prefix" - pattern: "examples/" - exclude_patterns: - - "\.gitignore$" # Exclude .gitignore - - "node_modules/" # Exclude dependencies - - "\.env$" # Exclude .env files - - "\.env\\..*$" # Exclude .env.local, .env.production, etc. - - "/dist/" # Exclude build output - - "/build/" # Exclude build artifacts - - "\.test\.(js|ts)$" # Exclude test files +workflows: + - name: "Copy examples" + exclude: + - "**/.gitignore" # Exclude .gitignore + - "**/node_modules/**" # Exclude dependencies + - "**/.env" # Exclude .env files + - "**/.env.*" # Exclude .env.local, .env.production, etc. + - "**/dist/**" # Exclude build output + - "**/build/**" # Exclude build artifacts + - "**/*.test.js" # Exclude test files ``` ### PR Template Integration @@ -167,20 +159,6 @@ commit_strategy: 2. Separator (`---`) 3. Your configured content (automation info) -### Batch Configuration -When `batch_by_repo: true`, use `batch_pr_config` for accurate file counts: - -```yaml -batch_by_repo: true - -batch_pr_config: - pr_title: "Update from ${source_repo}" - pr_body: | - Files: ${file_count} # Accurate count across all rules - Source: ${source_repo} PR #${pr_number} - use_pr_template: true -``` - ## Message Templates ### Available Variables @@ -385,44 +363,60 @@ go build -o test-webhook ./cmd/test-webhook ### Copy All Go Files ```yaml -source_pattern: - type: "regex" - pattern: "^examples/.*\\.go$" -targets: - - repo: "org/docs" - path_transform: "code/${path}" +workflows: + - name: "Copy Go files" + source: + repo: "org/source" + branch: "main" + destination: + repo: "org/docs" + branch: "main" + transformations: + - regex: + pattern: "^examples/.*\\.go$" + transform: "code/${path}" ``` ### Organize by Language ```yaml -source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P.+)$" -targets: - - repo: "org/docs" - path_transform: "languages/${lang}/${rest}" +workflows: + - name: "Organize by language" + transformations: + - regex: + pattern: "^examples/(?P[^/]+)/(?P.+)$" + transform: "languages/${lang}/${rest}" ``` -### Multiple Targets with Different Transforms +### Multiple Workflows for Different Destinations ```yaml -source_pattern: - type: "prefix" - pattern: "examples/" -targets: - - repo: "org/docs-v1" - path_transform: "examples/${path}" - - repo: "org/docs-v2" - path_transform: "code-samples/${path}" +workflows: + - name: "Copy to docs-v1" + destination: + repo: "org/docs-v1" + branch: "main" + transformations: + - move: + from: "examples" + to: "examples" + + - name: "Copy to docs-v2" + destination: + repo: "org/docs-v2" + branch: "main" + transformations: + - move: + from: "examples" + to: "code-samples" ``` ### Conditional Copying (by file type) ```yaml -source_pattern: - type: "regex" - pattern: "^examples/.*\\.(?Pgo|py|js)$" -targets: - - repo: "org/docs" - path_transform: "code/${ext}/${filename}" +workflows: + - name: "Copy by file type" + transformations: + - regex: + pattern: "^examples/.*\\.(?Pgo|py|js)$" + transform: "code/${ext}/${filename}" ``` ## Troubleshooting diff --git a/examples-copier/README.md b/examples-copier/README.md index f9e7840..4d2b29b 100644 --- a/examples-copier/README.md +++ b/examples-copier/README.md @@ -1,25 +1,26 @@ # GitHub Docs Code Example Copier -A GitHub app that automatically copies code examples and files from a source repository to one or more target -repositories when pull requests are merged. Features advanced pattern matching, path transformations, audit logging, -and comprehensive monitoring. +A GitHub app that automatically copies code examples and files from source repositories to target repositories when pull requests are merged. Features centralized configuration with distributed workflow management, $ref support for reusable components, advanced pattern matching, and comprehensive monitoring. ## Features ### Core Functionality +- **Main Config System** - Centralized configuration with distributed workflow management +- **Source Context Inference** - Workflows automatically inherit source repo/branch +- **$ref Support** - Reusable components for transformations, strategies, and excludes +- **Resilient Loading** - Continues processing when individual configs fail (logs warnings) - **Automated File Copying** - Copies files from source to target repos on PR merge - **Advanced Pattern Matching** - Prefix, glob, and regex patterns with variable extraction - **Path Transformations** - Template-based path transformations with variable substitution -- **Multiple Targets** - Copy files to multiple repositories and branches - **Flexible Commit Strategies** - Direct commits or pull requests with auto-merge - **Deprecation Tracking** - Automatic tracking of deleted files ### Enhanced Features -- **YAML Configuration** - Modern YAML config with JSON backward compatibility +- **Workflow References** - Local, remote (repo), or inline workflow configs +- **Default Precedence** - Workflow > Workflow config > Main config > System defaults - **Message Templating** - Template-ized commit messages and PR titles -- **Batch PR Support** - Combine multiple rules into one PR per target repo - **PR Template Integration** - Fetch and merge PR templates from target repos -- **File Exclusion** - Exclude patterns to filter out unwanted files (`.gitignore`, `node_modules`, etc.) +- **File Exclusion** - Exclude patterns to filter out unwanted files - **Audit Logging** - MongoDB-based event tracking for all operations - **Health & Metrics** - `/health` and `/metrics` endpoints for monitoring - **Development Tools** - Dry-run mode, CLI validation, enhanced logging @@ -56,87 +57,80 @@ go build -o config-validator ./cmd/config-validator 1. **Copy environment example file** ```bash -# For local development -cp configs/.env.local.example configs/.env - -# Or for YAML-based configuration -cp configs/env.yaml.example env.yaml +cp env.yaml.example env.yaml ``` 2. **Set required environment variables** -```bash +```yaml # GitHub Configuration -REPO_OWNER=your-org -REPO_NAME=your-repo -SRC_BRANCH=main -GITHUB_APP_ID=123456 -GITHUB_INSTALLATION_ID=789012 +GITHUB_APP_ID: "123456" +INSTALLATION_ID: "789012" # Optional fallback -# Google Cloud -GCP_PROJECT_ID=your-project -PEM_KEY_NAME=projects/123/secrets//versions/latest -WEBHOOK_SECRET_NAME=projects/123/secrets/webhook-secret +# Config Repository (where main config lives) +CONFIG_REPO_OWNER: "your-org" +CONFIG_REPO_NAME: "config-repo" +CONFIG_REPO_BRANCH: "main" + +# Main Config +MAIN_CONFIG_FILE: ".copier/workflows/main.yaml" +USE_MAIN_CONFIG: "true" + +# Secret Manager References +GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/.../secrets/PEM/versions/latest" +WEBHOOK_SECRET_NAME: "projects/.../secrets/webhook-secret/versions/latest" # Application Settings -PORT=8080 -CONFIG_FILE=copier-config.yaml -DEPRECATION_FILE=deprecated_examples.json +WEBSERVER_PATH: "/events" +DEPRECATION_FILE: "deprecated_examples.json" +COMMITTER_NAME: "GitHub Copier App" +COMMITTER_EMAIL: "bot@mongodb.com" + +# Feature Flags +AUDIT_ENABLED: "false" +METRICS_ENABLED: "true" +``` -# Optional: MongoDB Audit Logging -AUDIT_ENABLED=true -MONGO_URI=mongodb+srv://user:pass@cluster.mongodb.net -AUDIT_DATABASE=code_copier -AUDIT_COLLECTION=audit_events +3. **Create main configuration file** -# Optional: Development Features -DRY_RUN=false -METRICS_ENABLED=true +Create `.copier/workflows/main.yaml` in your config repository: + +```yaml +# Main config with global defaults and workflow references +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + exclude: + - "**/.env" + - "**/node_modules/**" + +workflow_configs: + # Reference workflows in source repo + - source: "repo" + repo: "your-org/source-repo" + branch: "main" + path: ".copier/workflows/config.yaml" + enabled: true ``` -3. **Create configuration file** +4. **Create workflow config in source repository** -Create `copier-config.yaml` in your source repository: +Create `.copier/workflows/config.yaml` in your source repository: ```yaml -source_repo: "your-org/source-repo" -source_branch: "main" -batch_by_repo: true # Optional: batch all changes into one PR per target repo - -# Optional: Customize batched PR metadata -batch_pr_config: - pr_title: "Update code examples from ${source_repo}" - pr_body: | - 🤖 Automated update of code examples - - **Source:** ${source_repo} PR #${pr_number} - **Files:** ${file_count} - use_pr_template: true # Fetch PR template from target repos - commit_message: "Update examples from ${source_repo} PR #${pr_number}" - -copy_rules: - - name: "Copy Go examples" - source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$" - exclude_patterns: # Optional: exclude unwanted files - - "\.gitignore$" - - "node_modules/" - - "\.env$" - targets: - - repo: "your-org/target-repo" - branch: "main" - path_transform: "docs/examples/${lang}/${category}/${file}" - commit_strategy: - type: "pull_request" - commit_message: "Update ${category} examples from ${lang}" - pr_title: "Update ${category} examples" - pr_body: "Automated update of ${lang} examples" - use_pr_template: true # Merge with target repo's PR template - auto_merge: false - deprecation_check: - enabled: true - file: "deprecated_examples.json" +workflows: + - name: "copy-examples" + # source.repo and source.branch inherited from workflow config reference + destination: + repo: "your-org/target-repo" + branch: "main" + transformations: + - move: { from: "examples", to: "docs/examples" } + commit_strategy: + type: "pull_request" + pr_title: "Update code examples" + use_pr_template: true ``` ### Running the Application @@ -157,40 +151,65 @@ copy_rules: ## Configuration -### Pattern Types +See [MAIN-CONFIG-README.md](configs/copier-config-examples/MAIN-CONFIG-README.md) for complete configuration documentation. + +### Main Config Structure + +The application uses a three-tier configuration system: + +1. **Main Config** - Centralized defaults and workflow references +2. **Workflow Configs** - Collections of workflows (local, remote, or inline) +3. **Individual Workflows** - Specific source → destination mappings + +### Transformation Types -#### Prefix Pattern -Simple string prefix matching: +#### Move Transformation +Move files from one directory to another: ```yaml -source_pattern: - type: "prefix" - pattern: "examples/go/" +transformations: + - move: + from: "examples/go" + to: "code/go" ``` -Matches: `examples/go/main.go`, `examples/go/database/connect.go` +Moves: `examples/go/main.go` → `code/go/main.go` -#### Glob Pattern -Wildcard matching with `*` and `?`: +#### Copy Transformation +Copy a single file to a new location: ```yaml -source_pattern: - type: "glob" - pattern: "examples/*/main.go" +transformations: + - copy: + from: "README.md" + to: "docs/README.md" ``` -Matches: `examples/go/main.go`, `examples/python/main.go` +Copies: `README.md` → `docs/README.md` -#### Regex Pattern +#### Glob Transformation +Wildcard matching with path transformation: + +```yaml +transformations: + - glob: + pattern: "examples/*/main.go" + transform: "code/${relative_path}" +``` + +Matches: `examples/go/main.go` → `code/examples/go/main.go` + +#### Regex Transformation Full regex with named capture groups: ```yaml -source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P.+)$" +transformations: + - regex: + pattern: "^examples/(?P[^/]+)/(?P.+)$" + transform: "code/${lang}/${file}" ``` -Matches: `examples/go/main.go` (extracts `lang=go`, `file=main.go`) +Matches: `examples/go/main.go` → `code/go/main.go` (extracts `lang=go`, `file=main.go`) ### Path Transformations @@ -232,28 +251,40 @@ commit_strategy: ### Advanced Features -#### Batch PRs by Repository +#### $ref Support for Reusable Components -Combine all changes from a single source PR into one PR per target repository: +Extract common configurations into separate files: ```yaml -batch_by_repo: true +# Workflow config +workflows: + - name: "mflix-java" + destination: + repo: "mongodb/sample-app-java-mflix" + branch: "main" + transformations: + $ref: "../transformations/mflix-java.yaml" + commit_strategy: + $ref: "../strategies/mflix-pr-strategy.yaml" + exclude: + $ref: "../common/mflix-excludes.yaml" +``` -batch_pr_config: - pr_title: "Update code examples from ${source_repo}" - pr_body: | - 🤖 Automated update +#### Source Context Inference - Files: ${file_count} - Source: ${source_repo} PR #${pr_number} - use_pr_template: true - commit_message: "Update from ${source_repo} PR #${pr_number}" -``` +Workflows automatically inherit source repo/branch from workflow config reference: -**Benefits:** -- Single PR per target repo instead of multiple PRs -- Accurate `${file_count}` across all matched rules -- Easier review process for related changes +```yaml +# No need to specify source.repo and source.branch! +workflows: + - name: "my-workflow" + # source.repo and source.branch inherited automatically + destination: + repo: "mongodb/dest-repo" + branch: "main" + transformations: + - move: { from: "src", to: "dest" } +``` #### PR Template Integration @@ -262,34 +293,21 @@ Automatically fetch and merge PR templates from target repositories: ```yaml commit_strategy: type: "pull_request" - pr_body: | - 🤖 Automated update - Files: ${file_count} + pr_body: "🤖 Automated update" use_pr_template: true # Fetches .github/pull_request_template.md ``` -**Result:** PR description shows the target repo's template first (with checklists and guidelines), followed by your configured content. +#### File Exclusion -#### File Exclusion Patterns - -Exclude unwanted files from being copied: +Exclude unwanted files at the workflow or workflow config level: ```yaml -source_pattern: - type: "prefix" - pattern: "examples/" - exclude_patterns: - - "\.gitignore$" # Exclude .gitignore files - - "node_modules/" # Exclude dependencies - - "\.env$" # Exclude environment files - - "/dist/" # Exclude build artifacts - - "\.test\.(js|ts)$" # Exclude test files -``` - -**Use cases:** -- Filter out configuration files -- Exclude build artifacts and dependencies -- Skip test files or documentation +exclude: + - "**/.gitignore" + - "**/node_modules/**" + - "**/.env" + - "**/dist/**" +``` ### Message Templates @@ -517,19 +535,20 @@ container := NewServiceContainer(config) ## Deployment -See [DEPLOYMENT.md](./docs/DEPLOYMENT.md) for complete deployment guide for step-by-step checklist. +See [DEPLOYMENT.md](./docs/DEPLOYMENT.md) for complete deployment guide. -### Google Cloud App Engine +### Google Cloud Run ```bash -gcloud app deploy +cd examples-copier +./scripts/deploy-cloudrun.sh ``` ### Docker ```bash docker build -t examples-copier . -docker run -p 8080:8080 --env-file .env examples-copier +docker run -p 8080:8080 --env-file env.yaml examples-copier ``` ## Security @@ -543,7 +562,8 @@ docker run -p 8080:8080 --env-file .env examples-copier ### Getting Started -- **[Configuration Guide](docs/CONFIGURATION-GUIDE.md)** - Complete configuration reference +- **[Main Config README](configs/copier-config-examples/MAIN-CONFIG-README.md)** - Complete main config documentation +- **[Quick Start Guide](configs/copier-config-examples/QUICK-START-MAIN-CONFIG.md)** - Get started in 5 minutes - **[Pattern Matching Guide](docs/PATTERN-MATCHING-GUIDE.md)** - Pattern matching with examples - **[Local Testing](docs/LOCAL-TESTING.md)** - Test locally before deploying - **[Deployment Guide](docs/DEPLOYMENT.md)** - Deploy to production @@ -553,7 +573,6 @@ docker run -p 8080:8080 --env-file .env examples-copier - **[Architecture](docs/ARCHITECTURE.md)** - System design and components - **[Troubleshooting](docs/TROUBLESHOOTING.md)** - Common issues and solutions - **[FAQ](docs/FAQ.md)** - Frequently asked questions -- **[Debug Logging](docs/DEBUG-LOGGING.md)** - Debug logging configuration - **[Deprecation Tracking](docs/DEPRECATION-TRACKING-EXPLAINED.md)** - How deprecation tracking works ### Features @@ -563,6 +582,4 @@ docker run -p 8080:8080 --env-file .env examples-copier ### Tools -- **[config-validator](cmd/config-validator/README.md)** - Validate and test configurations -- **[test-webhook](cmd/test-webhook/README.md)** - Test webhook processing -- **[Scripts](scripts/README.md)** - Helper scripts +- **[Scripts](scripts/README.md)** - Helper scripts for deployment and testing diff --git a/examples-copier/cmd/config-validator/README.md b/examples-copier/cmd/config-validator/README.md index 6330ef8..b1ae4d4 100644 --- a/examples-copier/cmd/config-validator/README.md +++ b/examples-copier/cmd/config-validator/README.md @@ -1,14 +1,15 @@ # config-validator -Command-line tool for validating and testing examples-copier configurations. +Command-line tool for validating and testing examples-copier workflow configurations. + +> **Note:** This tool validates individual workflow config files. It does not validate main config files. Main config validation is built into the application itself. ## Overview The `config-validator` tool helps you: -- Validate configuration files +- Validate workflow configuration files - Test pattern matching - Test path transformations -- Convert legacy JSON configs to YAML - Debug configuration issues ## Installation @@ -36,13 +37,13 @@ Validate a configuration file. **Examples:** ```bash -# Validate YAML config -./config-validator validate -config copier-config.yaml +# Validate workflow config +./config-validator validate -config .copier/workflows/config.yaml # Validate with verbose output -./config-validator validate -config copier-config.yaml -v +./config-validator validate -config .copier/workflows/config.yaml -v -# Validate JSON config +# Validate legacy JSON config ./config-validator validate -config config.json ``` @@ -176,38 +177,6 @@ Variables used: name = main ``` -### convert - -Convert legacy JSON configuration to YAML format. - -**Usage:** -```bash -./config-validator convert -input -output -``` - -**Options:** -- `-input` - Input JSON file (required) -- `-output` - Output YAML file (required) - -**Example:** - -```bash -./config-validator convert -input config.json -output copier-config.yaml -``` - -**Output:** -``` -✅ Conversion successful! - -Converted 2 legacy rules to YAML format. -Output written to: copier-config.yaml - -Next steps: -1. Review the generated copier-config.yaml -2. Enhance with new features (regex patterns, path transforms) -3. Validate: ./config-validator validate -config copier-config.yaml -``` - ## Common Use Cases ### Debugging Pattern Matching @@ -250,8 +219,8 @@ When files are copied to wrong locations: Before deploying a new configuration: ```bash -# Validate the config -./config-validator validate -config copier-config.yaml -v +# Validate the workflow config +./config-validator validate -config .copier/workflows/config.yaml -v # Test with sample file paths ./config-validator test-pattern \ @@ -269,11 +238,8 @@ Before deploying a new configuration: ### Migrating from JSON to YAML ```bash -# Convert -./config-validator convert -input config.json -output copier-config.yaml - # Validate -./config-validator validate -config copier-config.yaml -v +./config-validator validate -config workflow-config.yaml -v # Test patterns ./config-validator test-pattern \ @@ -304,24 +270,24 @@ Before deploying a new configuration: ### Complete Workflow ```bash -# 1. Create config -cat > copier-config.yaml << EOF -source_repo: "myorg/source-repo" -source_branch: "main" - -copy_rules: +# 1. Create workflow config +cat > workflow-config.yaml << EOF +workflows: - name: "Copy Go examples" - source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P.+)$" - targets: - - repo: "myorg/target-repo" - branch: "main" - path_transform: "docs/code-examples/\${lang}/\${file}" + source: + repo: "myorg/source-repo" + branch: "main" + destination: + repo: "myorg/target-repo" + branch: "main" + transformations: + - regex: + pattern: "^examples/(?P[^/]+)/(?P.+)$" + transform: "docs/code-examples/\${lang}/\${file}" EOF # 2. Validate -./config-validator validate -config copier-config.yaml -v +./config-validator validate -config workflow-config.yaml -v # 3. Test pattern ./config-validator test-pattern \ @@ -338,8 +304,7 @@ EOF ## See Also -- [Configuration Guide](../../docs/CONFIGURATION-GUIDE.md) - Complete configuration reference +- [Main Config README](../../configs/copier-config-examples/MAIN-CONFIG-README.md) - Main config documentation - [Pattern Matching Guide](../../docs/PATTERN-MATCHING-GUIDE.md) - Pattern matching help -- [FAQ](../../docs/FAQ.md) - Frequently asked questions (includes JSON to YAML conversion) -- [Quick Reference](../../QUICK-REFERENCE.md) - All commands +- [FAQ](../../docs/FAQ.md) - Frequently asked questions diff --git a/examples-copier/cmd/config-validator/main.go b/examples-copier/cmd/config-validator/main.go index 367f1b2..866cdcb 100644 --- a/examples-copier/cmd/config-validator/main.go +++ b/examples-copier/cmd/config-validator/main.go @@ -28,11 +28,7 @@ func main() { initCmd := flag.NewFlagSet("init", flag.ExitOnError) initTemplate := initCmd.String("template", "basic", "Template to use: basic, glob, or regex") - initOutput := initCmd.String("output", "copier-config.yaml", "Output file path") - - convertCmd := flag.NewFlagSet("convert", flag.ExitOnError) - convertInput := convertCmd.String("input", "", "Input config file (required)") - convertOutput := convertCmd.String("output", "", "Output config file (required)") + initOutput := initCmd.String("output", "workflow-config.yaml", "Output file path") if len(os.Args) < 2 { printUsage() @@ -71,15 +67,6 @@ func main() { initCmd.Parse(os.Args[2:]) initConfig(*initTemplate, *initOutput) - case "convert": - convertCmd.Parse(os.Args[2:]) - if *convertInput == "" || *convertOutput == "" { - fmt.Println("Error: -input and -output are required") - convertCmd.Usage() - os.Exit(1) - } - convertConfig(*convertInput, *convertOutput) - default: printUsage() os.Exit(1) @@ -87,24 +74,22 @@ func main() { } func printUsage() { - fmt.Println("Config Validator - Validate and test copier configurations") + fmt.Println("Config Validator - Validate and test copier workflow configurations") fmt.Println() fmt.Println("Usage:") fmt.Println(" config-validator [options]") fmt.Println() fmt.Println("Commands:") - fmt.Println(" validate Validate a configuration file") + fmt.Println(" validate Validate a workflow configuration file") fmt.Println(" test-pattern Test a pattern against a file path") fmt.Println(" test-transform Test a path transformation") - fmt.Println(" init Initialize a new config file from template") - fmt.Println(" convert Convert between JSON and YAML formats") + fmt.Println(" init Initialize a new workflow config file from template") fmt.Println() fmt.Println("Examples:") - fmt.Println(" config-validator validate -config copier-config.yaml -v") + fmt.Println(" config-validator validate -config .copier/workflows/config.yaml -v") fmt.Println(" config-validator test-pattern -type glob -pattern 'examples/**/*.go' -file 'examples/go/main.go'") fmt.Println(" config-validator test-transform -source 'examples/go/main.go' -template 'code/${filename}'") - fmt.Println(" config-validator init -template basic -output my-config.yaml") - fmt.Println(" config-validator convert -input config.json -output config.yaml") + fmt.Println(" config-validator init -template basic -output workflow-config.yaml") } func validateConfig(configFile string, verbose bool) { @@ -122,25 +107,20 @@ func validateConfig(configFile string, verbose bool) { } fmt.Println("✅ Configuration is valid!") - + if verbose { fmt.Println() - fmt.Printf("Source Repository: %s\n", config.SourceRepo) - fmt.Printf("Source Branch: %s\n", config.SourceBranch) - fmt.Printf("Number of Rules: %d\n", len(config.CopyRules)) + fmt.Printf("Number of Workflows: %d\n", len(config.Workflows)) fmt.Println() - - for i, rule := range config.CopyRules { - fmt.Printf("Rule %d: %s\n", i+1, rule.Name) - fmt.Printf(" Pattern Type: %s\n", rule.SourcePattern.Type) - fmt.Printf(" Pattern: %s\n", rule.SourcePattern.Pattern) - fmt.Printf(" Targets: %d\n", len(rule.Targets)) - for j, target := range rule.Targets { - fmt.Printf(" Target %d:\n", j+1) - fmt.Printf(" Repo: %s\n", target.Repo) - fmt.Printf(" Branch: %s\n", target.Branch) - fmt.Printf(" Transform: %s\n", target.PathTransform) - fmt.Printf(" Strategy: %s\n", target.CommitStrategy.Type) + + for i, workflow := range config.Workflows { + fmt.Printf("Workflow %d: %s\n", i+1, workflow.Name) + fmt.Printf(" Source: %s @ %s\n", workflow.Source.Repo, workflow.Source.Branch) + fmt.Printf(" Destination: %s @ %s\n", workflow.Destination.Repo, workflow.Destination.Branch) + fmt.Printf(" Transformations: %d\n", len(workflow.Transformations)) + fmt.Printf(" Commit Strategy: %s\n", workflow.CommitStrategy.Type) + if workflow.DeprecationCheck != nil && workflow.DeprecationCheck.Enabled { + fmt.Printf(" Deprecation Tracking: enabled (%s)\n", workflow.DeprecationCheck.File) } fmt.Println() } @@ -207,70 +187,35 @@ func testTransform(source, template, varsStr string) { } func initConfig(templateName, output string) { - templates := services.GetConfigTemplates() - var selectedTemplate *services.ConfigTemplate - - for _, tmpl := range templates { - if tmpl.Name == templateName { - selectedTemplate = &tmpl - break - } - } - - if selectedTemplate == nil { - fmt.Printf("❌ Unknown template: %s\n", templateName) - fmt.Println("\nAvailable templates:") - for _, tmpl := range templates { - fmt.Printf(" %s - %s\n", tmpl.Name, tmpl.Description) - } - os.Exit(1) - } - - err := os.WriteFile(output, []byte(selectedTemplate.Content), 0644) + // Simple workflow config template + template := `# Workflow Configuration +# This file defines workflows for copying code examples between repositories + +workflows: + - name: "example-workflow" + source: + repo: "mongodb/source-repo" + branch: "main" + path: "examples" + destination: + repo: "mongodb/dest-repo" + branch: "main" + transformations: + - move: + from: "examples" + to: "code-examples" + commit_strategy: + type: "pr" + pr_title: "Update code examples" + pr_body: "Automated update from source repository" +` + + err := os.WriteFile(output, []byte(template), 0644) if err != nil { fmt.Printf("❌ Error writing config file: %v\n", err) os.Exit(1) } - fmt.Printf("✅ Created config file: %s\n", output) - fmt.Printf("Template: %s\n", selectedTemplate.Description) + fmt.Printf("✅ Created workflow config file: %s\n", output) + fmt.Println("Edit this file to configure your workflows") } - -func convertConfig(input, output string) { - content, err := os.ReadFile(input) - if err != nil { - fmt.Printf("❌ Error reading input file: %v\n", err) - os.Exit(1) - } - - loader := services.NewConfigLoader() - config, err := loader.LoadConfigFromContent(string(content), input) - if err != nil { - fmt.Printf("❌ Error parsing input file: %v\n", err) - os.Exit(1) - } - - var outputContent string - if strings.HasSuffix(output, ".yaml") || strings.HasSuffix(output, ".yml") { - outputContent, err = services.ExportConfigAsYAML(config) - } else if strings.HasSuffix(output, ".json") { - outputContent, err = services.ExportConfigAsJSON(config) - } else { - fmt.Println("❌ Output file must have .yaml, .yml, or .json extension") - os.Exit(1) - } - - if err != nil { - fmt.Printf("❌ Error converting config: %v\n", err) - os.Exit(1) - } - - err = os.WriteFile(output, []byte(outputContent), 0644) - if err != nil { - fmt.Printf("❌ Error writing output file: %v\n", err) - os.Exit(1) - } - - fmt.Printf("✅ Converted %s to %s\n", input, output) -} - diff --git a/examples-copier/cmd/test-webhook/README.md b/examples-copier/cmd/test-webhook/README.md index eb9a273..69db911 100644 --- a/examples-copier/cmd/test-webhook/README.md +++ b/examples-copier/cmd/test-webhook/README.md @@ -108,7 +108,7 @@ Test your configuration locally before deploying: ```bash # 1. Start app in dry-run mode -DRY_RUN=true CONFIG_FILE=copier-config.yaml make run-local-quick +DRY_RUN=true make run-local-quick # 2. In another terminal, send test webhook ./test-webhook -payload test-payloads/example-pr-merged.json @@ -122,8 +122,8 @@ tail -f logs/app.log Test if your patterns match real PR files: ```bash -# 1. Start app with your config -CONFIG_FILE=copier-config.yaml make run-local-quick +# 1. Start app +make run-local-quick # 2. Send webhook with real PR data export GITHUB_TOKEN=ghp_... @@ -139,7 +139,7 @@ Verify files are copied to correct locations: ```bash # 1. Start app in dry-run mode -DRY_RUN=true CONFIG_FILE=copier-config.yaml ./examples-copier & +DRY_RUN=true ./examples-copier & # 2. Send test webhook ./test-webhook -payload test-payloads/example-pr-merged.json @@ -155,7 +155,7 @@ Test Slack integration: ```bash # 1. Start app with Slack enabled export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/..." -CONFIG_FILE=copier-config.yaml ./examples-copier & +./examples-copier & # 2. Send test webhook ./test-webhook -payload test-payloads/example-pr-merged.json @@ -170,7 +170,7 @@ Debug webhook processing: ```bash # 1. Enable debug logging export LOG_LEVEL=debug -CONFIG_FILE=copier-config.yaml ./examples-copier & +./examples-copier & # 2. Send test webhook ./test-webhook -payload test-payloads/example-pr-merged.json @@ -233,29 +233,20 @@ vim test-payloads/my-test.json ### Complete Testing Workflow ```bash -# 1. Validate configuration -./config-validator validate -config copier-config.yaml -v - -# 2. Test pattern matching -./config-validator test-pattern \ - -type regex \ - -pattern "^examples/(?P[^/]+)/(?P.+)$" \ - -file "examples/go/main.go" - -# 3. Start app in dry-run mode -DRY_RUN=true CONFIG_FILE=copier-config.yaml ./examples-copier & +# 1. Start app in dry-run mode +DRY_RUN=true ./examples-copier & -# 4. Test with example payload +# 2. Test with example payload ./test-webhook -payload test-payloads/example-pr-merged.json -# 5. Check metrics +# 3. Check metrics curl http://localhost:8080/metrics | jq -# 6. Test with real PR +# 4. Test with real PR export GITHUB_TOKEN=ghp_... ./test-webhook -pr 42 -owner myorg -repo myrepo -# 7. Review logs +# 5. Review logs grep "matched" logs/app.log ``` @@ -353,7 +344,7 @@ cat > run-tests.sh << 'EOF' set -e echo "Starting app..." -DRY_RUN=true CONFIG_FILE=copier-config.yaml ./examples-copier & +DRY_RUN=true ./examples-copier & APP_PID=$! sleep 2 @@ -399,7 +390,7 @@ jobs: - name: Test run: | - DRY_RUN=true CONFIG_FILE=copier-config.yaml ./examples-copier & + DRY_RUN=true ./examples-copier & sleep 2 ./test-webhook -payload test-payloads/example-pr-merged.json ``` diff --git a/examples-copier/configs/.env.local.example b/examples-copier/configs/.env.local.example index 4affbac..01656ae 100644 --- a/examples-copier/configs/.env.local.example +++ b/examples-copier/configs/.env.local.example @@ -15,11 +15,12 @@ # REQUIRED FOR LOCAL TESTING # ============================================================================ -# Source Repository Configuration (Required) -REPO_NAME="" -REPO_OWNER="" -# Source Branch (Optional - default: main) -SRC_BRANCH="test" +# Config Repository Configuration (Required) +# This is where your copier-config.yaml file is stored +CONFIG_REPO_NAME="" +CONFIG_REPO_OWNER="" +# Config Branch (Optional - default: main) +CONFIG_REPO_BRANCH="main" # Configuration Files CONFIG_FILE=copier-config.yaml diff --git a/examples-copier/configs/copier-config-examples/MAIN-CONFIG-README.md b/examples-copier/configs/copier-config-examples/MAIN-CONFIG-README.md new file mode 100644 index 0000000..3ab4fda --- /dev/null +++ b/examples-copier/configs/copier-config-examples/MAIN-CONFIG-README.md @@ -0,0 +1,309 @@ +# Main Config Architecture Guide + +This guide explains the main config architecture that supports centralized configuration with distributed workflow definitions. + +## Table of Contents + +- [Overview](#overview) +- [Architecture](#architecture) +- [Configuration Files](#configuration-files) +- [Getting Started](#getting-started) +- [Migration Guide](#migration-guide) +- [Best Practices](#best-practices) +- [Examples](#examples) + +## Overview + +The main config architecture introduces a hierarchical configuration system that separates global settings from workflow-specific configurations. This enables: + +- **Centralized defaults** in a main config file +- **Distributed workflows** in source repositories +- **Reusable components** for transformations, strategies, and excludes +- **Clear ownership** of workflow configurations + +## Architecture + +### Three-Tier Configuration + +``` +Main Config (Central) + ├── Global Defaults + └── Workflow Config References + ├── Local Workflow Configs (same repo) + ├── Remote Workflow Configs (source repos) + └── Inline Workflows (simple cases) + └── Individual Workflows +``` + +### Default Precedence + +Settings are applied in order of specificity (most specific wins): + +1. **Individual Workflow** settings (highest priority) +2. **Workflow Config** defaults +3. **Main Config** defaults +4. **System** defaults (lowest priority) + +## Configuration Files + +### 1. Main Config File + +**Location**: Specified in `env.yaml` as `MAIN_CONFIG_FILE` +**Purpose**: Central configuration with global defaults and workflow references + +```yaml +# main-config.yaml +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + exclude: + - "**/.env" + - "**/node_modules/**" + +workflow_configs: + - source: "repo" + repo: "mongodb/docs-sample-apps" + path: ".copier/workflows.yaml" +``` + +### 2. Workflow Config Files + +**Location**: In source repositories (e.g., `.copier/workflows.yaml`) +**Purpose**: Define workflows for a specific source repository + +```yaml +# .copier/workflows.yaml +defaults: + commit_strategy: + type: "pull_request" + +workflows: + - name: "mflix-java" + source: + repo: "mongodb/docs-sample-apps" + branch: "main" + destination: + repo: "mongodb/sample-app-java-mflix" + branch: "main" + transformations: + - move: { from: "mflix/client", to: "client" } +``` + +### 3. Reusable Component Files + +**Location**: In source repositories (e.g., `.copier/transformations/`) +**Purpose**: Extract common configurations for reuse + +```yaml +# .copier/transformations/mflix-java.yaml +- move: { from: "mflix/client", to: "client" } +- move: { from: "mflix/server/java-spring", to: "server" } +``` + +## Getting Started + +### Step 1: Set Environment Variables + +Add to your `env.yaml`: + +```yaml +env_variables: + # Main config settings + MAIN_CONFIG_FILE: "main-config.yaml" + USE_MAIN_CONFIG: "true" + + # Config repository + CONFIG_REPO_OWNER: "mongodb" + CONFIG_REPO_NAME: "code-copier-config" + CONFIG_REPO_BRANCH: "main" +``` + +### Step 2: Create Main Config + +Create `main-config.yaml` in your config repository: + +```yaml +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + exclude: + - "**/.env" + - "**/.env.*" + +workflow_configs: + - source: "repo" + repo: "mongodb/docs-sample-apps" + branch: "main" + path: ".copier/workflows.yaml" +``` + +### Step 3: Create Workflow Config in Source Repo + +Create `.copier/workflows.yaml` in your source repository: + +```yaml +workflows: + - name: "my-workflow" + source: + repo: "mongodb/docs-sample-apps" + branch: "main" + destination: + repo: "mongodb/my-destination" + branch: "main" + transformations: + - move: { from: "src", to: "dest" } +``` + +### Step 4: Deploy and Test + +1. Deploy the updated configuration +2. Trigger a webhook event (merge a PR in source repo) +3. Verify workflows execute correctly + +## Best Practices + +### 1. Organization + +**Recommended directory structure**: + +``` +Config Repo (mongodb/code-copier-config): + main-config.yaml + workflows/ + mflix-workflows.yaml + university-workflows.yaml + +Source Repo (mongodb/docs-sample-apps): + .copier/ + workflows.yaml + transformations/ + mflix-java.yaml + mflix-nodejs.yaml + strategies/ + mflix-pr-strategy.yaml + common/ + mflix-excludes.yaml +``` + +### 2. Workflow Config Placement + +- **Centralized**: Use `source: "local"` for workflows managed by central team +- **Distributed**: Use `source: "repo"` for workflows managed by source repo teams +- **Simple**: Use `source: "inline"` for one-off or simple workflows + +### 3. Reusable Components + +Extract common configurations: + +- **Transformations**: When multiple workflows use similar file mappings +- **Strategies**: When multiple workflows use the same PR format +- **Excludes**: When multiple workflows exclude the same patterns + +### 4. Default Strategy + +Set sensible defaults at each level: + +- **Main config**: Organization-wide defaults +- **Workflow config**: Source repo defaults +- **Individual workflow**: Workflow-specific overrides + +### 5. Testing + +- Test workflow configs in source repo PRs +- Use `DRY_RUN=true` for testing without side effects +- Validate configurations before deploying + +## Examples + +See the example files in this directory: + +- `main-config-example.yaml` - Complete main config example +- `source-repo-workflows-example.yaml` - Workflow config in source repo +- `reusable-components/` - Examples of reusable components + - `transformations-example.yaml` + - `strategy-example.yaml` + - `excludes-example.yaml` + +## Reference Syntax + +### Workflow Config References + +```yaml +# Local file in config repo +- source: "local" + path: "workflows/my-workflows.yaml" + +# Remote file in source repo +- source: "repo" + repo: "owner/repo" + branch: "main" + path: ".copier/workflows.yaml" + +# Inline workflows +- source: "inline" + workflows: + - name: "my-workflow" + # ... workflow definition ... +``` + +### Component References + +You can use `$ref` to reference external files for transformations, commit_strategy, and exclude patterns: + +```yaml +# Reference transformations +transformations: + $ref: "transformations/mflix-java.yaml" + +# Reference strategy +commit_strategy: + $ref: "strategies/mflix-pr-strategy.yaml" + +# Reference excludes +exclude: + $ref: "common/mflix-excludes.yaml" +``` + +**Benefits:** +- Share common configurations across multiple workflows +- Keep workflow configs clean and focused +- Organize related files in a logical directory structure + +**Path Resolution:** +- Relative paths are resolved relative to the workflow config file +- Example: If your workflow config is at `.copier/workflows.yaml`, then `transformations/mflix-java.yaml` resolves to `.copier/transformations/mflix-java.yaml` + +## Troubleshooting + +### Config Not Loading + +- Check `MAIN_CONFIG_FILE` is set correctly +- Verify `USE_MAIN_CONFIG=true` +- Check file exists in config repository +- Review logs for parsing errors + +### Workflows Not Executing + +- Verify workflow source repo matches webhook repo +- Check workflow config is referenced in main config +- Validate workflow config syntax +- Review logs for validation errors + +### Authentication Issues + +- Ensure GitHub App has access to all repos +- Verify installation IDs are correct +- Check app permissions + +## Support + +For questions or issues: + +1. Check the example configurations +2. Review the logs for error messages +3. Validate your configuration syntax +4. Test with `DRY_RUN=true` + + diff --git a/examples-copier/configs/copier-config-examples/QUICK-START-MAIN-CONFIG.md b/examples-copier/configs/copier-config-examples/QUICK-START-MAIN-CONFIG.md new file mode 100644 index 0000000..4331153 --- /dev/null +++ b/examples-copier/configs/copier-config-examples/QUICK-START-MAIN-CONFIG.md @@ -0,0 +1,301 @@ +# Quick Start: Main Config Architecture + +Get started with the main config architecture in 5 minutes. + +## What is Main Config? + +Main config is a centralized configuration system that: +- Stores global defaults in one place +- References workflow configs in source repositories +- Supports reusable components +- Enables distributed workflow management + +## Quick Setup + +### 1. Update env.yaml + +Add these variables to your `env.yaml`: + +```yaml +env_variables: + # Enable main config + MAIN_CONFIG_FILE: "main-config.yaml" + USE_MAIN_CONFIG: "true" + + # Config repository (where main config lives) + CONFIG_REPO_OWNER: "mongodb" + CONFIG_REPO_NAME: "code-copier-config" + CONFIG_REPO_BRANCH: "main" +``` + +### 2. Create Main Config + +Create `main-config.yaml` in your config repository: + +```yaml +# Global defaults +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + exclude: + - "**/.env" + - "**/.env.*" + +# Workflow references +workflow_configs: + # Reference workflow config in source repo + - source: "repo" + repo: "mongodb/docs-sample-apps" + branch: "main" + path: ".copier/workflows.yaml" +``` + +### 3. Create Workflow Config in Source Repo + +Create `.copier/workflows.yaml` in your source repository: + +```yaml +workflows: + - name: "my-first-workflow" + source: + repo: "mongodb/docs-sample-apps" + branch: "main" + destination: + repo: "mongodb/my-destination" + branch: "main" + transformations: + - move: + from: "examples" + to: "code-examples" +``` + +### 4. Deploy and Test + +```bash +# Deploy the app +gcloud app deploy app.yaml + +# Test by merging a PR in your source repo +# The workflow should execute automatically +``` + +## Common Patterns + +### Pattern 1: Centralized Workflows + +Keep all workflows in the config repo: + +```yaml +# main-config.yaml +workflow_configs: + - source: "local" + path: "workflows/mflix-workflows.yaml" + - source: "local" + path: "workflows/university-workflows.yaml" +``` + +**Use when**: Central team manages all workflows + +### Pattern 2: Distributed Workflows + +Keep workflows in source repos: + +```yaml +# main-config.yaml +workflow_configs: + - source: "repo" + repo: "mongodb/docs-sample-apps" + path: ".copier/workflows.yaml" + - source: "repo" + repo: "10gen/university-content" + path: ".copier/workflows.yaml" +``` + +**Use when**: Source repo teams manage their own workflows + +### Pattern 3: Hybrid Approach + +Mix centralized and distributed: + +```yaml +# main-config.yaml +workflow_configs: + # Centralized (managed by central team) + - source: "local" + path: "workflows/critical-workflows.yaml" + + # Distributed (managed by source teams) + - source: "repo" + repo: "mongodb/docs-sample-apps" + path: ".copier/workflows.yaml" + + # Inline (simple one-offs) + - source: "inline" + workflows: + - name: "simple-copy" + source: + repo: "mongodb/docs" + branch: "main" + destination: + repo: "mongodb/docs-public" + branch: "main" + transformations: + - move: { from: "examples", to: "public-examples" } +``` + +**Use when**: You need flexibility + +## Directory Structure + +### Recommended Structure + +``` +Config Repo (mongodb/code-copier-config): +├── main-config.yaml # Main config file +└── workflows/ # Centralized workflows (optional) + ├── mflix-workflows.yaml + └── university-workflows.yaml + +Source Repo (mongodb/docs-sample-apps): +└── .copier/ # Workflow config directory + ├── workflows.yaml # Workflow definitions + ├── transformations/ # Reusable transformations + │ ├── mflix-java.yaml + │ └── mflix-nodejs.yaml + ├── strategies/ # Reusable strategies + │ └── mflix-pr-strategy.yaml + └── common/ # Common configs + └── mflix-excludes.yaml +``` + +## Default Precedence + +Settings are applied from least to most specific: + +``` +System Defaults + ↓ +Main Config Defaults + ↓ +Workflow Config Defaults + ↓ +Individual Workflow Settings (wins) +``` + +Example: + +```yaml +# main-config.yaml +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false # Global default + +# .copier/workflows.yaml +defaults: + commit_strategy: + auto_merge: true # Overrides main config + +workflows: + - name: "my-workflow" + commit_strategy: + auto_merge: false # Overrides workflow config default +``` + +## Troubleshooting + +### Config Not Loading + +**Problem**: App can't find main config file + +**Solution**: +1. Check `MAIN_CONFIG_FILE` is set correctly +2. Verify file exists in config repository +3. Check `CONFIG_REPO_OWNER` and `CONFIG_REPO_NAME` +4. Review app logs for errors + +### Workflows Not Executing + +**Problem**: Workflows don't run when PR is merged + +**Solution**: +1. Verify workflow `source.repo` matches webhook repo +2. Check workflow config is referenced in main config +3. Validate YAML syntax +4. Check app logs for validation errors + +### Authentication Errors + +**Problem**: Can't access source or destination repos + +**Solution**: +1. Verify GitHub App has access to all repos +2. Check app is installed in all required orgs +3. Verify app permissions include repo read/write + +## Next Steps + +1. **Read the full guide**: See `MAIN-CONFIG-README.md` +2. **Review examples**: Check `main-config-example.yaml` +3. **Test locally**: Use `DRY_RUN=true` for testing +4. **Add more workflows**: Expand your configuration +5. **Use reusable components**: Extract common configs + +## Support + +- **Documentation**: `MAIN-CONFIG-README.md` +- **Examples**: `main-config-example.yaml`, `source-repo-workflows-example.yaml` +- **Reusable Components**: `reusable-components/` directory + +## Common Use Cases + +### Use Case 1: Monorepo with Many Workflows + +**Problem**: 50+ workflows in one config file +**Solution**: Split into multiple workflow config files + +```yaml +workflow_configs: + - source: "local" + path: "workflows/mflix-workflows.yaml" # 10 workflows + - source: "local" + path: "workflows/university-workflows.yaml" # 15 workflows + - source: "local" + path: "workflows/docs-workflows.yaml" # 25 workflows +``` + +### Use Case 2: Multiple Source Repos + +**Problem**: Workflows for different source repos mixed together +**Solution**: Each source repo has its own workflow config + +```yaml +workflow_configs: + - source: "repo" + repo: "mongodb/docs-sample-apps" + path: ".copier/workflows.yaml" + - source: "repo" + repo: "10gen/university-content" + path: ".copier/workflows.yaml" +``` + +### Use Case 3: Team Ownership + +**Problem**: Multiple teams need to manage workflows +**Solution**: Each team manages workflows in their source repo + +```yaml +workflow_configs: + # Team A's workflows + - source: "repo" + repo: "mongodb/team-a-repo" + path: ".copier/workflows.yaml" + + # Team B's workflows + - source: "repo" + repo: "mongodb/team-b-repo" + path: ".copier/workflows.yaml" +``` + +--- \ No newline at end of file diff --git a/examples-copier/configs/copier-config-examples/config.example.json b/examples-copier/configs/copier-config-examples/config.example.json deleted file mode 100644 index 177518d..0000000 --- a/examples-copier/configs/copier-config-examples/config.example.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "source_directory": "generated-examples/go", - "target_repo": "example-repo", - "target_branch": "main", - "target_directory": "go", - "recursive_copy": true, - "copier_commit_strategy": "pr", - "pr_title": "Update Go Examples", - "commit_message": "Copy latest Go examples from generated-examples", - "merge_without_review": false - }, - { - "source_directory": "examples/python", - "target_repo": "another-repo", - "target_branch": "v2.0", - "target_directory": "python", - "recursive_copy": false, - "copier_commit_strategy": "direct", - "pr_title": "", - "commit_message": "Copy v2 Python examples", - "merge_without_review": true - } -] diff --git a/examples-copier/configs/copier-config-examples/copier-config-batch-example.yaml b/examples-copier/configs/copier-config-examples/copier-config-batch-example.yaml deleted file mode 100644 index b89e304..0000000 --- a/examples-copier/configs/copier-config-examples/copier-config-batch-example.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Example configuration demonstrating batch_by_repo with batch_pr_config -# This shows how to batch multiple copy rules into a single PR per target repo -# with custom PR metadata and accurate file counts - -source_repo: "mongodb/code-examples" -source_branch: "main" - -# Enable batching - all rules targeting the same repo will be combined into one PR -batch_by_repo: true - -# Configure PR metadata for batched PRs -# These templates will be rendered with accurate file counts after all files are collected -batch_pr_config: - pr_title: "Update code examples from ${source_repo}" - pr_body: | - 🤖 Automated update of code examples - - **Source Information:** - - Repository: ${source_repo} - - Branch: ${source_branch} - - PR: #${pr_number} - - Commit: ${commit_sha} - - **Changes:** - - Total files: ${file_count} - - Target branch: ${target_branch} - - This PR includes files from multiple copy rules batched together for easier review. - use_pr_template: true # Optional: Fetch PR template from target repos - commit_message: "Update examples from ${source_repo} PR #${pr_number} (${file_count} files)" - -copy_rules: - # Rule 1: Copy client files - - name: "client-files" - source_pattern: - type: "prefix" - pattern: "examples/client/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "source/code-examples/client/${relative_path}" - commit_strategy: - type: "pull_request" - # Individual rule commit messages are still used for commits - # But PR title/body come from batch_pr_config above - commit_message: "Update client examples" - - # Rule 2: Copy server files - - name: "server-files" - source_pattern: - type: "prefix" - pattern: "examples/server/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "source/code-examples/server/${relative_path}" - commit_strategy: - type: "pull_request" - commit_message: "Update server examples" - - # Rule 3: Copy README files - - name: "readme-files" - source_pattern: - type: "glob" - pattern: "examples/**/README.md" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "source/code-examples/${matched_pattern}" - commit_strategy: - type: "pull_request" - commit_message: "Update README files" - -# Result: All three rules will be batched into ONE PR in mongodb/docs -# The PR will use the title/body from batch_pr_config with accurate file count -# For example, if rule 1 matches 10 files, rule 2 matches 5 files, and rule 3 matches 2 files, -# the PR will show "Total files: 17" in the body - diff --git a/examples-copier/configs/copier-config-examples/copier-config-exclude-example.yaml b/examples-copier/configs/copier-config-examples/copier-config-exclude-example.yaml deleted file mode 100644 index 0a23776..0000000 --- a/examples-copier/configs/copier-config-examples/copier-config-exclude-example.yaml +++ /dev/null @@ -1,189 +0,0 @@ -# Example Configuration with exclude_patterns -# This demonstrates how to use exclude_patterns to filter out specific files - -source_repo: "mongodb/code-examples" -source_branch: "main" -batch_by_repo: false # Create separate PRs for each rule - -copy_rules: - # Example 1: Exclude config files from examples - - name: "go-examples-no-config" - source_pattern: - type: "prefix" - pattern: "examples/go/" - exclude_patterns: - - "\.gitignore$" # Exclude .gitignore - - "\.env$" # Exclude .env files - - "\.env\\..*$" # Exclude .env.local, .env.production, etc. - - "config\\.local\\." # Exclude config.local.* files - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "source/code-examples/go/${relative_path}" - commit_strategy: - type: "pull_request" - pr_title: "Update Go examples (no config files)" - pr_body: "Automated update of Go code examples, excluding configuration files" - - # Example 2: Exclude test files and build artifacts - - name: "java-server-production-only" - source_pattern: - type: "regex" - pattern: "^server/java-spring/(?P.+)$" - exclude_patterns: - - "/test/" # Exclude test directories - - "Test\\.java$" # Exclude test files - - "/target/" # Exclude Maven build directory - - "\\.class$" # Exclude compiled files - targets: - - repo: "mongodb/sample-app-java" - branch: "main" - path_transform: "server/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update Java server (production code only)" - pr_body: "Automated update excluding tests and build artifacts" - - # Example 3: Exclude dependencies and build artifacts from Node.js project - - name: "nodejs-app-source-only" - source_pattern: - type: "prefix" - pattern: "apps/nodejs/" - exclude_patterns: - - "node_modules/" # Exclude dependencies - - "/dist/" # Exclude build output - - "/build/" # Exclude build directory - - "\\.min\\.js$" # Exclude minified files - - "\\.min\\.css$" # Exclude minified CSS - - "\\.map$" # Exclude source maps - targets: - - repo: "mongodb/sample-app-nodejs" - branch: "main" - path_transform: "${relative_path}" - commit_strategy: - type: "pull_request" - pr_title: "Update Node.js app (source code only)" - pr_body: "Automated update excluding dependencies and build artifacts" - - # Example 4: Exclude all hidden files - - name: "python-examples-no-hidden" - source_pattern: - type: "glob" - pattern: "examples/python/**/*.py" - exclude_patterns: - - "/\\.[^/]+$" # Exclude files starting with . - - "/__pycache__/" # Exclude Python cache - - "\\.pyc$" # Exclude compiled Python files - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "source/code-examples/python/${matched_pattern}" - commit_strategy: - type: "pull_request" - pr_title: "Update Python examples (no hidden files)" - pr_body: "Automated update excluding hidden files and cache" - - # Example 5: Exclude documentation from code examples - - name: "typescript-code-no-docs" - source_pattern: - type: "prefix" - pattern: "examples/typescript/" - exclude_patterns: - - "README\\.md$" # Exclude README files - - "\\.md$" # Exclude all markdown files - - "/docs/" # Exclude docs directory - - "CHANGELOG" # Exclude changelog files - targets: - - repo: "mongodb/typescript-examples" - branch: "main" - path_transform: "examples/${relative_path}" - commit_strategy: - type: "pull_request" - pr_title: "Update TypeScript examples (code only)" - pr_body: "Automated update excluding documentation files" - - # Example 6: Complex exclusions for a full-stack app - - name: "fullstack-app-clean" - source_pattern: - type: "prefix" - pattern: "apps/mflix/" - exclude_patterns: - # Config files - - "\.gitignore$" - - "\.env$" - - "\.env\\..*$" - # Dependencies - - "node_modules/" - - "vendor/" - - "__pycache__/" - # Build artifacts - - "/dist/" - - "/build/" - - "/target/" - - "\.min\.(js|css)$" - # Test files - - "/test/" - - "/tests/" - - "\.test\.(js|ts)$" - - "\.spec\.(js|ts)$" - # Documentation - - "README\\.md$" - - "/docs/" - # IDE files - - "\.vscode/" - - "\.idea/" - targets: - - repo: "mongodb/sample-app-mflix" - branch: "main" - path_transform: "${relative_path}" - commit_strategy: - type: "pull_request" - pr_title: "Update MFlix app (clean source)" - pr_body: | - Automated update of MFlix application - - Excludes: - - Configuration files - - Dependencies - - Build artifacts - - Test files - - Documentation - - IDE files - - # Example 7: Exclude specific file types - - name: "images-and-media-only" - source_pattern: - type: "glob" - pattern: "assets/**/*" - exclude_patterns: - - "\\.txt$" # Exclude text files - - "\\.md$" # Exclude markdown - - "\\.json$" # Exclude JSON - - "\\.xml$" # Exclude XML - - "\\.yaml$" # Exclude YAML - - "\\.yml$" # Exclude YML - targets: - - repo: "mongodb/docs-assets" - branch: "main" - path_transform: "images/${matched_pattern}" - commit_strategy: - type: "direct" - commit_message: "Update media assets (images only)" - - # Example 8: Exclude by directory depth - - name: "top-level-files-only" - source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P[^/]+)$" # Only matches files directly in lang dir - # No subdirectories will match due to pattern, but we can add extra safety: - exclude_patterns: - - "/" # Exclude anything with a slash (subdirectories) - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "examples/${lang}/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update top-level example files" - pr_body: "Automated update of top-level example files only" - diff --git a/examples-copier/configs/copier-config-examples/copier-config-pr-template-example.yaml b/examples-copier/configs/copier-config-examples/copier-config-pr-template-example.yaml deleted file mode 100644 index 726856d..0000000 --- a/examples-copier/configs/copier-config-examples/copier-config-pr-template-example.yaml +++ /dev/null @@ -1,213 +0,0 @@ -# Example Configuration with PR Template Support -# This demonstrates how to use use_pr_template to fetch and merge PR templates from target repos -# -# When use_pr_template: true, the service will: -# 1. Fetch the PR template from the target repo (.github/pull_request_template.md) -# 2. Place the template content FIRST (checklists, review guidelines) -# 3. Add a separator (---) -# 4. Append your configured pr_body (automation info) -# -# This ensures reviewers see the target repo's guidelines prominently. - -source_repo: "mongodb/code-examples" -source_branch: "main" -batch_by_repo: false # Create separate PRs for each rule - -copy_rules: - # Example 1: Use PR template from target repo - - name: "go-examples-with-template" - source_pattern: - type: "prefix" - pattern: "examples/go/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "source/code-examples/go/${relative_path}" - commit_strategy: - type: "pull_request" - pr_title: "Update Go examples" - pr_body: | - 🤖 **Automated Update of Go Examples** - - **Source Information:** - - Repository: ${source_repo} - - Branch: ${source_branch} - - PR: #${pr_number} - - Commit: ${commit_sha} - - **Changes:** - - Files updated: ${file_count} - - Target branch: ${target_branch} - use_pr_template: true # Fetch and merge PR template from mongodb/docs - auto_merge: false - - # Example 2: Different templates for different targets - - name: "shared-examples" - source_pattern: - type: "regex" - pattern: "^shared/(?P[^/]+)/(?P.+)$" - targets: - # Target 1: Main docs - use their PR template - - repo: "mongodb/docs" - branch: "main" - path_transform: "examples/${lang}/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update ${lang} examples" - pr_body: | - Automated update of ${lang} examples - - Files: ${file_count} - use_pr_template: true # Use mongodb/docs template - auto_merge: false - - # Target 2: Tutorials - use their different template - - repo: "mongodb/tutorials" - branch: "main" - path_transform: "code/${lang}/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update ${lang} tutorial code" - pr_body: | - Tutorial code update for ${lang} - - Source PR: #${pr_number} - use_pr_template: true # Use mongodb/tutorials template - auto_merge: false - - # Example 3: Conditional template usage - # Use template for production, skip for staging - - name: "python-examples-production" - source_pattern: - type: "prefix" - pattern: "examples/python/" - targets: - # Production: Use PR template for thorough review - - repo: "mongodb/docs" - branch: "main" - path_transform: "source/code/python/${relative_path}" - commit_strategy: - type: "pull_request" - pr_title: "Update Python examples (Production)" - pr_body: | - 🚀 **Production Update** - - Files: ${file_count} - Source: ${source_repo} - use_pr_template: true # Use template for production - auto_merge: false - - # Staging: Skip template for faster iteration - - repo: "mongodb/docs-staging" - branch: "main" - path_transform: "source/code/python/${relative_path}" - commit_strategy: - type: "pull_request" - pr_title: "Update Python examples (Staging)" - pr_body: | - 🧪 **Staging Update** - - Files: ${file_count} - use_pr_template: false # Skip template for staging - auto_merge: true # Auto-merge staging changes - - # Example 4: Rich PR body with template - - name: "java-examples-detailed" - source_pattern: - type: "regex" - pattern: "^examples/java/(?P[^/]+)/(?P.+)$" - targets: - - repo: "mongodb/java-driver" - branch: "main" - path_transform: "examples/${category}/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update ${category} examples" - pr_body: | - # Automated Example Update - - ## Summary - This PR updates Java code examples in the `${category}` category. - - ## Details - - **Category:** ${category} - - **Files Updated:** ${file_count} - - **Source Repository:** ${source_repo} - - **Source Branch:** ${source_branch} - - **Source PR:** #${pr_number} - - **Source Commit:** ${commit_sha} - - **Target Branch:** ${target_branch} - - ## Changes - These examples were automatically copied from the source repository. - All files have been validated and are ready for review. - - ## Testing - - [ ] Examples compile successfully - - [ ] Examples run without errors - - [ ] Documentation is up to date - - --- - - _This PR was automatically created by the Code Example Copier service._ - use_pr_template: true # Template appears first, then this content - auto_merge: false - - # Example 5: Minimal config with template - - name: "typescript-examples-simple" - source_pattern: - type: "prefix" - pattern: "examples/typescript/" - targets: - - repo: "mongodb/node-mongodb-native" - branch: "main" - path_transform: "examples/${relative_path}" - commit_strategy: - type: "pull_request" - pr_title: "Update TypeScript examples" - pr_body: "Automated update from ${source_repo} PR #${pr_number}" - use_pr_template: true # Template provides structure, this adds context - auto_merge: false - - # Example 6: No template (traditional approach) - - name: "rust-examples-no-template" - source_pattern: - type: "prefix" - pattern: "examples/rust/" - targets: - - repo: "mongodb/mongo-rust-driver" - branch: "main" - path_transform: "examples/${relative_path}" - commit_strategy: - type: "pull_request" - pr_title: "Update Rust examples" - pr_body: | - ## Automated Update - - This PR updates Rust code examples. - - **Details:** - - Files: ${file_count} - - Source: ${source_repo} - - PR: #${pr_number} - - **Review Checklist:** - - [ ] Examples compile - - [ ] Tests pass - - [ ] Documentation updated - use_pr_template: false # Use only this configured body - auto_merge: false - - # Example 7: Direct commit (template not applicable) - - name: "config-files-direct" - source_pattern: - type: "glob" - pattern: "config/**/*.yaml" - targets: - - repo: "mongodb/docs-config" - branch: "main" - path_transform: "examples/${matched_pattern}" - commit_strategy: - type: "direct" # Direct commits don't use PR templates - commit_message: "Update config files from ${source_repo}" - diff --git a/examples-copier/configs/copier-config-examples/copier-config-workflow.yaml b/examples-copier/configs/copier-config-examples/copier-config-workflow.yaml new file mode 100644 index 0000000..5f67f69 --- /dev/null +++ b/examples-copier/configs/copier-config-examples/copier-config-workflow.yaml @@ -0,0 +1,208 @@ +# MFlix Workflow Configuration +# This is the workflow-based format with multi-org support +# Replaces the default copy_rules format + +# Global defaults (apply to all workflows unless overridden) +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + deprecation_check: + enabled: true + file: "deprecated_examples.json" + exclude: + - "**/.env" + - "**/.env.*" + +# ============================================================================ +# WORKFLOWS +# ============================================================================ +# Each workflow defines a complete source → destination mapping +# ============================================================================ + +workflows: + # ========================================================================== + # Java MFlix: mongodb/docs-sample-apps → mongodb/sample-app-java-mflix + # ========================================================================== + + - name: "mflix-java" + source: + repo: "mongodb/docs-sample-apps" + branch: "main" + destination: + repo: "mongodb/sample-app-java-mflix" + branch: "main" + transformations: + # Copy client directory + - move: + from: "mflix/client" + to: "client" + + # Copy Java server (removing java-spring subdirectory) + - move: + from: "mflix/server/java-spring" + to: "server" + + # Copy README (renaming) + - copy: + from: "mflix/README-JAVA-SPRING.md" + to: "README.md" + + # Copy .gitignore (renaming) + - copy: + from: "mflix/.gitignore-java" + to: ".gitignore" + + commit_strategy: + type: "pull_request" + pr_title: "Update MFlix application from docs-sample-apps" + pr_body: | + Automated update of MFlix Java application + + **Source Information:** + - Repository: mongodb/docs-sample-apps + - Branch: main + + **Changes:** + - Client files (Next.js) + - Server files (Java/Spring Boot) + - README and .gitignore + + This PR includes all synchronized files from the source repository. + auto_merge: false + + # ========================================================================== + # Node.js MFlix: mongodb/docs-sample-apps → mongodb/sample-app-nodejs-mflix + # ========================================================================== + + - name: "mflix-nodejs" + source: + repo: "mongodb/docs-sample-apps" + branch: "main" + destination: + repo: "mongodb/sample-app-nodejs-mflix" + branch: "main" + transformations: + # Copy client directory + - move: + from: "mflix/client" + to: "client" + + # Copy Node.js server (removing js-express subdirectory) + - move: + from: "mflix/server/js-express" + to: "server" + + # Copy README (renaming) + - copy: + from: "mflix/README-JAVASCRIPT-EXPRESS.md" + to: "README.md" + + # Copy .gitignore (renaming) + - copy: + from: "mflix/.gitignore-js" + to: ".gitignore" + + commit_strategy: + type: "pull_request" + pr_title: "Update MFlix application from docs-sample-apps" + pr_body: | + Automated update of MFlix Node.js application + + **Source Information:** + - Repository: mongodb/docs-sample-apps + - Branch: main + + **Changes:** + - Client files (Next.js) + - Server files (Node.js/Express) + - README and .gitignore + + This PR includes all synchronized files from the source repository. + auto_merge: false + + # ========================================================================== + # Python MFlix: mongodb/docs-sample-apps → mongodb/sample-app-python-mflix + # ========================================================================== + + - name: "mflix-python" + source: + repo: "mongodb/docs-sample-apps" + branch: "main" + destination: + repo: "mongodb/sample-app-python-mflix" + branch: "main" + transformations: + # Copy client directory + - move: + from: "mflix/client" + to: "client" + + # Copy Python server (removing python-fastapi subdirectory) + - move: + from: "mflix/server/python-fastapi" + to: "server" + + # Copy README (renaming) + - copy: + from: "mflix/README-PYTHON-FASTAPI.md" + to: "README.md" + + # Copy .gitignore (renaming) + - copy: + from: "mflix/.gitignore-python" + to: ".gitignore" + + commit_strategy: + type: "pull_request" + pr_title: "Update MFlix application from docs-sample-apps" + pr_body: | + Automated update of MFlix Python application + + **Source Information:** + - Repository: mongodb/docs-sample-apps + - Branch: main + + **Changes:** + - Client files (Next.js) + - Server files (Python/FastAPI) + - README and .gitignore + + This PR includes all synchronized files from the source repository. + auto_merge: false + +# ============================================================================ +# ADDING A NEW APP +# ============================================================================ +# +# Just add a new workflow: +# +# - name: "new-app-java" +# source: +# repo: "mongodb/docs-sample-apps" +# branch: "main" +# destination: +# repo: "mongodb/sample-app-java-newapp" +# branch: "main" +# transformations: +# - move: { from: "newapp/client", to: "client" } +# - move: { from: "newapp/server/java", to: "server" } +# - copy: { from: "newapp/README-JAVA.md", to: "README.md" } +# +# ============================================================================ +# MULTI-ORG EXAMPLE +# ============================================================================ +# +# To copy from a different org, just change the source repo: +# +# - name: "university-course" +# source: +# repo: "10gen/university-content" # Different org! +# branch: "main" +# destination: +# repo: "mongodb-university/python-course" +# branch: "main" +# transformations: +# - move: { from: "courses/python/public", to: "content" } +# +# The app will auto-discover the installation ID for each org. diff --git a/examples-copier/configs/copier-config-examples/copier-config.example.yaml b/examples-copier/configs/copier-config-examples/copier-config.example.yaml deleted file mode 100644 index c4cfc55..0000000 --- a/examples-copier/configs/copier-config-examples/copier-config.example.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# Example YAML configuration for the GitHub Code Example Copier -# This file demonstrates the new pattern matching and path transformation features - -source_repo: "mongodb/docs-code-examples" -source_branch: "main" - -copy_rules: - # Example 1: Simple prefix matching with recursive copy - - name: "go-examples-basic" - source_pattern: - type: "prefix" - pattern: "generated-examples/go" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "source/code-examples/go/${relative_path}" - commit_strategy: - type: "direct" - commit_message: "Update Go code examples from ${source_repo}" - deprecation_check: - enabled: true - file: "deprecated_examples.json" - - # Example 2: Glob pattern matching for specific file types - - name: "python-examples-glob" - source_pattern: - type: "glob" - pattern: "examples/**/*.py" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "source/code-examples/python/${filename}" - commit_strategy: - type: "pull_request" - pr_title: "Update Python examples" - pr_body: "Automated update of Python code examples from ${source_repo} (PR #${pr_number})" - auto_merge: false - deprecation_check: - enabled: true - - # Example 3: Advanced regex pattern with variable extraction - - name: "language-categorized-examples" - source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$" - targets: - # Target 1: Main docs repository - - repo: "mongodb/docs" - branch: "main" - path_transform: "source/code-examples/${lang}/${category}/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update ${lang} ${category} examples" - pr_body: | - Automated update of ${lang} examples in category: ${category} - - Files updated: ${file_count} - Source: ${source_repo} - PR: #${pr_number} - Commit: ${commit_sha} - auto_merge: false - deprecation_check: - enabled: true - file: "deprecated_examples.json" - - # Target 2: Tutorials repository (different structure) - - repo: "mongodb/tutorials" - branch: "main" - path_transform: "code/${lang}/${file}" - commit_strategy: - type: "direct" - commit_message: "Update ${lang} tutorial code from ${source_repo}" - - # Example 4: Batch commit strategy for high-volume updates - - name: "test-examples-batch" - source_pattern: - type: "glob" - pattern: "tests/**/*.test.js" - targets: - - repo: "mongodb/test-examples" - branch: "develop" - path_transform: "examples/${filename}" - commit_strategy: - type: "batch" - batch_size: 50 - commit_message: "Batch update of test examples (${file_count} files)" - deprecation_check: - enabled: false - - # Example 5: Non-main branch target for staging - - name: "staging-examples" - source_pattern: - type: "prefix" - pattern: "staging/" - targets: - - repo: "mongodb/docs" - branch: "staging" - path_transform: "source/staging/${relative_path}" - commit_strategy: - type: "pull_request" - pr_title: "Update staging examples" - auto_merge: true - deprecation_check: - enabled: true - diff --git a/examples-copier/configs/copier-config-examples/main-config-example.yaml b/examples-copier/configs/copier-config-examples/main-config-example.yaml new file mode 100644 index 0000000..42c67cb --- /dev/null +++ b/examples-copier/configs/copier-config-examples/main-config-example.yaml @@ -0,0 +1,181 @@ +# Main Configuration File Example +# This is the central config file that references individual workflow configs +# Specified in env.yaml as MAIN_CONFIG_FILE + +# ============================================================================ +# GLOBAL DEFAULTS +# ============================================================================ +# These defaults apply to all workflows across all workflow config files +# unless overridden at the workflow config level or individual workflow level + +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + deprecation_check: + enabled: true + file: "deprecated_examples.json" + exclude: + - "**/.env" + - "**/.env.*" + - "**/node_modules/**" + - "**/.DS_Store" + - "**/coverage/**" + - "**/__pycache__/**" + - "**/*.pyc" + +# ============================================================================ +# WORKFLOW CONFIG REFERENCES +# ============================================================================ +# Each reference points to a workflow config file that contains workflows +# Supports three source types: local, repo, and inline +# +# Optional 'enabled' field (default: true) allows you to disable workflow +# configs without removing them from the main config. + +workflow_configs: + # -------------------------------------------------------------------------- + # LOCAL REFERENCES + # -------------------------------------------------------------------------- + # Reference workflow configs in the same repo as this main config + + - source: "local" + path: "workflows/mflix-workflows.yaml" + enabled: true # Optional: explicitly enable (default: true) + # This file would be in the same repo at: workflows/mflix-workflows.yaml + + - source: "local" + path: "workflows/university-workflows.yaml" + enabled: false # Temporarily disable this workflow config + # This file would be in the same repo at: workflows/university-workflows.yaml + + # -------------------------------------------------------------------------- + # REPO REFERENCES + # -------------------------------------------------------------------------- + # Reference workflow configs in source repositories + # This allows each source repo to manage its own workflow configurations + # + # IMPORTANT: Workflows in repo-sourced configs automatically inherit the + # repo and branch from this reference! They don't need to specify + # source.repo or source.branch unless overriding. + # + # Example: If you reference mongodb/docs-sample-apps, workflows in that + # config file can omit source.repo and source.branch - they'll be + # automatically inferred from this reference. + + - source: "repo" + repo: "mongodb/docs-sample-apps" + branch: "main" # optional, defaults to main + path: ".copier/workflows.yaml" + enabled: true # Optional: explicitly enable (default: true) + # This fetches the workflow config from the source repo itself + # The source repo maintains its own workflow definitions + # Workflows in .copier/workflows.yaml inherit repo/branch automatically + + - source: "repo" + repo: "10gen/university-content" + branch: "main" + path: ".copier/workflows.yaml" + enabled: false # Temporarily disable this entire repo's workflows + # Different org - app will auto-discover installation ID + + - source: "repo" + repo: "mongodb/docs" + branch: "main" + path: ".github/copier-workflows.yaml" + # enabled defaults to true when not specified + # Can be anywhere in the repo + + # -------------------------------------------------------------------------- + # INLINE WORKFLOWS + # -------------------------------------------------------------------------- + # For simple cases, you can define workflows directly in the main config + + - source: "inline" + workflows: + - name: "simple-docs-copy" + source: + repo: "mongodb/docs-internal" + branch: "main" + destination: + repo: "mongodb/docs-public" + branch: "main" + transformations: + - move: + from: "public-examples" + to: "examples" + commit_strategy: + type: "pull_request" + pr_title: "Update public examples" + pr_body: "Automated sync from internal docs" + auto_merge: false + +# ============================================================================ +# HOW IT WORKS +# ============================================================================ +# +# 1. CONFIGURATION LOADING +# - App loads this main config file on startup +# - Resolves all workflow_configs references +# - Merges all workflows into a single configuration +# - Applies default precedence (workflow > workflow config > main config) +# +# 2. DEFAULT PRECEDENCE (most specific wins) +# - Individual workflow settings (highest priority) +# - Workflow config file defaults +# - Main config defaults +# - System defaults (lowest priority) +# +# 3. WEBHOOK PROCESSING +# - PR merged in mongodb/docs-sample-apps +# - App finds workflows with source.repo = "mongodb/docs-sample-apps" +# - Processes all matching workflows +# +# 4. MULTI-ORG SUPPORT +# - Each repo reference can be in a different org +# - App auto-discovers installation ID for each org +# - Handles authentication automatically +# +# ============================================================================ +# ENVIRONMENT VARIABLES +# ============================================================================ +# +# Required in env.yaml: +# MAIN_CONFIG_FILE: "main-config.yaml" +# USE_MAIN_CONFIG: "true" # if "false", use optional CONFIG_FILE instead +# CONFIG_REPO_OWNER: "mongodb" +# CONFIG_REPO_NAME: "code-copier-config" +# CONFIG_REPO_BRANCH: "main" +# +# Optional: +# CONFIG_FILE: "copier-config.yaml" # if USE_MAIN_CONFIG: "false" +# +# ============================================================================ +# EXAMPLE DIRECTORY STRUCTURE +# ============================================================================ +# +# Config Repo (mongodb/code-copier-config): +# main-config.yaml # This file +# workflows/ +# mflix-workflows.yaml # MFlix workflows +# university-workflows.yaml # University workflows +# strategies/ +# standard-pr.yaml # Reusable PR strategy +# auto-merge-pr.yaml # Auto-merge PR strategy +# common/ +# standard-excludes.yaml # Common exclude patterns +# +# Source Repo (mongodb/docs-sample-apps): +# .copier/ +# workflows.yaml # Workflows for this repo +# transformations/ +# mflix-java.yaml # Java transformations +# mflix-nodejs.yaml # Node.js transformations +# mflix-python.yaml # Python transformations +# strategies/ +# mflix-pr-strategy.yaml # MFlix PR strategy +# common/ +# mflix-excludes.yaml # MFlix exclude patterns +# +# ============================================================================ + diff --git a/examples-copier/configs/copier-config-examples/multi-org-config-example.yaml b/examples-copier/configs/copier-config-examples/multi-org-config-example.yaml new file mode 100644 index 0000000..e365b31 --- /dev/null +++ b/examples-copier/configs/copier-config-examples/multi-org-config-example.yaml @@ -0,0 +1,301 @@ +# ============================================================================ +# MULTI-ORG WORKFLOW CONFIG EXAMPLE +# ============================================================================ +# This config demonstrates how to use workflows with multiple GitHub orgs +# Store this in a centralized config repo (e.g., mongodb/code-copier-config) +# ============================================================================ + +# Global defaults (apply to all workflows unless overridden) +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + deprecation_check: + enabled: true + exclude: + - "**/.env" + - "**/.env.*" + - "**/node_modules/**" + - "**/__pycache__/**" + +# ============================================================================ +# WORKFLOWS +# ============================================================================ +# Each workflow defines: +# - source: where files come from (repo + branch) +# - destination: where files go to (repo + branch) +# - transformations: how to copy/move files +# ============================================================================ + +workflows: + # ========================================================================== + # MongoDB Org: docs-sample-apps → sample app repos + # ========================================================================== + + - name: "mflix-java" + source: + repo: "mongodb/docs-sample-apps" + branch: "main" + destination: + repo: "mongodb/sample-app-java-mflix" + branch: "main" + transformations: + - move: { from: "mflix/client", to: "client" } + - move: { from: "mflix/server/java-spring", to: "server" } + - copy: { from: "mflix/README-JAVA-SPRING.md", to: "README.md" } + - copy: { from: "mflix/.gitignore-java", to: ".gitignore" } + + - name: "mflix-nodejs" + source: + repo: "mongodb/docs-sample-apps" + branch: "main" + destination: + repo: "mongodb/sample-app-nodejs-mflix" + branch: "main" + transformations: + - move: { from: "mflix/client", to: "client" } + - move: { from: "mflix/server/js-express", to: "server" } + - copy: { from: "mflix/README-JAVASCRIPT-EXPRESS.md", to: "README.md" } + - copy: { from: "mflix/.gitignore-js", to: ".gitignore" } + + - name: "mflix-python" + source: + repo: "mongodb/docs-sample-apps" + branch: "main" + destination: + repo: "mongodb/sample-app-python-mflix" + branch: "main" + transformations: + - move: { from: "mflix/client", to: "client" } + - move: { from: "mflix/server/python-fastapi", to: "server" } + - copy: { from: "mflix/README-PYTHON-FASTAPI.md", to: "README.md" } + - copy: { from: "mflix/.gitignore-python", to: ".gitignore" } + + # ========================================================================== + # MongoDB Org: Atlas SDK Go examples → Architecture Center + # ========================================================================== + + - name: "atlas-sdk-go-project-examples" + source: + repo: "mongodb/docs-mongodb-internal" + branch: "main" + destination: + repo: "mongodb/atlas-architecture-go-sdk" + branch: "main" + transformations: + - glob: + pattern: "content/code-examples/tested/go/atlas-sdk/project-copy/**/*" + strip_prefix: "content/code-examples/tested/go/atlas-sdk/project-copy/" + pr_config: + title: "Update Atlas SDK Go examples from ${source_repo} PR ${pr_number}" + # Option 1: Use inline body + body: | + Automated update of Atlas SDK Go project examples + + **Source Details:** + - Repository: ${source_repo} + - Branch: ${source_branch} + - Source PR: #${pr_number} + - Commit: ${commit_sha} + + **Changes:** + - Files updated: ${file_count} + # Option 2: Reference a PR template from the target repo (uncomment to use) + # pr_template: ".github/pull_request_template.md" + auto_merge: false + deprecation_check: + enabled: true + file: "deprecated_examples.json" + + # ========================================================================== + # 10gen Org: internal university content → public university repos + # ========================================================================== + + - name: "university-python-course" + source: + repo: "10gen/university-content" + branch: "main" + destination: + repo: "mongodb-university/python-course" + branch: "main" + transformations: + - move: { from: "courses/python/public", to: "content" } + - move: { from: "courses/python/exercises", to: "exercises" } + - copy: { from: "courses/python/README.md", to: "README.md" } + exclude: + - "**/internal/**" + - "**/solutions/**" + - "**/.env" + commit_strategy: + type: "pull_request" + pr_title: "Update Python course content" + auto_merge: false + + - name: "university-java-course" + source: + repo: "10gen/university-content" + branch: "main" + destination: + repo: "mongodb-university/java-course" + branch: "main" + transformations: + - move: { from: "courses/java/public", to: "content" } + - move: { from: "courses/java/exercises", to: "exercises" } + - copy: { from: "courses/java/README.md", to: "README.md" } + exclude: + - "**/internal/**" + - "**/solutions/**" + + - name: "university-nodejs-course" + source: + repo: "10gen/university-content" + branch: "main" + destination: + repo: "mongodb-university/nodejs-course" + branch: "main" + transformations: + - move: { from: "courses/nodejs/public", to: "content" } + - copy: { from: "courses/nodejs/README.md", to: "README.md" } + + # ========================================================================== + # MongoDB University Org: course source → course public + # ========================================================================== + + - name: "course-materials-sync" + source: + repo: "mongodb-university/course-source" + branch: "main" + destination: + repo: "mongodb-university/course-public" + branch: "main" + transformations: + - move: { from: "materials/public", to: "content" } + - move: { from: "materials/assets", to: "assets" } + - copy: { from: "materials/README.md", to: "README.md" } + commit_strategy: + type: "direct" # Override default - commit directly + commit_message: "Sync course materials" + + # ========================================================================== + # Cross-Org Example: 10gen → mongodb + # ========================================================================== + + - name: "internal-docs-to-public" + source: + repo: "10gen/internal-docs" + branch: "main" + destination: + repo: "mongodb/docs" + branch: "main" + transformations: + - move: { from: "public-docs", to: "source" } + exclude: + - "**/internal/**" + - "**/confidential/**" + commit_strategy: + type: "pull_request" + pr_title: "Update public docs from internal source" + pr_body: | + Automated sync from internal docs repository. + + **Review checklist:** + - [ ] No confidential information included + - [ ] All links work correctly + - [ ] Images are properly copied + auto_merge: false + +# ============================================================================ +# HOW IT WORKS +# ============================================================================ +# +# 1. WEBHOOK ROUTING +# - PR merged in mongodb/docs-sample-apps +# - App finds workflows with source.repo = "mongodb/docs-sample-apps" +# - Processes: mflix-java, mflix-nodejs, mflix-python +# +# 2. AUTHENTICATION +# - App auto-discovers installation ID for "mongodb" org +# - Gets installation token for mongodb org +# - Reads files from mongodb/docs-sample-apps +# - Writes files to mongodb/sample-app-* repos +# +# 3. MULTI-ORG EXAMPLE +# - PR merged in 10gen/university-content +# - App finds workflows with source.repo = "10gen/university-content" +# - Processes: university-python-course, university-java-course, university-nodejs-course +# - Authenticates to 10gen org (source) and mongodb-university org (destination) +# +# 4. CROSS-ORG EXAMPLE +# - PR merged in 10gen/internal-docs +# - App authenticates to 10gen org (read) and mongodb org (write) +# - Copies files between orgs +# +# ============================================================================ +# INSTALLATION REQUIREMENTS +# ============================================================================ +# +# The GitHub App must be installed in ALL organizations used: +# 1. mongodb - for docs-sample-apps and sample-app-* repos +# 2. 10gen - for university-content and internal-docs repos +# 3. mongodb-university - for course repos +# +# The app will auto-discover installation IDs for each org. +# +# ============================================================================ +# OPTIONAL: EXPLICIT INSTALLATION IDS +# ============================================================================ +# +# If auto-discovery doesn't work, you can specify installation IDs explicitly: +# +# - name: "example-with-explicit-ids" +# source: +# repo: "mongodb/docs-sample-apps" +# branch: "main" +# installation_id: "12345678" # Optional override +# destination: +# repo: "mongodb/sample-app-java" +# branch: "main" +# installation_id: "12345678" # Optional override +# transformations: [...] +# +# ============================================================================ +# CONFIG LOCATION +# ============================================================================ +# +# OPTION 1: Centralized Config Repo (Recommended for multi-org) +# - Store this file in: mongodb/code-copier-config/config.yaml +# - App reads config from this repo on startup +# - Single source of truth for all workflows +# - Easy to see all copy operations across orgs +# +# OPTION 2: Per-Source Config (Alternative) +# - Store config in each source repo +# - mongodb/docs-sample-apps/copier-config.yaml +# - 10gen/university-content/copier-config.yaml +# - Each source repo has its own workflows +# +# For multi-org, OPTION 1 is strongly recommended. +# +# ============================================================================ +# ADDING A NEW ORG +# ============================================================================ +# +# 1. Install GitHub App in the new org +# 2. Add workflows with source.repo in the new org +# 3. Deploy updated config +# 4. Test with a PR in the new org's source repo +# +# Example: +# +# - name: "new-org-workflow" +# source: +# repo: "new-org/source-repo" +# branch: "main" +# destination: +# repo: "mongodb/target-repo" +# branch: "main" +# transformations: +# - move: { from: "src", to: "dest" } +# +# ============================================================================ + diff --git a/examples-copier/configs/copier-config-examples/reusable-components/excludes-example.yaml b/examples-copier/configs/copier-config-examples/reusable-components/excludes-example.yaml new file mode 100644 index 0000000..9870a8d --- /dev/null +++ b/examples-copier/configs/copier-config-examples/reusable-components/excludes-example.yaml @@ -0,0 +1,135 @@ +# Reusable Exclude Patterns File +# This file contains exclude patterns that can be referenced from workflow configs +# Location: .copier/common/mflix-excludes.yaml (in source repo) + +# ============================================================================ +# MFLIX EXCLUDE PATTERNS +# ============================================================================ +# These patterns define files that should NOT be copied to destination repos +# Uses glob pattern matching + +# Environment files (never copy secrets) +- "**/.env" +- "**/.env.*" +- "!**/.env.example" # But DO copy .env.example + +# Dependency directories +- "**/node_modules/**" +- "**/venv/**" +- "**/.venv/**" +- "**/target/**" +- "**/build/**" +- "**/dist/**" + +# IDE and editor files +- "**/.vscode/**" +- "**/.idea/**" +- "**/*.swp" +- "**/*.swo" +- "**/*~" +- "**/.DS_Store" + +# Test coverage and reports +- "**/coverage/**" +- "**/.nyc_output/**" +- "**/htmlcov/**" +- "**/.pytest_cache/**" +- "**/.coverage" + +# Python cache files +- "**/__pycache__/**" +- "**/*.pyc" +- "**/*.pyo" +- "**/*.pyd" + +# Java build files +- "**/*.class" +- "**/.gradle/**" +- "**/.mvn/**" + +# Logs +- "**/*.log" +- "**/logs/**" + +# Temporary files +- "**/.tmp/**" +- "**/tmp/**" +- "**/*.tmp" + +# OS files +- "**/.DS_Store" +- "**/Thumbs.db" +- "**/.Spotlight-V100" +- "**/.Trashes" + +# Git files (usually don't want to copy) +- "**/.git/**" +- "**/.gitignore" # Usually replaced by destination-specific .gitignore + +# ============================================================================ +# USAGE IN WORKFLOW CONFIG +# ============================================================================ +# +# In your workflow config file (.copier/workflows.yaml): +# +# defaults: +# exclude: +# $ref: "common/mflix-excludes.yaml" +# +# Or in a specific workflow: +# +# workflows: +# - name: "mflix-java" +# # ... other config ... +# exclude: +# $ref: "common/mflix-excludes.yaml" +# +# ============================================================================ +# COMBINING WITH INLINE PATTERNS +# ============================================================================ +# +# You can reference a file and add additional patterns: +# +# exclude: +# $ref: "common/mflix-excludes.yaml" +# - "**/custom-exclude/**" # Add workflow-specific excludes +# +# ============================================================================ +# PATTERN SYNTAX +# ============================================================================ +# +# Glob patterns support: +# - * matches any characters except / +# - ** matches any characters including / +# - ? matches a single character +# - [abc] matches any character in the set +# - [!abc] matches any character NOT in the set +# - ! at start negates the pattern (include instead of exclude) +# +# Examples: +# - "*.js" - matches all .js files in root +# - "**/*.js" - matches all .js files anywhere +# - "src/**/*.test.js" - matches test files in src +# - "!important.js" - include this file even if excluded by other patterns +# +# ============================================================================ +# COMMON EXCLUDE PATTERN SETS +# ============================================================================ +# +# You can create multiple exclude files for different purposes: +# +# common/ +# standard-excludes.yaml # Basic excludes (env, node_modules, etc.) +# security-excludes.yaml # Security-sensitive files +# build-excludes.yaml # Build artifacts +# test-excludes.yaml # Test files and coverage +# ide-excludes.yaml # IDE and editor files +# +# Then combine them in your workflow: +# +# exclude: +# $ref: "common/standard-excludes.yaml" +# $ref: "common/security-excludes.yaml" +# +# ============================================================================ + diff --git a/examples-copier/configs/copier-config-examples/reusable-components/strategy-example.yaml b/examples-copier/configs/copier-config-examples/reusable-components/strategy-example.yaml new file mode 100644 index 0000000..b938df9 --- /dev/null +++ b/examples-copier/configs/copier-config-examples/reusable-components/strategy-example.yaml @@ -0,0 +1,97 @@ +# Reusable Commit Strategy File +# This file contains a commit strategy definition that can be referenced from workflow configs +# Location: .copier/strategies/mflix-pr-strategy.yaml (in source repo) + +# ============================================================================ +# MFLIX PR STRATEGY +# ============================================================================ +# This strategy is used for all MFlix application workflows +# It creates a PR with a standard title and body + +type: "pull_request" + +pr_title: "Update MFlix application from docs-sample-apps" + +pr_body: | + Automated update of MFlix application + + **Source Information:** + - Repository: mongodb/docs-sample-apps + - Branch: main + + **What's Changed:** + This PR synchronizes the latest changes from the source repository, + including: + - Client files (Next.js frontend) + - Server files (backend implementation) + - Configuration files + - Documentation updates + + **Review Checklist:** + - [ ] All tests pass + - [ ] No breaking changes + - [ ] Documentation is up to date + - [ ] Environment variables are correct + + **Deployment:** + This PR can be merged once all checks pass and it has been reviewed. + + --- + *This PR was automatically created by the Examples Copier service.* + +auto_merge: false + +# ============================================================================ +# USAGE IN WORKFLOW CONFIG +# ============================================================================ +# +# In your workflow config file (.copier/workflows.yaml): +# +# workflows: +# - name: "mflix-java" +# source: +# repo: "mongodb/docs-sample-apps" +# branch: "main" +# destination: +# repo: "mongodb/sample-app-java-mflix" +# branch: "main" +# transformations: +# # ... transformations ... +# commit_strategy: +# $ref: "strategies/mflix-pr-strategy.yaml" +# +# ============================================================================ +# INLINE OVERRIDES +# ============================================================================ +# +# You can also reference a strategy and override specific fields: +# +# commit_strategy: +# $ref: "strategies/mflix-pr-strategy.yaml" +# auto_merge: true # Override just this field +# +# ============================================================================ +# OTHER STRATEGY EXAMPLES +# ============================================================================ +# +# DIRECT COMMIT STRATEGY: +# ---------------------- +# type: "direct" +# commit_message: "Update examples from source repository" +# +# AUTO-MERGE PR STRATEGY: +# ---------------------- +# type: "pull_request" +# pr_title: "Auto-update examples" +# pr_body: "Automated update - will auto-merge if tests pass" +# auto_merge: true +# +# PR WITH TEMPLATE: +# ---------------- +# type: "pull_request" +# use_pr_template: true # Fetch and use PR template from target repo +# pr_title: "Update examples" +# auto_merge: false +# +# ============================================================================ + diff --git a/examples-copier/configs/copier-config-examples/reusable-components/transformations-example.yaml b/examples-copier/configs/copier-config-examples/reusable-components/transformations-example.yaml new file mode 100644 index 0000000..69c0ced --- /dev/null +++ b/examples-copier/configs/copier-config-examples/reusable-components/transformations-example.yaml @@ -0,0 +1,99 @@ +# Reusable Transformations File +# This file contains transformation definitions that can be referenced from workflow configs +# Location: .copier/transformations/mflix-java.yaml (in source repo) + +# ============================================================================ +# MFLIX JAVA TRANSFORMATIONS +# ============================================================================ +# These transformations are specific to the MFlix Java application +# They define how files are copied from the monorepo to the standalone app repo + +# Copy client directory (shared across all MFlix apps) +- move: + from: "mflix/client" + to: "client" + +# Copy Java server (removing java-spring subdirectory) +- move: + from: "mflix/server/java-spring" + to: "server" + +# Copy Java-specific README +- copy: + from: "mflix/README-JAVA-SPRING.md" + to: "README.md" + +# Copy Java-specific .gitignore +- copy: + from: "mflix/.gitignore-java" + to: ".gitignore" + +# Copy shared configuration files +- copy: + from: "mflix/docker-compose.yml" + to: "docker-compose.yml" + +# Copy environment template +- copy: + from: "mflix/.env.example" + to: ".env.example" + +# ============================================================================ +# USAGE IN WORKFLOW CONFIG +# ============================================================================ +# +# In your workflow config file (.copier/workflows.yaml): +# +# workflows: +# - name: "mflix-java" +# source: +# repo: "mongodb/docs-sample-apps" +# branch: "main" +# destination: +# repo: "mongodb/sample-app-java-mflix" +# branch: "main" +# transformations: +# $ref: "transformations/mflix-java.yaml" +# +# ============================================================================ +# BENEFITS +# ============================================================================ +# +# 1. REUSABILITY +# - Define transformations once +# - Reference from multiple workflows if needed +# - Easy to update in one place +# +# 2. CLARITY +# - Transformations are clearly documented +# - Easy to understand what's being copied +# - Self-documenting configuration +# +# 3. MAINTAINABILITY +# - Update transformations without touching workflow config +# - Test transformations independently +# - Version control with source code +# +# 4. ORGANIZATION +# - Keep workflow config clean and focused +# - Separate concerns (what vs how) +# - Easy to find and update +# +# ============================================================================ +# ADVANCED TRANSFORMATIONS +# ============================================================================ +# +# You can also use glob and regex transformations for more complex scenarios: +# +# # Glob transformation (copy all .js files) +# - glob: +# pattern: "mflix/server/**/*.js" +# transform: "server/${relative_path}" +# +# # Regex transformation (rename files) +# - regex: +# pattern: "mflix/server/(?P.*)\\.ts$" +# transform: "server/${path}.js" +# +# ============================================================================ + diff --git a/examples-copier/configs/copier-config-examples/source-repo-workflows-example.yaml b/examples-copier/configs/copier-config-examples/source-repo-workflows-example.yaml new file mode 100644 index 0000000..669c3c9 --- /dev/null +++ b/examples-copier/configs/copier-config-examples/source-repo-workflows-example.yaml @@ -0,0 +1,256 @@ +# Workflow Configuration File +# This file lives in the source repository (e.g., mongodb/docs-sample-apps/.copier/workflows.yaml) +# Referenced from main config file + +# ============================================================================ +# LOCAL DEFAULTS +# ============================================================================ +# These defaults apply to all workflows in this file +# They override main config defaults but can be overridden by individual workflows + +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + deprecation_check: + enabled: true + file: "deprecated_examples.json" + exclude: + - "**/.env" + - "**/.env.*" + - "**/node_modules/**" + - "**/.DS_Store" + +# ============================================================================ +# WORKFLOWS +# ============================================================================ +# Each workflow defines a complete source → destination mapping + +workflows: + # -------------------------------------------------------------------------- + # MFlix Java Application + # -------------------------------------------------------------------------- + - name: "mflix-java" + # NOTE: source.repo and source.branch are automatically inferred from the + # workflow config reference in the main config. No need to specify them! + # The main config references this file with: + # source: "repo" + # repo: "mongodb/docs-sample-apps" + # branch: "main" + # So all workflows in this file inherit those values by default. + destination: + repo: "mongodb/sample-app-java-mflix" + branch: "main" + + # Transformations define how files are copied/moved + transformations: + - move: + from: "mflix/client" + to: "client" + - move: + from: "mflix/server/java-spring" + to: "server" + - copy: + from: "mflix/README-JAVA-SPRING.md" + to: "README.md" + - copy: + from: "mflix/.gitignore-java" + to: ".gitignore" + + # Override commit strategy for this workflow + commit_strategy: + type: "pull_request" + pr_title: "Update MFlix Java application from docs-sample-apps" + pr_body: | + Automated update of MFlix Java application + + **Source Information:** + - Repository: mongodb/docs-sample-apps + - Branch: main + + **Changes:** + - Client files (Next.js) + - Server files (Java/Spring Boot) + - README and .gitignore + + This PR includes all synchronized files from the source repository. + auto_merge: false + + # -------------------------------------------------------------------------- + # MFlix Node.js Application + # -------------------------------------------------------------------------- + - name: "mflix-nodejs" + # source.repo and source.branch inherited from workflow config reference + destination: + repo: "mongodb/sample-app-nodejs-mflix" + branch: "main" + + transformations: + - move: + from: "mflix/client" + to: "client" + - move: + from: "mflix/server/js-express" + to: "server" + - copy: + from: "mflix/README-JAVASCRIPT-EXPRESS.md" + to: "README.md" + - copy: + from: "mflix/.gitignore-js" + to: ".gitignore" + + commit_strategy: + type: "pull_request" + pr_title: "Update MFlix Node.js application from docs-sample-apps" + pr_body: | + Automated update of MFlix Node.js application + + **Source Information:** + - Repository: mongodb/docs-sample-apps + - Branch: main + + **Changes:** + - Client files (Next.js) + - Server files (Node.js/Express) + - README and .gitignore + auto_merge: false + + # -------------------------------------------------------------------------- + # MFlix Python Application + # -------------------------------------------------------------------------- + - name: "mflix-python" + # source.repo and source.branch inherited from workflow config reference + destination: + repo: "mongodb/sample-app-python-mflix" + branch: "main" + + transformations: + - move: + from: "mflix/client" + to: "client" + - move: + from: "mflix/server/python-fastapi" + to: "server" + - copy: + from: "mflix/README-PYTHON-FASTAPI.md" + to: "README.md" + - copy: + from: "mflix/.gitignore-python" + to: ".gitignore" + + commit_strategy: + type: "pull_request" + pr_title: "Update MFlix Python application from docs-sample-apps" + pr_body: | + Automated update of MFlix Python application + + **Source Information:** + - Repository: mongodb/docs-sample-apps + - Branch: main + + **Changes:** + - Client files (Next.js) + - Server files (Python/FastAPI) + - README and .gitignore + auto_merge: false + +# ============================================================================ +# SOURCE CONTEXT INFERENCE +# ============================================================================ +# +# When a workflow config is referenced from a source repo, workflows can omit +# source.repo and source.branch - they're automatically inferred! +# +# EXAMPLE 1: Fully inferred (most common) +# - name: "my-workflow" +# # source.repo and source.branch inherited from workflow config reference +# destination: +# repo: "mongodb/dest-repo" +# transformations: +# - move: { from: "src", to: "dest" } +# +# EXAMPLE 2: Override branch only +# - name: "my-workflow-develop" +# source: +# branch: "develop" # Override branch, repo still inferred +# destination: +# repo: "mongodb/dest-repo" +# transformations: +# - move: { from: "src", to: "dest" } +# +# EXAMPLE 3: Explicit source (if needed) +# - name: "cross-repo-workflow" +# source: +# repo: "mongodb/other-repo" # Explicitly specify different repo +# branch: "main" +# destination: +# repo: "mongodb/dest-repo" +# transformations: +# - move: { from: "src", to: "dest" } +# +# ============================================================================ +# USING REFERENCES FOR REUSABILITY +# ============================================================================ +# +# For even better organization, you can extract common parts into separate files: +# +# .copier/ +# workflows.yaml # This file (references others) +# transformations/ +# mflix-java.yaml # Java transformations +# mflix-nodejs.yaml # Node.js transformations +# mflix-python.yaml # Python transformations +# strategies/ +# mflix-pr-strategy.yaml # Common PR strategy +# common/ +# mflix-excludes.yaml # Common exclude patterns +# +# Then use $ref to reference them: +# +# workflows: +# - name: "mflix-java" +# # source.repo and source.branch inherited from workflow config reference +# destination: +# repo: "mongodb/sample-app-java-mflix" +# branch: "main" +# transformations: +# $ref: "transformations/mflix-java.yaml" +# commit_strategy: +# $ref: "strategies/mflix-pr-strategy.yaml" +# +# ============================================================================ +# ADDING A NEW WORKFLOW +# ============================================================================ +# +# To add a new workflow to this source repo: +# +# 1. Add a new workflow entry to this file +# 2. Define destination and transformations +# 3. Optionally override commit strategy +# 4. Commit and push to source repo +# 5. No changes needed to main config! +# +# Example (with source context inference): +# +# - name: "new-app-java" +# # source.repo and source.branch inherited automatically +# destination: +# repo: "mongodb/sample-app-java-newapp" +# branch: "main" +# transformations: +# - move: { from: "newapp/client", to: "client" } +# - move: { from: "newapp/server/java", to: "server" } +# +# ============================================================================ +# TESTING WORKFLOW CONFIGS +# ============================================================================ +# +# You can test workflow configs locally before deploying: +# +# 1. Create a test config file +# 2. Use DRY_RUN=true to test without making changes +# 3. Validate transformations work as expected +# 4. Check PR titles and bodies +# +# ============================================================================ + diff --git a/examples-copier/configs/env.yaml.example b/examples-copier/configs/env.yaml.example index 33c2da2..f58d376 100644 --- a/examples-copier/configs/env.yaml.example +++ b/examples-copier/configs/env.yaml.example @@ -6,8 +6,11 @@ env_variables: # GitHub App Configuration GITHUB_APP_ID: "YOUR_GITHUB_APP_ID" # Your GitHub App ID (required) INSTALLATION_ID: "YOUR_INSTALLATION_ID" # GitHub App Installation ID (required) - REPO_OWNER: "your-org" # Source repository owner (required) - REPO_NAME: "your-repo" # Source repository name (required) + + # Config Repository (where copier-config.yaml is stored) + CONFIG_REPO_OWNER: "your-org" # Config repository owner (required) + CONFIG_REPO_NAME: "your-config-repo" # Config repository name (required) + CONFIG_REPO_BRANCH: "main" # Config file branch (default: main) # ============================================================================= # SECRET MANAGER REFERENCES (RECOMMENDED - Most Secure) @@ -42,12 +45,16 @@ env_variables: WEBSERVER_PATH: "/events" # Webhook endpoint path (default: /webhook) # Configuration Files - CONFIG_FILE: "copier-config.yaml" # Config file name in source repo (default: copier-config.yaml) + CONFIG_FILE: "copier-config.yaml" # Config file name in config repo (default: copier-config.yaml) + + # Main Config (for centralized config with workflow references) + # MAIN_CONFIG_FILE: "main-config.yaml" # Main config file with workflow references (optional) + # MAIN_CONFIG_FILE: ".copier/config.yaml" # Can be any path in the config repo + # USE_MAIN_CONFIG: "true" # Enable main config format (default: false, auto-enabled if MAIN_CONFIG_FILE is set) + # See configs/copier-config-examples/MAIN-CONFIG-README.md for details + DEPRECATION_FILE: "deprecated_examples.json" # Deprecation tracking file (default: deprecated_examples.json) - # Source Branch - SRC_BRANCH: "main" # Branch to copy from (default: main) - # ============================================================================= # COMMITTER INFORMATION # ============================================================================= @@ -195,4 +202,47 @@ env_variables: # See docs/DEPLOYMENT.md for deployment guide # See docs/DEPLOYMENT-CHECKLIST.md for step-by-step checklist # +# ============================================================================= +# MAIN CONFIG ARCHITECTURE +# ============================================================================= +# +# The main config architecture supports centralized configuration with +# distributed workflow definitions. This enables: +# +# - Centralized defaults in a main config file +# - Distributed workflows in source repositories +# - Reusable components for transformations, strategies, and excludes +# - Clear ownership of workflow configurations +# - Enable/disable workflow configs without removing them +# - Automatic source context inference (workflows inherit repo/branch) +# +# Key Features: +# 1. FLEXIBLE PATHS: MAIN_CONFIG_FILE can be any path in the config repo +# Examples: "main-config.yaml", ".copier/config.yaml", "configs/main.yaml" +# +# 2. WORKFLOW REFERENCES: Reference workflows from three sources: +# - inline: Workflows embedded directly in main config +# - local: Workflow configs in same repo as main config +# - repo: Workflow configs in source repositories +# +# 3. ENABLE/DISABLE: Add 'enabled: false' to workflow config references +# to temporarily disable them without removing from config +# +# 4. SOURCE CONTEXT INFERENCE: Workflows in repo-sourced configs automatically +# inherit source.repo and source.branch from the workflow config reference. +# No need to repeat this information in every workflow! +# +# To use the architecture: +# 1. Set MAIN_CONFIG_FILE to your main config file path (any path in config repo) +# 2. Set USE_MAIN_CONFIG to "true" (or leave unset - auto-enabled) +# 3. Create main config file with workflow references +# 4. Create workflow config files in source repositories +# +# For detailed documentation and examples: +# - See configs/copier-config-examples/MAIN-CONFIG-README.md +# - See configs/copier-config-examples/main-config-example.yaml +# - See configs/copier-config-examples/source-repo-workflows-example.yaml +# +# The default CONFIG_FILE format is still supported. +# diff --git a/examples-copier/configs/env.yaml.production b/examples-copier/configs/env.yaml.production index 37a008a..2a919b1 100644 --- a/examples-copier/configs/env.yaml.production +++ b/examples-copier/configs/env.yaml.production @@ -2,10 +2,12 @@ # GitHub Configuration (Non-sensitive) # ============================================================================= GITHUB_APP_ID: "1166559" -INSTALLATION_ID: "62138132" -REPO_OWNER: "mongodb" -REPO_NAME: "docs-mongodb-internal" -SRC_BRANCH: "main" +INSTALLATION_ID: "YOUR_INSTALLATION_ID" # Replace with your GitHub App Installation ID + +# Config Repository (where copier-config.yaml is stored) +CONFIG_REPO_OWNER: "mongodb" +CONFIG_REPO_NAME: "code-example-tooling" +CONFIG_REPO_BRANCH: "main" # ============================================================================= # Secret Manager References (Sensitive Data - SECURE!) @@ -25,6 +27,13 @@ MONGO_URI_SECRET_NAME: "projects/1054147886816/secrets/mongo-uri/versions/latest # PORT is automatically set by App Engine Flexible (do not override) WEBSERVER_PATH: "/events" CONFIG_FILE: "copier-config.yaml" + +# Main Config (for centralized config with workflow references) +# Uncomment to use the main config architecture: +# MAIN_CONFIG_FILE: "main-config.yaml" # Main config file with workflow references +# MAIN_CONFIG_FILE: ".copier/config.yaml" # Can be any path in the config repo +# USE_MAIN_CONFIG: "true" # Auto-enabled if MAIN_CONFIG_FILE is set + DEPRECATION_FILE: "deprecated_examples.json" # ============================================================================= diff --git a/examples-copier/configs/environment.go b/examples-copier/configs/environment.go index 2d02571..bf9ff1f 100644 --- a/examples-copier/configs/environment.go +++ b/examples-copier/configs/environment.go @@ -12,17 +12,19 @@ import ( type Config struct { EnvFile string Port string - RepoName string - RepoOwner string + ConfigRepoName string // Repository where config file is stored + ConfigRepoOwner string // Owner of repository where config file is stored AppId string AppClientId string InstallationId string CommitterName string CommitterEmail string ConfigFile string + MainConfigFile string // Main config file with workflow references (optional) + UseMainConfig bool // Whether to use main config format DeprecationFile string WebserverPath string - SrcBranch string + ConfigRepoBranch string // Branch to fetch config file from PEMKeyName string WebhookSecretName string WebhookSecret string @@ -32,21 +34,21 @@ type Config struct { DefaultPRMerge bool DefaultCommitMessage string - // New features - DryRun bool - AuditEnabled bool - MongoURI string - MongoURISecretName string - AuditDatabase string - AuditCollection string - MetricsEnabled bool + // Optional features + DryRun bool + AuditEnabled bool + MongoURI string + MongoURISecretName string + AuditDatabase string + AuditCollection string + MetricsEnabled bool // Slack notifications - SlackWebhookURL string - SlackChannel string - SlackUsername string - SlackIconEmoji string - SlackEnabled bool + SlackWebhookURL string + SlackChannel string + SlackUsername string + SlackIconEmoji string + SlackEnabled bool // GitHub API retry configuration GitHubAPIMaxRetries int @@ -58,59 +60,61 @@ type Config struct { } const ( - EnvFile = "ENV" - Port = "PORT" - RepoName = "REPO_NAME" - RepoOwner = "REPO_OWNER" - AppId = "GITHUB_APP_ID" - AppClientId = "GITHUB_APP_CLIENT_ID" - InstallationId = "INSTALLATION_ID" - CommitterName = "COMMITTER_NAME" - CommitterEmail = "COMMITTER_EMAIL" - ConfigFile = "CONFIG_FILE" - DeprecationFile = "DEPRECATION_FILE" - WebserverPath = "WEBSERVER_PATH" - SrcBranch = "SRC_BRANCH" - PEMKeyName = "PEM_NAME" - WebhookSecretName = "WEBHOOK_SECRET_NAME" - WebhookSecret = "WEBHOOK_SECRET" - CopierLogName = "COPIER_LOG_NAME" - GoogleCloudProjectId = "GOOGLE_CLOUD_PROJECT_ID" - DefaultRecursiveCopy = "DEFAULT_RECURSIVE_COPY" - DefaultPRMerge = "DEFAULT_PR_MERGE" - DefaultCommitMessage = "DEFAULT_COMMIT_MESSAGE" - DryRun = "DRY_RUN" - AuditEnabled = "AUDIT_ENABLED" - MongoURI = "MONGO_URI" - MongoURISecretName = "MONGO_URI_SECRET_NAME" - AuditDatabase = "AUDIT_DATABASE" - AuditCollection = "AUDIT_COLLECTION" - MetricsEnabled = "METRICS_ENABLED" - SlackWebhookURL = "SLACK_WEBHOOK_URL" - SlackChannel = "SLACK_CHANNEL" - SlackUsername = "SLACK_USERNAME" - SlackIconEmoji = "SLACK_ICON_EMOJI" - SlackEnabled = "SLACK_ENABLED" - GitHubAPIMaxRetries = "GITHUB_API_MAX_RETRIES" - GitHubAPIInitialRetryDelay = "GITHUB_API_INITIAL_RETRY_DELAY" - PRMergePollMaxAttempts = "PR_MERGE_POLL_MAX_ATTEMPTS" - PRMergePollInterval = "PR_MERGE_POLL_INTERVAL" + EnvFile = "ENV" + Port = "PORT" + ConfigRepoName = "CONFIG_REPO_NAME" + ConfigRepoOwner = "CONFIG_REPO_OWNER" + AppId = "GITHUB_APP_ID" + AppClientId = "GITHUB_APP_CLIENT_ID" + InstallationId = "INSTALLATION_ID" + CommitterName = "COMMITTER_NAME" + CommitterEmail = "COMMITTER_EMAIL" + ConfigFile = "CONFIG_FILE" + MainConfigFile = "MAIN_CONFIG_FILE" + UseMainConfig = "USE_MAIN_CONFIG" + DeprecationFile = "DEPRECATION_FILE" + WebserverPath = "WEBSERVER_PATH" + ConfigRepoBranch = "CONFIG_REPO_BRANCH" + PEMKeyName = "PEM_NAME" + WebhookSecretName = "WEBHOOK_SECRET_NAME" + WebhookSecret = "WEBHOOK_SECRET" + CopierLogName = "COPIER_LOG_NAME" + GoogleCloudProjectId = "GOOGLE_CLOUD_PROJECT_ID" + DefaultRecursiveCopy = "DEFAULT_RECURSIVE_COPY" + DefaultPRMerge = "DEFAULT_PR_MERGE" + DefaultCommitMessage = "DEFAULT_COMMIT_MESSAGE" + DryRun = "DRY_RUN" + AuditEnabled = "AUDIT_ENABLED" + MongoURI = "MONGO_URI" + MongoURISecretName = "MONGO_URI_SECRET_NAME" + AuditDatabase = "AUDIT_DATABASE" + AuditCollection = "AUDIT_COLLECTION" + MetricsEnabled = "METRICS_ENABLED" + SlackWebhookURL = "SLACK_WEBHOOK_URL" + SlackChannel = "SLACK_CHANNEL" + SlackUsername = "SLACK_USERNAME" + SlackIconEmoji = "SLACK_ICON_EMOJI" + SlackEnabled = "SLACK_ENABLED" + GitHubAPIMaxRetries = "GITHUB_API_MAX_RETRIES" + GitHubAPIInitialRetryDelay = "GITHUB_API_INITIAL_RETRY_DELAY" + PRMergePollMaxAttempts = "PR_MERGE_POLL_MAX_ATTEMPTS" + PRMergePollInterval = "PR_MERGE_POLL_INTERVAL" ) // NewConfig returns a new Config instance with default values func NewConfig() *Config { return &Config{ - Port: "8080", - CommitterName: "Copier Bot", - CommitterEmail: "bot@example.com", - ConfigFile: "copier-config.yaml", - DeprecationFile: "deprecated_examples.json", - WebserverPath: "/webhook", - SrcBranch: "main", // Default branch to copy from (NOTE: we are purposefully only allowing copying from `main` branch right now) - PEMKeyName: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest", // default secret name for GCP Secret Manager - WebhookSecretName: "projects/1054147886816/secrets/webhook-secret/versions/latest", // default webhook secret name for GCP Secret Manager - CopierLogName: "copy-copier-log", // default log name for logging to GCP - GoogleCloudProjectId: "github-copy-code-examples", // default project ID for logging to GCP + Port: "8080", + CommitterName: "Copier Bot", + CommitterEmail: "bot@example.com", + ConfigFile: "copier-config.yaml", + DeprecationFile: "deprecated_examples.json", + WebserverPath: "/webhook", + ConfigRepoBranch: "main", // Default branch to fetch config file from + PEMKeyName: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest", // default secret name for GCP Secret Manager + WebhookSecretName: "projects/1054147886816/secrets/webhook-secret/versions/latest", // default webhook secret name for GCP Secret Manager + CopierLogName: "copy-copier-log", // default log name for logging to GCP + GoogleCloudProjectId: "github-copy-code-examples", // default project ID for logging to GCP DefaultRecursiveCopy: true, // system-wide default for recursive copying that individual config entries can override. DefaultPRMerge: false, // system-wide default for PR merge without review that individual config entries can override. DefaultCommitMessage: "Automated PR with updated examples", // default commit message used when per-config commit_message is absent. @@ -152,17 +156,19 @@ func LoadEnvironment(envFile string) (*Config, error) { // Populate config from environment variables, with defaults where applicable config.Port = getEnvWithDefault(Port, config.Port) - config.RepoName = os.Getenv(RepoName) - config.RepoOwner = os.Getenv(RepoOwner) + config.ConfigRepoName = os.Getenv(ConfigRepoName) + config.ConfigRepoOwner = os.Getenv(ConfigRepoOwner) config.AppId = os.Getenv(AppId) config.AppClientId = os.Getenv(AppClientId) config.InstallationId = os.Getenv(InstallationId) config.CommitterName = getEnvWithDefault(CommitterName, config.CommitterName) config.CommitterEmail = getEnvWithDefault(CommitterEmail, config.CommitterEmail) config.ConfigFile = getEnvWithDefault(ConfigFile, config.ConfigFile) + config.MainConfigFile = os.Getenv(MainConfigFile) + config.UseMainConfig = getBoolEnvWithDefault(UseMainConfig, config.MainConfigFile != "") config.DeprecationFile = getEnvWithDefault(DeprecationFile, config.DeprecationFile) config.WebserverPath = getEnvWithDefault(WebserverPath, config.WebserverPath) - config.SrcBranch = getEnvWithDefault(SrcBranch, config.SrcBranch) + config.ConfigRepoBranch = getEnvWithDefault(ConfigRepoBranch, config.ConfigRepoBranch) config.PEMKeyName = getEnvWithDefault(PEMKeyName, config.PEMKeyName) config.WebhookSecretName = getEnvWithDefault(WebhookSecretName, config.WebhookSecretName) config.WebhookSecret = os.Getenv(WebhookSecret) @@ -172,7 +178,7 @@ func LoadEnvironment(envFile string) (*Config, error) { config.GoogleCloudProjectId = getEnvWithDefault(GoogleCloudProjectId, config.GoogleCloudProjectId) config.DefaultCommitMessage = getEnvWithDefault(DefaultCommitMessage, config.DefaultCommitMessage) - // New features + // Optional features config.DryRun = getBoolEnvWithDefault(DryRun, false) config.AuditEnabled = getBoolEnvWithDefault(AuditEnabled, false) config.MongoURI = os.Getenv(MongoURI) @@ -199,17 +205,19 @@ func LoadEnvironment(envFile string) (*Config, error) { // Export resolved values back into environment so downstream os.Getenv sees defaults _ = os.Setenv(Port, config.Port) - _ = os.Setenv(RepoName, config.RepoName) - _ = os.Setenv(RepoOwner, config.RepoOwner) + _ = os.Setenv(ConfigRepoName, config.ConfigRepoName) + _ = os.Setenv(ConfigRepoOwner, config.ConfigRepoOwner) _ = os.Setenv(AppId, config.AppId) _ = os.Setenv(AppClientId, config.AppClientId) _ = os.Setenv(InstallationId, config.InstallationId) _ = os.Setenv(CommitterName, config.CommitterName) _ = os.Setenv(CommitterEmail, config.CommitterEmail) _ = os.Setenv(ConfigFile, config.ConfigFile) + _ = os.Setenv(MainConfigFile, config.MainConfigFile) + _ = os.Setenv(UseMainConfig, fmt.Sprintf("%t", config.UseMainConfig)) _ = os.Setenv(DeprecationFile, config.DeprecationFile) _ = os.Setenv(WebserverPath, config.WebserverPath) - _ = os.Setenv(SrcBranch, config.SrcBranch) + _ = os.Setenv(ConfigRepoBranch, config.ConfigRepoBranch) _ = os.Setenv(PEMKeyName, config.PEMKeyName) _ = os.Setenv(CopierLogName, config.CopierLogName) _ = os.Setenv(GoogleCloudProjectId, config.GoogleCloudProjectId) @@ -260,10 +268,10 @@ func validateConfig(config *Config) error { var missingVars []string requiredVars := map[string]string{ - RepoName: config.RepoName, - RepoOwner: config.RepoOwner, - AppId: config.AppId, - InstallationId: config.InstallationId, + ConfigRepoName: config.ConfigRepoName, + ConfigRepoOwner: config.ConfigRepoOwner, + AppId: config.AppId, + InstallationId: config.InstallationId, } for name, value := range requiredVars { diff --git a/examples-copier/docs/ARCHITECTURE.md b/examples-copier/docs/ARCHITECTURE.md index 05f5414..0098d30 100644 --- a/examples-copier/docs/ARCHITECTURE.md +++ b/examples-copier/docs/ARCHITECTURE.md @@ -1,9 +1,51 @@ # Examples Copier Architecture -This document describes the architecture and design of the examples-copier application, including its core components, pattern matching system, configuration management, deprecation tracking, and operational features. +This document describes the architecture and design of the examples-copier application, including its core components, main config system, pattern matching, configuration management, deprecation tracking, and operational features. ## Core Architecture +### Main Config System + +The application uses a **centralized main config** with **distributed workflow configs**: + +**Files:** +- `services/main_config_loader.go` - Main config loading and reference resolution +- `types/config.go` - Configuration types including MainConfig and WorkflowConfigRef + +**Key Features:** +- **Centralized Defaults** - Global defaults in main config file +- **Distributed Workflows** - Workflow configs in source repositories +- **Three Reference Types**: + - `inline` - Workflows embedded directly in main config + - `local` - Workflow configs in same repo as main config + - `repo` - Workflow configs in source repositories +- **Source Context Inference** - Workflows automatically inherit source.repo and source.branch from workflow config reference +- **$ref Support** - Reference external files for transformations, commit_strategy, and exclude patterns +- **Resilient Loading** - Continues processing when individual workflow configs fail to load (logs warnings instead of failing) + +**Configuration Structure:** +```yaml +# Main config (.copier/workflows/main.yaml in config repo) +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + +workflow_configs: + - source: "repo" + repo: "mongodb/docs-sample-apps" + branch: "main" + path: ".copier/workflows/config.yaml" + enabled: true +``` + +**Benefits:** +- Separation of concerns - each repo manages its own workflows +- Scalability - works for monorepos with many workflows +- Flexibility - mix centralized and distributed configs +- Discoverability - configs live near source code +- Maintainability - update workflows without touching main config + ### Service Container Pattern The application uses a **Service Container** to manage dependencies and provide thread-safe access to shared services: @@ -50,11 +92,64 @@ type UploadKey struct { - Ensures uniqueness when multiple files are deprecated to the same deprecation file - Prevents map key collisions -## Features +## Key Features + +### 1. Main Config with Workflow References -### 1. Enhanced Pattern Matching +**Files:** +- `services/main_config_loader.go` - Main config loading and workflow reference resolution +- `types/config.go` - MainConfig and WorkflowConfigRef types + +**Capabilities:** +- **Three-tier configuration**: Main config → Workflow configs → Individual workflows +- **Default precedence**: Workflow > Workflow config > Main config > System defaults +- **Workflow config references**: Local, remote (repo), or inline workflows +- **Source context inference**: Workflows inherit source.repo/branch from workflow config reference +- **Resilient loading**: Logs warnings for missing configs and continues processing +- **Validation**: Comprehensive validation at each level + +**Example:** +```yaml +# Main config +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + +workflow_configs: + - source: "repo" + repo: "mongodb/docs-sample-apps" + path: ".copier/workflows/config.yaml" +``` + +### 2. $ref Support for Reusable Components + +**Files:** +- `services/main_config_loader.go` - Reference resolution logic +- `types/config.go` - RefOrValue types for $ref support + +**Capabilities:** +- **Transformations references**: Extract common file mappings +- **Strategy references**: Reuse PR strategies across workflows +- **Exclude references**: Share exclude patterns +- **Relative paths**: Resolved relative to workflow config file +- **Repo references**: `repo://owner/repo/path/file.yaml@branch` format -**Files Created:** +**Example:** +```yaml +workflows: + - name: "mflix-java" + transformations: + $ref: "../transformations/mflix-java.yaml" + commit_strategy: + $ref: "../strategies/mflix-pr-strategy.yaml" + exclude: + $ref: "../common/mflix-excludes.yaml" +``` + +### 3. Enhanced Pattern Matching + +**Files:** - `services/pattern_matcher.go` - Pattern matching engine **Capabilities:** @@ -69,9 +164,9 @@ source_pattern: pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$" ``` -### 2. Path Transformations +### 4. Path Transformations -**Files Created:** +**Files:** - `services/pattern_matcher.go` (PathTransformer interface) **Capabilities:** @@ -85,7 +180,7 @@ source_pattern: path_transform: "source/code-examples/${lang}/${category}/${file}" ``` -### 3. Deprecation Tracking +### 5. Deprecation Tracking **Files:** - `services/webhook_handler_new.go` - Deprecation detection and queuing @@ -144,47 +239,23 @@ targets: - Returns early if no files to deprecate - Prevents blank commits to source repository -### 4. YAML Configuration Support +### 6. YAML Configuration Support -**Files Created:** -- `types/config.go` - New configuration types -- `services/config_loader.go` - Configuration loader with YAML/JSON support -- `configs/copier-config.example.yaml` - Example YAML configuration +**Files:** +- `types/config.go` - Configuration types with $ref support +- `services/config_loader.go` - Configuration loader +- `services/main_config_loader.go` - Main config loader with reference resolution **Capabilities:** - Native YAML support with `gopkg.in/yaml.v3` -- Backward compatible JSON support -- Automatic legacy config conversion +- Custom unmarshaling for $ref support - Comprehensive validation - Default value handling +- Reference resolution (relative paths and repo:// format) -**Configuration Structure:** -```yaml -source_repo: "mongodb/docs-code-examples" -source_branch: "main" - -copy_rules: - - name: "go-examples" - source_pattern: - type: "glob" - pattern: "examples/**/*.go" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/${filename}" - commit_strategy: - type: "pull_request" # or "direct" - pr_title: "Update Go examples" - pr_body: "Automated update" - auto_merge: false - deprecation_check: - enabled: true - file: "deprecated_examples.json" -``` - -### 5. Template Engine for Messages +### 7. Template Engine for Messages -**Files Created:** +**Files:** - `services/pattern_matcher.go` (MessageTemplater interface) - `types/config.go` (MessageContext) @@ -202,9 +273,9 @@ commit_strategy: pr_body: "Automated update of ${lang} examples (${file_count} files)" ``` -### 6. MongoDB Audit Logging +### 8. MongoDB Audit Logging -**Files Created:** +**Files:** - `services/audit_logger.go` - MongoDB audit logger **Capabilities:** @@ -247,9 +318,9 @@ AUDIT_COLLECTION="events" - Logs errors with full context - Thread-safe operation through ServiceContainer -### 7. Health Check and Metrics Endpoints +### 9. Health Check and Metrics Endpoints -**Files Created:** +**Files:** - `services/health_metrics.go` - Health and metrics implementation **Endpoints:** @@ -318,9 +389,9 @@ Returns detailed metrics: - GitHub API call tracking - Success rates and error rates -### 8. CLI Validation Tool +### 10. CLI Validation Tool -**Files Created:** +**Files:** - `cmd/config-validator/main.go` - CLI tool for configuration management **Commands:** @@ -341,26 +412,25 @@ config-validator test-transform \ -template "code/${filename}" # Initialize new config from template -config-validator init -template basic -output my-copier-config.yaml - -# Convert between formats -config-validator convert -input config.json -output copier-config.yaml +config-validator init -template basic -output my-workflow-config.yaml ``` -### 9. Development/Testing Features +### 11. Development/Testing Features **Features:** - **Dry Run Mode**: `DRY_RUN="true"` - No actual changes made - **Non-main Branch Support**: Configure any target branch -- **Enhanced Logging**: Structured logging with context (JSON format) +- **Enhanced Logging**: Structured logging with context - **Metrics Collection**: Optional metrics tracking - **Context-aware Operations**: All operations support context cancellation +- **Resilient Config Loading**: Continues processing when individual configs fail **Logging Features:** -- Structured JSON logs with contextual information +- Structured logs with contextual information - Operation tracking with elapsed time - File status logging (ADDED, MODIFIED, DELETED) - Deprecation event logging +- Warning logs for missing configs (non-fatal) - Error logging with full context ## Webhook Processing Flow @@ -419,116 +489,93 @@ if file.Status == "DELETED" { ## Configuration Examples -### Basic YAML Config +### Main Config with Workflow References ```yaml -source_repo: "mongodb/docs-code-examples" -source_branch: "main" - -copy_rules: - - name: "go-examples" - source_pattern: - type: "prefix" - pattern: "examples/go" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/go/${relative_path}" - commit_strategy: - type: "direct" - commit_message: "Update Go examples from ${source_repo}" +# .copier/workflows/main.yaml (in config repo) +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + exclude: + - "**/.env" + - "**/node_modules/**" + +workflow_configs: + # Workflows in source repo + - source: "repo" + repo: "mongodb/docs-sample-apps" + branch: "main" + path: ".copier/workflows/config.yaml" + enabled: true + + # Local workflows in config repo + - source: "local" + path: "workflows/internal-workflows.yaml" + enabled: true + + # Inline workflow for simple cases + - source: "inline" + workflows: + - name: "simple-copy" + source: + repo: "mongodb/source-repo" + branch: "main" + destination: + repo: "mongodb/dest-repo" + branch: "main" + transformations: + - move: { from: "src", to: "dest" } ``` -### Advanced Regex Config with Deprecation +### Workflow Config in Source Repo ```yaml -source_repo: "mongodb/docs-code-examples" -source_branch: "main" - -copy_rules: - - name: "language-examples" - source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P.+)$" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/${lang}/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update ${lang} examples" - pr_body: "Updated ${file_count} ${lang} files from ${source_repo}" - auto_merge: false - deprecation_check: - enabled: true - file: "deprecated_examples.json" +# .copier/workflows/config.yaml (in source repo) +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + deprecation_check: + enabled: true + +workflows: + - name: "mflix-java" + # source.repo and source.branch inherited from workflow config reference + destination: + repo: "mongodb/sample-app-java-mflix" + branch: "main" + transformations: + - move: { from: "mflix/client", to: "client" } + - move: { from: "mflix/server/java-spring", to: "server" } + commit_strategy: + $ref: "../strategies/mflix-pr-strategy.yaml" ``` -### Multi-Target Config +### Reusable Strategy File ```yaml -source_repo: "mongodb/aggregation-examples" -source_branch: "main" - -copy_rules: - # Java examples - - name: "java-examples" - source_pattern: - type: "regex" - pattern: "^java/(?P.+\\.java)$" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "java/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update Java examples" - auto_merge: false - deprecation_check: - enabled: true - file: "deprecated_examples.json" - - # Node.js examples - - name: "nodejs-examples" - source_pattern: - type: "regex" - pattern: "^nodejs/(?P.+\\.(js|ts))$" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "node/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update Node.js examples" - auto_merge: true - deprecation_check: - enabled: true - file: "deprecated_examples.json" - - # Python examples - - name: "python-examples" - source_pattern: - type: "regex" - pattern: "^python/(?P.+\\.py)$" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "python/${file}" - commit_strategy: - type: "direct" - commit_message: "Update Python examples" - deprecation_check: - enabled: true - file: "deprecated_examples.json" +# .copier/strategies/mflix-pr-strategy.yaml +type: "pull_request" +pr_title: "🤖 Automated update from source repo" +pr_body: | + This PR was automatically generated by the code copier app. + + **Files updated:** ${file_count} + **Source:** ${source_repo} +use_pr_template: true +auto_merge: false ``` ## Key Benefits -1. **Flexible Pattern Matching**: Regex patterns with variable extraction and multiple pattern types -2. **Better Developer Experience**: YAML configs are more readable and maintainable -3. **Observable**: Health checks, metrics, and comprehensive audit logging -4. **Testable**: CLI tools for validation and testing, dry-run mode -5. **Production Ready**: Thread-safe operations, proper error handling, monitoring -6. **Deprecation Tracking**: Automatic detection and tracking of deleted files -7. **Batch Operations**: Efficient batching of multiple files per target -8. **Template Engine**: Dynamic message generation with variables +1. **Centralized Configuration**: Main config with distributed workflow management +2. **Source Context Inference**: Workflows automatically inherit source repo/branch +3. **Reusable Components**: $ref support for transformations, strategies, and excludes +4. **Resilient Loading**: Continues processing when individual configs fail +5. **Flexible Pattern Matching**: Regex patterns with variable extraction +6. **Observable**: Health checks, metrics, and comprehensive audit logging +7. **Testable**: CLI tools for validation and testing, dry-run mode +8. **Production Ready**: Thread-safe operations, proper error handling, monitoring +9. **Deprecation Tracking**: Automatic detection and tracking of deleted files +10. **Template Engine**: Dynamic message generation with variables ## Thread Safety @@ -557,43 +604,51 @@ The application is designed for concurrent operations: ## Deployment -**Platform**: Google Cloud App Engine (Flexible Environment) +**Platform**: Google Cloud Run **Environment Variables:** -```bash -# Required -REPO_OWNER="mongodb" -REPO_NAME="docs-code-examples" -SRC_BRANCH="main" -GITHUB_TOKEN="ghp_..." -WEBHOOK_SECRET="..." - -# Optional -AUDIT_ENABLED="true" -MONGO_URI="mongodb+srv://..." -DRY_RUN="false" -CONFIG_FILE="copier-config.yaml" +```yaml +# GitHub Configuration +GITHUB_APP_ID: "1166559" +INSTALLATION_ID: "62138132" # Optional fallback + +# Config Repository +CONFIG_REPO_OWNER: "mongodb" +CONFIG_REPO_NAME: "code-example-tooling" +CONFIG_REPO_BRANCH: "main" + +# Main Config +MAIN_CONFIG_FILE: ".copier/workflows/main.yaml" +USE_MAIN_CONFIG: "true" + +# Secret Manager References +GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/.../secrets/CODE_COPIER_PEM/versions/latest" +WEBHOOK_SECRET_NAME: "projects/.../secrets/webhook-secret/versions/latest" +MONGO_URI_SECRET_NAME: "projects/.../secrets/mongo-uri/versions/latest" + +# Application Settings +WEBSERVER_PATH: "/events" +DEPRECATION_FILE: "deprecated_examples.json" +COMMITTER_NAME: "GitHub Copier App" +COMMITTER_EMAIL: "bot@mongodb.com" + +# Feature Flags +AUDIT_ENABLED: "false" +METRICS_ENABLED: "true" ``` **Health Monitoring:** - `/health` endpoint for liveness checks - `/metrics` endpoint for monitoring -- Structured JSON logs for analysis - -## Breaking Changes - -None - the refactoring maintains backward compatibility with existing JSON configs through automatic conversion. +- Structured logs for analysis ## Future Enhancements -Potential improvements documented in codebase: +Potential improvements: 1. **Automatic Cleanup PRs** - Create PRs to remove deprecated files from targets 2. **Expiration Dates** - Auto-remove deprecation entries after X days -3. **Cleanup Verification** - Check if deprecated files still exist in targets -4. **Batch Cleanup Tool** - CLI tool to clean up all deprecated files -5. **Notifications** - Alert when deprecation file grows large -6. **Retry Logic** - Automatic retry for failed GitHub API calls -7. **Rate Limiting** - Respect GitHub API rate limits -8. **Webhook Queue** - Queue webhooks for processing during high load +3. **Config Validation CLI** - Enhanced validation tool +4. **Retry Logic** - Automatic retry for failed GitHub API calls +5. **Rate Limiting** - Respect GitHub API rate limits diff --git a/examples-copier/docs/CONFIGURATION-GUIDE.md b/examples-copier/docs/CONFIGURATION-GUIDE.md deleted file mode 100644 index 128f0c0..0000000 --- a/examples-copier/docs/CONFIGURATION-GUIDE.md +++ /dev/null @@ -1,1524 +0,0 @@ -# Configuration Guide - -Complete guide to configuring the examples-copier application. - -## Table of Contents - -- [Overview](#overview) -- [Configuration File Structure](#configuration-file-structure) -- [Top-Level Fields](#top-level-fields) -- [Copy Rules](#copy-rules) -- [Source Patterns](#source-patterns) -- [Target Configuration](#target-configuration) -- [Commit Strategies](#commit-strategies) -- [Deprecation Tracking](#deprecation-tracking) -- [Built-in Variables](#built-in-variables) -- [Complete Examples](#complete-examples) -- [Validation](#validation) -- [Best Practices](#best-practices) -- [Pattern Matching Cheatsheet](#pattern-matching-cheat-sheet) - -## Overview - -The examples-copier uses a YAML configuration file (default: `copier-config.yaml`) to define how files are copied from a source repository to one or more target repositories. - -**Key Features:** -- Pattern matching (prefix, glob, regex) -- Path transformation with variables -- Multiple targets per rule -- Flexible commit strategies -- Deprecation tracking -- Template-based messages - -## Configuration File Structure - -```yaml -# Top-level configuration -source_repo: "owner/source-repository" -source_branch: "main" -batch_by_repo: false # Optional: batch all changes into one PR per target repo - -# Copy rules define what to copy and where -copy_rules: - - name: "rule-name" - source_pattern: - type: "prefix|glob|regex" - pattern: "pattern-string" - targets: - - repo: "owner/target-repository" - branch: "main" - path_transform: "target/path/${variable}" - commit_strategy: - type: "direct|pull_request|batch" - # ... strategy options - deprecation_check: - enabled: true - file: "deprecated_examples.json" -``` - -## Top-Level Fields - -### source_repo - -**Type:** String (required) -**Format:** `owner/repository` - -The source repository where files are copied from. - -```yaml -source_repo: "mongodb/docs-code-examples" -``` - -### source_branch - -**Type:** String (optional) -**Default:** `"main"` - -The branch to copy files from. - -```yaml -source_branch: "main" -``` - -### batch_by_repo - -**Type:** Boolean (optional) -**Default:** `false` - -When `true`, all changes from a single source PR are batched into **one pull request per target repository**, regardless of how many copy rules match files. - -When `false` (default), each copy rule creates a **separate pull request** in the target repository. - -**Example - Separate PRs per rule (default):** -```yaml -batch_by_repo: false # or omit this field - -copy_rules: - - name: "copy-client" - # ... matches 5 files - - name: "copy-server" - # ... matches 3 files - - name: "copy-readme" - # ... matches 1 file - -# Result: 3 separate PRs in the target repo -``` - -**Example - Single batched PR:** -```yaml -batch_by_repo: true - -copy_rules: - - name: "copy-client" - # ... matches 5 files - - name: "copy-server" - # ... matches 3 files - - name: "copy-readme" - # ... matches 1 file - -# Result: 1 PR containing all 9 files in the target repo -``` - -**Use Cases:** -- ✅ **Use `batch_by_repo: true`** when you want all related changes in a single PR for easier review -- ✅ **Use `batch_by_repo: false`** when different rules need separate review processes or different reviewers - -**Note:** When batching is enabled, use `batch_pr_config` (see below) to customize PR metadata, or a generic title/body will be generated. - -### batch_pr_config - -**Type:** Object (optional) -**Used when:** `batch_by_repo: true` - -Defines PR metadata (title, body, commit message) for batched pull requests. This allows you to customize the PR with accurate file counts and custom messaging. - -**Fields:** -- `pr_title` - (optional) PR title template -- `pr_body` - (optional) PR body template -- `commit_message` - (optional) Commit message template -- `use_pr_template` - (optional) Fetch and merge PR template from target repo (default: false) - -**Available template variables:** -- `${source_repo}` - Source repository (e.g., "owner/repo") -- `${target_repo}` - Target repository -- `${source_branch}` - Source branch name -- `${target_branch}` - Target branch name -- `${file_count}` - **Accurate** total number of files in the batched PR -- `${pr_number}` - Source PR number -- `${commit_sha}` - Source commit SHA - -**Example:** -```yaml -source_repo: "mongodb/code-examples" -source_branch: "main" -batch_by_repo: true - -batch_pr_config: - pr_title: "Update code examples from ${source_repo}" - pr_body: | - 🤖 Automated update of code examples - - **Source Information:** - - Repository: ${source_repo} - - PR: #${pr_number} - - Commit: ${commit_sha} - - **Changes:** - - Total files: ${file_count} - - Target branch: ${target_branch} - commit_message: "Update examples from ${source_repo} PR #${pr_number}" - use_pr_template: true # Fetch PR template from target repos - -copy_rules: - - name: "copy-client" - # ... rule config - - name: "copy-server" - # ... rule config -``` - -**Default behavior (if `batch_pr_config` is not specified):** -```yaml -# Default PR title: -"Update files from owner/repo PR #123" - -# Default PR body: -"Automated update from owner/repo - -Source PR: #123 -Commit: abc1234 -Files: 42" -``` - -### copy_rules - -**Type:** Array (required) -**Minimum:** 1 rule - -List of copy rules that define what files to copy and where. - -```yaml -copy_rules: - - name: "first-rule" - # ... rule configuration - - name: "second-rule" - # ... rule configuration -``` - -## Copy Rules - -Each copy rule defines a pattern to match files and one or more targets to copy them to. - -### name - -**Type:** String (required) - -Descriptive name for the rule. Used in logs and metrics. - -```yaml -name: "Copy Go examples" -``` - -### source_pattern - -**Type:** Object (required) - -Defines how to match source files. See [Source Patterns](#source-patterns). - -### targets - -**Type:** Array (required) -**Minimum:** 1 target - -List of target repositories and configurations. See [Target Configuration](#target-configuration). - -## Source Patterns - -Source patterns define which files to match in the source repository. - -### Pattern Types - -#### 1. Prefix Pattern - -Matches files that start with a specific prefix. - -```yaml -source_pattern: - type: "prefix" - pattern: "examples/go" -``` - -**Matches:** -- `examples/go/main.go` ✓ -- `examples/go/database/connect.go` ✓ -- `examples/python/main.py` ✗ - -**Variables Extracted:** -- `matched_prefix` - The matched prefix -- `relative_path` - Path after the prefix - -**Example:** -```yaml -# File: examples/go/database/connect.go -# Variables: -# matched_prefix: "examples/go" -# relative_path: "database/connect.go" -``` - -#### 2. Glob Pattern - -Matches files using wildcard patterns. - -```yaml -source_pattern: - type: "glob" - pattern: "examples/*/*.go" -``` - -**Wildcards:** -- `*` - Matches any characters except `/` -- `**` - Matches any characters including `/` -- `?` - Matches single character - -**Examples:** -```yaml -# Match all Go files in any language directory -pattern: "examples/*/*.go" - -# Match all files in examples directory (recursive) -pattern: "examples/**/*" - -# Match specific file types -pattern: "examples/**/*.{go,py,js}" -``` - -**Variables Extracted:** -- `matched_pattern` - The pattern that matched - -#### 3. Regex Pattern - -Matches files using regular expressions with named capture groups. - -```yaml -source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$" -``` - -**Named Capture Groups:** -Use `(?P...)` syntax to extract variables. - -**Example:** -```yaml -# File: examples/go/database/connect.go -# Pattern: ^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$ -# Variables: -# lang: "go" -# category: "database" -# file: "connect.go" -``` - -**Common Patterns:** -```yaml -# Language and file -pattern: "^examples/(?P[^/]+)/(?P.+)$" - -# Language, category, and file -pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$" - -# Version-specific examples -pattern: "^examples/v(?P[0-9]+)/(?P[^/]+)/(?P.+)$" - -# Optional segments -pattern: "^examples/(?P[^/]+)(/(?P[^/]+))?/(?P[^/]+)$" -``` - -### Excluding Files with `exclude_patterns` - -**Type:** Array of strings (optional) -**Format:** Go-compatible regex patterns - -You can exclude specific files from being matched by adding `exclude_patterns` to any source pattern. This is useful for filtering out files like `.gitignore`, `.env`, `node_modules`, build artifacts, etc. - -**Important:** Exclude patterns use **Go regex syntax** (no negative lookahead `(?!...)`). - -#### Basic Example - -```yaml -source_pattern: - type: "prefix" - pattern: "examples/" - exclude_patterns: - - "\.gitignore$" # Exclude .gitignore files - - "\.env$" # Exclude .env files - - "node_modules/" # Exclude node_modules directory -``` - -#### How It Works - -1. **Main pattern matches first** - The file must match the main pattern (`type` and `pattern`) -2. **Then exclusions are checked** - If the file matches any `exclude_patterns`, it's excluded -3. **Result** - File is only copied if it matches the main pattern AND doesn't match any exclusions - -#### Examples by Pattern Type - -**Prefix Pattern with Exclusions:** -```yaml -- name: "copy-examples-no-config" - source_pattern: - type: "prefix" - pattern: "examples/" - exclude_patterns: - - "\.gitignore$" - - "\.env$" - - "/node_modules/" - - "/dist/" - - "/build/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code-examples/${relative_path}" -``` - -**Regex Pattern with Exclusions:** -```yaml -- name: "java-server-no-tests" - source_pattern: - type: "regex" - pattern: "^mflix/server/java-spring/(?P.+)$" - exclude_patterns: - - "/test/" # Exclude test directories - - "Test\.java$" # Exclude test files - - "\.gitignore$" # Exclude .gitignore - targets: - - repo: "mongodb/sample-app-java" - branch: "main" - path_transform: "server/${file}" -``` - -**Glob Pattern with Exclusions:** -```yaml -- name: "js-files-no-minified" - source_pattern: - type: "glob" - pattern: "examples/**/*.js" - exclude_patterns: - - "\.min\.js$" # Exclude minified files - - "\.test\.js$" # Exclude test files - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/${matched_pattern}" -``` - -#### Common Exclusion Patterns - -```yaml -# Exclude hidden files (starting with .) -exclude_patterns: - - "/\\.[^/]+$" - -# Exclude build artifacts -exclude_patterns: - - "/dist/" - - "/build/" - - "\.min\\.(js|css)$" - -# Exclude dependencies -exclude_patterns: - - "node_modules/" - - "vendor/" - - "__pycache__/" - -# Exclude config files -exclude_patterns: - - "\.gitignore$" - - "\.env$" - - "\.env\\..*$" - - "config\\.local\\." - -# Exclude test files -exclude_patterns: - - "/test/" - - "/tests/" - - "Test\\.java$" - - "_test\\.go$" - - "\\.test\\.(js|ts)$" - - "\\.spec\\.(js|ts)$" - -# Exclude documentation -exclude_patterns: - - "README\\.md$" - - "\\.md$" - - "/docs/" -``` - -#### Regex Syntax Notes - -**✅ Supported (Go regex):** -- Character classes: `[abc]`, `[a-z]`, `[^abc]` -- Quantifiers: `*`, `+`, `?`, `{n}`, `{n,}`, `{n,m}` -- Anchors: `^` (start), `$` (end) -- Alternation: `(js|ts|jsx|tsx)` -- Escaping: `\.`, `\(`, `\[`, etc. - -**❌ Not Supported:** -- Negative lookahead: `(?!...)` - Use multiple patterns instead -- Lookbehind: `(?<=...)`, `(?[^/]+)/(?P[^/]+)/(?P.+)$" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "source/code/${lang}/${category}/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update ${lang} ${category} examples" - pr_body: | - Automated update of ${lang} examples - - Category: ${category} - Files: ${file_count} - Source PR: #${pr_number} - auto_merge: false - deprecation_check: - enabled: true -``` - -### Example 3: Multiple Targets - -Copy same files to multiple repositories. - -```yaml -source_repo: "mongodb/code-examples" -source_branch: "main" - -copy_rules: - - name: "Shared examples" - source_pattern: - type: "regex" - pattern: "^shared/(?P[^/]+)/(?P.+)$" - targets: - # Target 1: Main docs - - repo: "mongodb/docs" - branch: "main" - path_transform: "examples/${lang}/${file}" - commit_strategy: - type: "direct" - - # Target 2: Tutorials - - repo: "mongodb/tutorials" - branch: "main" - path_transform: "code/${lang}/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update ${lang} examples" - - # Target 3: API reference - - repo: "mongodb/api-docs" - branch: "main" - path_transform: "reference/${lang}/examples/${file}" - commit_strategy: - type: "direct" -``` - -### Example 4: Version-Specific Examples - -Handle versioned examples with different targets. - -```yaml -source_repo: "mongodb/code-examples" -source_branch: "main" - -copy_rules: - - name: "Versioned examples" - source_pattern: - type: "regex" - pattern: "^examples/v(?P[0-9]+)/(?P[^/]+)/(?P.+)$" - targets: - - repo: "mongodb/docs" - branch: "v${version}" - path_transform: "source/code/${lang}/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update v${version} ${lang} examples" - pr_body: | - Update ${lang} examples for version ${version} - - Files: ${file_count} - Source: ${source_repo} -``` - -### Example 5: Glob Pattern with File Type Filtering - -Copy specific file types using glob patterns. - -```yaml -source_repo: "mongodb/code-examples" -source_branch: "main" - -copy_rules: - - name: "Go source files" - source_pattern: - type: "glob" - pattern: "examples/**/*.go" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/go/${path}" - commit_strategy: - type: "batch" - batch_size: 25 - commit_message: "Update ${file_count} Go example files" -``` - -## Validation - -### Validate Configuration - -Use the `config-validator` tool to validate your configuration: - -```bash -# Validate syntax and structure -./config-validator validate -config copier-config.yaml -v - -# Test pattern matching -./config-validator test-pattern \ - -type regex \ - -pattern "^examples/(?P[^/]+)/(?P.+)$" \ - -file "examples/go/main.go" - -# Test path transformation -./config-validator test-transform \ - -source "examples/go/main.go" \ - -template "docs/${lang}/${file}" \ - -var "lang=go" \ - -var "file=main.go" -``` - -### Validation Rules - -**Required Fields:** -- `source_repo` - Must be in `owner/repo` format -- `copy_rules` - At least one rule required -- `copy_rules[].name` - Each rule must have a name -- `copy_rules[].source_pattern.type` - Must be `prefix`, `glob`, or `regex` -- `copy_rules[].source_pattern.pattern` - Pattern string required -- `copy_rules[].targets` - At least one target required -- `targets[].repo` - Must be in `owner/repo` format -- `targets[].path_transform` - Template string required - -**Optional Fields with Defaults:** -- `source_branch` - Defaults to `"main"` -- `targets[].branch` - Defaults to `"main"` -- `targets[].commit_strategy.type` - Defaults to `"direct"` -- `deprecation_check.file` - Defaults to `"deprecated_examples.json"` - -**Validation Errors:** - -```bash -# Missing required field -Error: copy_rules[0]: name is required - -# Invalid pattern type -Error: copy_rules[0].source_pattern: invalid pattern type: invalid (must be prefix, glob, or regex) - -# Invalid commit strategy -Error: copy_rules[0].targets[0].commit_strategy: invalid type: invalid (must be direct, pull_request, or batch) - -# Invalid regex pattern -Error: copy_rules[0].source_pattern: invalid regex pattern: missing closing ) -``` - -## Best Practices - -### 1. Use Descriptive Rule Names - -```yaml -# Good -name: "Copy Go database examples to docs" - -# Bad -name: "rule1" -``` - -### 2. Start Simple, Then Add Complexity - -```yaml -# Start with prefix patterns -source_pattern: - type: "prefix" - pattern: "examples/go" - -# Graduate to regex when needed -source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P.+)$" -``` - -### 3. Use Specific Patterns - -```yaml -# Good - specific pattern -pattern: "^examples/(?Pgo|python|java)/(?P.+\\.(?Pgo|py|java))$" - -# Risky - too broad -pattern: "^examples/.+$" -``` - -### 4. Test Patterns Before Deploying - -```bash -# Test locally first -./config-validator test-pattern \ - -type regex \ - -pattern "^examples/(?P[^/]+)/(?P.+)$" \ - -file "examples/go/main.go" - -# Validate entire config -./config-validator validate -config copier-config.yaml -v -``` - -### 5. Use Pull Requests for Important Changes - -```yaml -# For production docs -commit_strategy: - type: "pull_request" - auto_merge: false - -# For staging/dev -commit_strategy: - type: "direct" -``` - -### 6. Enable Deprecation Tracking - -```yaml -deprecation_check: - enabled: true - file: "deprecated_examples.json" -``` - -### 7. Use Meaningful Commit Messages - -```yaml -commit_strategy: - type: "pull_request" - pr_title: "Update ${lang} examples - ${file_count} files" - pr_body: | - ## Summary - Automated update of ${lang} code examples - - ## Details - - Files updated: ${file_count} - - Source: ${source_repo} - - Source PR: #${pr_number} - - Commit: ${commit_sha} - - ## Review Checklist - - [ ] Examples compile/run correctly - - [ ] Documentation is up to date - - [ ] No breaking changes -``` - -### 8. Organize Rules Logically - -```yaml -copy_rules: - # Group by language - - name: "Go examples" - # ... - - - name: "Python examples" - # ... - - # Or group by target - - name: "Main docs - all languages" - # ... - - - name: "Tutorials - all languages" - # ... -``` - -### 9. Use Environment-Specific Configs - -```bash -# Development -copier-config.dev.yaml - -# Staging -copier-config.staging.yaml - -# Production -copier-config.yaml -``` - -### 10. Document Your Configuration - -```yaml -# Add comments to explain complex patterns -copy_rules: - # This rule copies Go examples from the generated-examples directory - # to the main docs repository. It extracts the project name and - # preserves the directory structure. - - name: "Generated Go examples" - source_pattern: - type: "regex" - # Pattern: generated-examples/{project}/{rest-of-path} - pattern: "^generated-examples/(?P[^/]+)/(?P.+)$" - targets: - - repo: "mongodb/docs" - branch: "main" - # Transform: examples/{project}/{rest-of-path} - path_transform: "examples/${project}/${rest}" -``` - -## Configuration File Location - -### Default Location - -The application looks for `copier-config.yaml` in: -1. Current directory -2. Source repository (fetched from GitHub) - -### Custom Location - -Use the `CONFIG_FILE` environment variable: - -```bash -# Use custom config file -export CONFIG_FILE=my-config.yaml -./examples-copier - -# Use environment-specific config -export CONFIG_FILE=copier-config.production.yaml -./examples-copier -``` - -### Local vs Remote - -**Local File (for testing):** -```bash -# Create local config -cp configs/copier-config.example.yaml copier-config.yaml - -# Edit and test -vim copier-config.yaml -./examples-copier -``` - -**Remote File (for production):** -```bash -# Add config to source repository -git add copier-config.yaml -git commit -m "Add copier configuration" -git push origin main - -# Application fetches from GitHub -./examples-copier -``` - -## Troubleshooting - -### Config Not Found - -**Error:** -``` -[ERROR] failed to load config | {"error":"failed to retrieve config file: 404 Not Found"} -``` - -**Solutions:** -1. Create local config file: `copier-config.yaml` -2. Add config to source repository -3. Check `CONFIG_FILE` environment variable -4. Verify file name matches exactly - -### Invalid Pattern - -**Error:** -``` -Error: copy_rules[0].source_pattern: invalid regex pattern -``` - -**Solutions:** -1. Test pattern with `config-validator` -2. Check regex syntax -3. Escape special characters -4. Use raw strings for complex patterns - -### Path Transform Failed - -**Error:** -``` -[ERROR] failed to transform path | {"error":"variable not found: lang"} -``` - -**Solutions:** -1. Verify variable is extracted by pattern -2. Check variable name spelling -3. Test with `config-validator test-transform` -4. Use built-in variables if pattern variables unavailable - -### Validation Failed - -**Error:** -``` -Error: copy_rules[0]: name is required -``` - -**Solutions:** -1. Run `config-validator validate -config copier-config.yaml -v` -2. Check all required fields are present -3. Verify YAML syntax is correct -4. Check indentation (YAML is whitespace-sensitive) - -# Pattern Matching Cheat Sheet - -Quick reference for pattern matching in examples-copier. - -## Pattern Types at a Glance - -| Type | Use When | Example | Extracts Variables? | -|------------|---------------------------------------|---------------------------------|-------------------------------| -| **Prefix** | Simple directory matching | `examples/` | ✅ Yes (prefix, relative_path) | -| **Glob** | Wildcard matching | `**/*.go` | ❌ No | -| **Regex** | Complex patterns, variable extraction | `^examples/(?P[^/]+)/.*$` | ✅ Yes (custom) | - -## Prefix Patterns - -### Syntax -```yaml -source_pattern: - type: "prefix" - pattern: "examples/" -``` - -### Examples -| Pattern | Matches | Doesn't Match | -|-------------|-----------------------|------------------------| -| `examples/` | `examples/go/main.go` | `src/examples/test.go` | -| `src/` | `src/main.go` | `examples/src/test.go` | -| `docs/api/` | `docs/api/readme.md` | `docs/guide/api.md` | - -### Variables -- `${matched_prefix}` - The matched prefix -- `${relative_path}` - Path after the prefix - -## Glob Patterns - -### Wildcards -| Symbol | Matches | Example | -|--------|-------------------------|-----------------------------| -| `*` | Any characters (no `/`) | `*.go` → `main.go` | -| `**` | Any directories | `**/*.go` → `a/b/c/main.go` | -| `?` | Single character | `test?.go` → `test1.go` | - -### Examples -| Pattern | Matches | Doesn't Match | -|--------------------|------------------------|---------------| -| `*.go` | `main.go` | `src/main.go` | -| `**/*.go` | `a/b/c/main.go` | `main.py` | -| `examples/**/*.js` | `examples/node/app.js` | `src/app.js` | -| `test?.go` | `test1.go`, `testA.go` | `test12.go` | - -## Regex Patterns - -### Common Building Blocks - -| Pattern | Matches | Example | -|--------------|-----------------------------|------------------------| -| `[^/]+` | One or more non-slash chars | Directory or file name | -| `.+` | One or more any chars | Rest of path | -| `.*` | Zero or more any chars | Optional content | -| `[0-9]+` | One or more digits | Version numbers | -| `(foo\|bar)` | Either foo or bar | Specific values | -| `\.go$` | Ends with .go | File extension | -| `^examples/` | Starts with examples/ | Path prefix | - -### Named Capture Groups - -```regex -(?Ppattern) -``` - -**Example:** -```regex -^examples/(?P[^/]+)/(?P.+)$ -``` - -Extracts: -- `lang` from first directory -- `file` from rest of path - -### Common Patterns - -#### Language + File -```regex -^examples/(?P[^/]+)/(?P.+)$ -``` -- `examples/go/main.go` → `lang=go, file=main.go` - -#### Language + Category + File -```regex -^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$ -``` -- `examples/go/database/connect.go` → `lang=go, category=database, file=connect.go` - -#### Project + Rest -```regex -^generated-examples/(?P[^/]+)/(?P.+)$ -``` -- `generated-examples/app/cmd/main.go` → `project=app, rest=cmd/main.go` - -#### Version Support -```regex -^examples/(?P[^/]+)/(?Pv[0-9]+\\.x)/(?P.+)$ -``` -- `examples/node/v6.x/app.js` → `lang=node, version=v6.x, file=app.js` - -#### Type + Language + File -```regex -^source/examples/(?Pgenerated|manual)/(?P[^/]+)/(?P.+)$ -``` -- `source/examples/generated/node/app.js` → `type=generated, lang=node, file=app.js` - -## Path Transformation - -### Syntax -```yaml -path_transform: "docs/${lang}/${file}" -``` - -### Built-in Variables - -| Variable | Value for `examples/go/database/connect.go` | -|---------------|---------------------------------------------| -| `${path}` | `examples/go/database/connect.go` | -| `${filename}` | `connect.go` | -| `${dir}` | `examples/go/database` | -| `${ext}` | `.go` | -| `${name}` | `connect` | - -### Common Transformations - -| Transform | Input | Output | -|------------------------------------|--------------------------|----------------------------| -| `${path}` | `examples/go/main.go` | `examples/go/main.go` | -| `docs/${path}` | `examples/go/main.go` | `docs/examples/go/main.go` | -| `docs/${relative_path}` | `examples/go/main.go` | `docs/go/main.go` | -| `${lang}/${file}` | `examples/go/main.go` | `go/main.go` | -| `docs/${lang}/${category}/${file}` | `examples/go/db/conn.go` | `docs/go/db/conn.go` | - -## Complete Examples - -### Example 1: Simple Copy -```yaml -source_pattern: - type: "prefix" - pattern: "examples/" -targets: - - path_transform: "docs/${path}" -``` -**Result:** `examples/go/main.go` → `docs/examples/go/main.go` - -### Example 2: Language-Based -```yaml -source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P.+)$" -targets: - - path_transform: "docs/code-examples/${lang}/${file}" -``` -**Result:** `examples/go/main.go` → `docs/code-examples/go/main.go` - -### Example 3: Categorized -```yaml -source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$" -targets: - - path_transform: "docs/${lang}/${category}/${file}" -``` -**Result:** `examples/go/database/connect.go` → `docs/go/database/connect.go` - -### Example 4: Glob for Extensions -```yaml -source_pattern: - type: "glob" - pattern: "examples/**/*.go" -targets: - - path_transform: "docs/${path}" -``` -**Result:** `examples/go/auth/login.go` → `docs/examples/go/auth/login.go` - -### Example 5: Project-Based -```yaml -source_pattern: - type: "regex" - pattern: "^generated-examples/(?P[^/]+)/(?P.+)$" -targets: - - path_transform: "examples/${project}/${rest}" -``` -**Result:** `generated-examples/app/cmd/main.go` → `examples/app/cmd/main.go` - -## Testing Commands - -### Test Pattern -```bash -./config-validator test-pattern \ - -type regex \ - -pattern "^examples/(?P[^/]+)/(?P.+)$" \ - -file "examples/go/main.go" -``` - -### Test Transform -```bash -./config-validator test-transform \ - -source "examples/go/main.go" \ - -template "docs/${lang}/${file}" \ - -vars "lang=go,file=main.go" -``` - -### Validate Config -```bash -./config-validator validate -config copier-config.yaml -v -``` - -## Decision Tree - -``` -What do you need? -│ -├─ Copy entire directory tree -│ └─ Use PREFIX pattern -│ pattern: "examples/" -│ transform: "docs/${path}" -│ -├─ Match by file extension -│ └─ Use GLOB pattern -│ pattern: "**/*.go" -│ transform: "docs/${path}" -│ -├─ Extract language from path -│ └─ Use REGEX pattern -│ pattern: "^examples/(?P[^/]+)/(?P.+)$" -│ transform: "docs/${lang}/${file}" -│ -└─ Complex matching with multiple variables - └─ Use REGEX pattern - pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$" - transform: "docs/${lang}/${category}/${file}" -``` - -## Common Mistakes - -### ❌ Missing Anchors -```yaml -# Wrong - matches partial paths -pattern: "examples/(?P[^/]+)/(?P.+)" - -# Right - matches full path -pattern: "^examples/(?P[^/]+)/(?P.+)$" -``` - -### ❌ Wrong Character Class -```yaml -# Wrong - .+ matches slashes too -pattern: "^examples/(?P.+)/(?P.+)$" -# Right - [^/]+ doesn't match slashes -pattern: "^examples/(?P[^/]+)/(?P.+)$" -``` - -### ❌ Unnamed Groups -```yaml -# Wrong - doesn't extract variables -pattern: "^examples/([^/]+)/(.+)$" - -# Right - named groups extract variables -pattern: "^examples/(?P[^/]+)/(?P.+)$" -``` - -### ❌ Variable Name Mismatch -```yaml -# Pattern extracts "lang" -pattern: "^examples/(?P[^/]+)/(?P.+)$" - -# Wrong - uses "language" -path_transform: "docs/${language}/${file}" - -# Right - uses "lang" -path_transform: "docs/${lang}/${file}" -``` - -## Tips - -1. **Start simple** - Use prefix, then add regex when needed -2. **Test first** - Use `config-validator` before deploying -3. **Use anchors** - Always use `^` and `$` in regex -4. **Be specific** - Use `[^/]+` instead of `.+` for directories -5. **Name clearly** - Use descriptive variable names like `lang`, not `a` -6. **Check logs** - Look for "sample file path" to see actual paths - -## See Also - -- [Pattern Matching Guide](PATTERN-MATCHING-GUIDE.md) - Detailed pattern matching documentation -- [Quick Reference](../QUICK-REFERENCE.md) - Command reference -- [Deployment Guide](DEPLOYMENT.md) - Deploying the application -- [Architecture](ARCHITECTURE.md) - System architecture overview - ---- - -**Need Help?** -- See [Troubleshooting Guide](TROUBLESHOOTING.md) -- See [FAQ](FAQ.md) -- Check example configs in `configs/copier-config.example.yaml` - diff --git a/examples-copier/docs/DEPLOYMENT.md b/examples-copier/docs/DEPLOYMENT.md index f562129..6717bba 100644 --- a/examples-copier/docs/DEPLOYMENT.md +++ b/examples-copier/docs/DEPLOYMENT.md @@ -217,8 +217,8 @@ The `env-cloudrun.yaml` file contains environment variables for Cloud Run deploy ```bash cd examples-copier -# Copy from production template (if available) or create new -cp configs/env.yaml.production env-cloudrun.yaml +# Copy from example or create new +cp env.yaml env-cloudrun.yaml # Edit with your values nano env-cloudrun.yaml # or vim, code, etc. @@ -257,7 +257,8 @@ MONGO_URI_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/mongo-uri/versions/lates # ============================================================================= # PORT: "8080" # DO NOT SET - Cloud Run sets this automatically WEBSERVER_PATH: "/events" -CONFIG_FILE: "copier-config.yaml" +MAIN_CONFIG_FILE: ".copier/workflows/main.yaml" +USE_MAIN_CONFIG: "true" DEPRECATION_FILE: "deprecated_examples.json" # ============================================================================= @@ -539,7 +540,7 @@ db.audit_events.aggregate([ # Deployment Checklist -Quick reference checklist for deploying the GitHub Code Example Copier to Google Cloud App Engine. +Quick reference checklist for deploying the GitHub Code Example Copier to Google Cloud Run. ## 📋 Pre-Deployment @@ -694,13 +695,13 @@ env: flex ## 🚀 Deployment -### ☐ 8. Deploy to App Engine +### ☐ 8. Deploy to Cloud Run ```bash cd examples-copier -# Deploy (env.yaml is included via 'includes' directive in app.yaml) -gcloud app deploy app.yaml +# Deploy using the deployment script +./scripts/deploy-cloudrun.sh ``` **Expected output:** @@ -949,12 +950,13 @@ gcloud app deploy app.yaml ### Error: "Config file not found" -**Cause:** `copier-config.yaml` missing from source repository +**Cause:** Main config file missing from config repository **Fix:** ```bash -# Add copier-config.yaml to your source repository -# See documentation for config file format +# Add main config file to your config repository +# Default location: .copier/workflows/main.yaml +# See MAIN-CONFIG-README.md for format ``` --- @@ -964,7 +966,7 @@ gcloud app deploy app.yaml All items should be ✅: - ✅ Deployment completes without errors -- ✅ App Engine is running +- ✅ Cloud Run service is running - ✅ Health endpoint returns 200 OK - ✅ Logs show no secret loading errors - ✅ Webhook receives PR events @@ -1033,7 +1035,7 @@ gcloud app services set-traffic default --splits=PREVIOUS_VERSION=1 | "failed to load webhook secret" | Secret Manager access denied | Run `./grant-secret-access.sh` | | "webhook signature verification failed" | Secret mismatch | Verify secret matches GitHub webhook | | "MONGO_URI is required" | Audit enabled but no URI | Set `MONGO_URI_SECRET_NAME` or disable audit | -| "Config file not found" | Missing copier-config.yaml | Add config file to source repo | +| "Config file not found" | Missing main config | Add main config to config repo | ### Quick Fixes diff --git a/examples-copier/docs/DEPRECATION-TRACKING-EXPLAINED.md b/examples-copier/docs/DEPRECATION-TRACKING-EXPLAINED.md index 8202797..c2d22bc 100644 --- a/examples-copier/docs/DEPRECATION-TRACKING-EXPLAINED.md +++ b/examples-copier/docs/DEPRECATION-TRACKING-EXPLAINED.md @@ -54,13 +54,8 @@ Check deprecation queue ### ✅ YES - Protected Against Blank Commits -**Both implementations now have built-in protection** against blank commits! ✅ +The implementation has built-in protection against blank commits: -### ✅ Legacy Implementation - FIXED - -The legacy implementation (`UpdateDeprecationFile()`) has been **updated with blank commit protection**: - - ````go func UpdateDeprecationFile() { // ✅ Early return if there are no files to deprecate - prevents blank commits @@ -70,55 +65,8 @@ func UpdateDeprecationFile() { } // ... rest of update logic ... - - for key, value := range FilesToDeprecate { - newDeprecatedFileEntry := DeprecatedFileEntry{ - FileName: key, - Repo: value.TargetRepo, - Branch: value.TargetBranch, - DeletedOn: time.Now().Format(time.RFC3339), - } - deprecationFile = append(deprecationFile, newDeprecatedFileEntry) - } - - // Only reached if FilesToDeprecate has entries - uploadDeprecationFileChanges(message, string(updatedJSON)) - - LogInfo(fmt.Sprintf("Successfully updated %s with %d entries", - os.Getenv(configs.DeprecationFile), len(FilesToDeprecate))) } ```` - - -**Protection:** -- ✅ Checks if `FilesToDeprecate` is empty -- ✅ Returns early if nothing to deprecate -- ✅ **No commit is made** if queue is empty -- ✅ Logs skip message for visibility -- ✅ Logs success message with entry count - -**Fix Applied:** 2025-10-08 (see [BLANK-COMMIT-FIX.md](../BLANK-COMMIT-FIX.md)) - -### ✅ New Implementation - Already Protected - -The newer implementation has always had proper protection: - -````go -func UpdateDeprecationFileWithContextAndConfig(...) { - filesToDeprecate := fileStateService.GetFilesToDeprecate() - - // ✅ PROTECTION: Early return if nothing to deprecate - if len(filesToDeprecate) == 0 { - LogConfigOperation(ctx, "skip_update", config.DeprecationFile, - "No deprecated files to record; skipping deprecation file update", nil) - return // ← NO COMMIT MADE - } - - // Only continues if there are files to deprecate - // ... rest of update logic ... -} -```` - **Protection:** - ✅ Checks if deprecation queue is empty @@ -156,21 +104,24 @@ The deprecation file is a JSON array stored in the **source repository**: ## Configuration -Enable deprecation tracking in your config: +Enable deprecation tracking in your workflow config: ```yaml -copy_rules: - - name: "Go examples" - source_pattern: - type: "prefix" - pattern: "examples/go" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/${relative_path}" - deprecation_check: - enabled: true # ← Enable tracking - file: "deprecated_examples.json" # ← Optional: custom filename +workflows: + - name: "Copy Go examples" + source: + repo: "mongodb/source-repo" + branch: "main" + destination: + repo: "mongodb/docs" + branch: "main" + transformations: + - move: + from: "examples/go" + to: "code" + deprecation_check: + enabled: true # ← Enable tracking + file: "deprecated_examples.json" # ← Optional: custom filename ``` **Options:** diff --git a/examples-copier/docs/FAQ.md b/examples-copier/docs/FAQ.md index 486a1f7..e26c4c5 100644 --- a/examples-copier/docs/FAQ.md +++ b/examples-copier/docs/FAQ.md @@ -33,48 +33,48 @@ Examples-copier is a GitHub app that automatically copies code examples and file ## Configuration -### Do I need to use YAML configuration? +### Can I use multiple transformations? -No. The app supports both YAML and legacy JSON configurations. YAML is recommended for new deployments because it supports advanced features like pattern matching and path transformations. - -### Can I use multiple patterns? - -Yes! You can define multiple copy rules, each with its own pattern and targets: +Yes! You can define multiple workflows, each with its own transformations: ```yaml -copy_rules: +workflows: - name: "Go examples" - source_pattern: - type: "regex" - pattern: "^examples/go/(?P.+)$" - targets: [...] - + source: + repo: "owner/source" + branch: "main" + destination: + repo: "owner/target" + branch: "main" + transformations: + - regex: + pattern: "^examples/go/(?P.+)$" + transform: "code/go/${file}" + - name: "Python examples" - source_pattern: - type: "regex" - pattern: "^examples/python/(?P.+)$" - targets: [...] + source: + repo: "owner/source" + branch: "main" + destination: + repo: "owner/target" + branch: "main" + transformations: + - regex: + pattern: "^examples/python/(?P.+)$" + transform: "code/python/${file}" ``` -### Can one file match multiple rules? +### Can one file match multiple workflows? -Yes. A file can match multiple rules and be copied to multiple targets. This is useful for copying the same file to different repositories or branches. +Yes. A file can match multiple workflows and be copied to multiple targets. This is useful for copying the same file to different repositories or branches. ### Where should I store the config file? -**For production:** Store `copier-config.yaml` in your source repository (the repo being monitored for PRs). - -**For local testing:** Store `copier-config.yaml` in the examples-copier directory and set `CONFIG_FILE=copier-config.yaml`. +**Main config:** Store in a central config repository and set `MAIN_CONFIG_FILE` in env.yaml. -### How do I migrate from JSON to YAML? - -Use the config-validator tool: - -```bash -./config-validator convert -input config.json -output copier-config.yaml -``` +**Workflow configs:** Store in `.copier/workflows/config.yaml` in source repositories, or reference them from the main config. -The tool will automatically convert your legacy JSON configuration to the new YAML format while preserving all settings. +**For local testing:** Store config files in the examples-copier directory and set appropriate environment variables. ## Pattern Matching @@ -337,27 +337,6 @@ commit_strategy: - `${commit_sha}` - Commit SHA - Plus any variables extracted from pattern matching -### How do I batch multiple rules into one PR? - -Use `batch_by_repo: true` to combine all changes into one PR per target repository: - -```yaml -batch_by_repo: true - -batch_pr_config: - pr_title: "Update from ${source_repo}" - pr_body: | - 🤖 Automated update - Files: ${file_count} # Accurate count across all rules - use_pr_template: true - commit_message: "Update from ${source_repo} PR #${pr_number}" -``` - -**Benefits:** -- Single PR per target repo instead of multiple PRs -- Accurate `${file_count}` across all matched rules -- Easier review for related changes - ### How do I use PR templates from target repos? Set `use_pr_template: true` in your commit strategy or batch config: diff --git a/examples-copier/docs/LOCAL-TESTING.md b/examples-copier/docs/LOCAL-TESTING.md index d842b82..06ee239 100644 --- a/examples-copier/docs/LOCAL-TESTING.md +++ b/examples-copier/docs/LOCAL-TESTING.md @@ -48,7 +48,8 @@ For basic local testing, you only need: # configs/.env COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true -CONFIG_FILE=config.json +MAIN_CONFIG_FILE=.copier/workflows/main.yaml +USE_MAIN_CONFIG=true ``` ### 3. For Testing with Real PRs @@ -144,8 +145,8 @@ export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-s ### Scenario 1: Test Configuration Changes ```bash -# 1. Edit your config file -nano config.json # or copier-config.yaml +# 1. Edit your main config file +nano .copier/workflows/main.yaml # 2. Validate it ./config-validator validate -config config.json -v @@ -278,7 +279,8 @@ DRY_RUN=true # Don't make actual commits LOG_LEVEL=debug # Detailed logging COPIER_DEBUG=true # Extra debug info METRICS_ENABLED=true # Enable /metrics endpoint -CONFIG_FILE=config.json # Your config file +MAIN_CONFIG_FILE=.copier/workflows/main.yaml # Your main config file +USE_MAIN_CONFIG=true # Enable main config system ``` ### Optional (for Real PR Testing) @@ -449,7 +451,7 @@ curl http://localhost:8080/metrics | jq # Check health curl http://localhost:8080/health | jq -# Validate config -./config-validator validate -config copier-config.yaml -v +# Validate config (if using legacy config validator) +# Note: Main config validation is built into the app ``` diff --git a/examples-copier/docs/PATTERN-MATCHING-GUIDE.md b/examples-copier/docs/PATTERN-MATCHING-GUIDE.md index 564b4dc..18fe427 100644 --- a/examples-copier/docs/PATTERN-MATCHING-GUIDE.md +++ b/examples-copier/docs/PATTERN-MATCHING-GUIDE.md @@ -559,21 +559,23 @@ gh pr view 42 --json files --jq '.files[].path' -file "examples/go/database/connect.go" ``` -### 5. Order Rules from Specific to General +### 5. Order Workflows from Specific to General ```yaml -copy_rules: - # More specific rule first +workflows: + # More specific workflow first - name: "Copy Go examples" - source_pattern: - type: "regex" - pattern: "^examples/go/(?P.+)$" - - # General fallback rule last + transformations: + - regex: + pattern: "^examples/go/(?P.+)$" + transform: "code/go/${file}" + + # General fallback workflow last - name: "Copy all examples" - source_pattern: - type: "prefix" - pattern: "examples/" + transformations: + - move: + from: "examples" + to: "code" ``` ### 6. Use Descriptive Variable Names @@ -666,23 +668,25 @@ pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$" ### Files Matched Multiple Times -**Problem:** Same file matches multiple rules +**Problem:** Same file matches multiple workflows -**Solution:** This is expected! Files can match multiple rules and be copied to multiple targets. If you want only one rule to match, make patterns mutually exclusive: +**Solution:** This is expected! Files can match multiple workflows and be copied to multiple targets. If you want only one workflow to match, make transformations mutually exclusive: ```yaml -copy_rules: +workflows: # Only Go files - name: "Go examples" - source_pattern: - type: "regex" - pattern: "^examples/go/(?P.+)$" - + transformations: + - regex: + pattern: "^examples/go/(?P.+)$" + transform: "code/go/${file}" + # Only Python files (won't match Go) - name: "Python examples" - source_pattern: - type: "regex" - pattern: "^examples/python/(?P.+)$" + transformations: + - regex: + pattern: "^examples/python/(?P.+)$" + transform: "code/python/${file}" ``` ## Advanced Examples diff --git a/examples-copier/docs/multi-source/MULTI-SOURCE-IMPLEMENTATION-PLAN.md b/examples-copier/docs/multi-source/MULTI-SOURCE-IMPLEMENTATION-PLAN.md deleted file mode 100644 index e3e0bde..0000000 --- a/examples-copier/docs/multi-source/MULTI-SOURCE-IMPLEMENTATION-PLAN.md +++ /dev/null @@ -1,514 +0,0 @@ -# Multi-Source Repository Support - Implementation Plan - -## Executive Summary - -This document outlines the implementation plan for adding support for multiple source repositories to the examples-copier application. Currently, the application supports only a single source repository defined in the configuration. This enhancement will allow the copier to monitor and process webhooks from multiple source repositories, each with their own copy rules and configurations. - -## Current Architecture Analysis - -### Current Limitations - -1. **Single Source Repository**: The configuration schema (`YAMLConfig`) has a single `source_repo` and `source_branch` field at the root level -2. **Hardcoded Repository Context**: Environment variables `REPO_OWNER` and `REPO_NAME` are set globally and used throughout the codebase -3. **Webhook Validation**: The webhook handler validates that incoming webhooks match the configured `source_repo` (lines 228-236 in `webhook_handler_new.go`) -4. **Config File Location**: Configuration is fetched from the single source repository defined in environment variables -5. **GitHub App Installation**: Single installation ID is configured globally - -### Current Flow - -``` -Webhook Received → Validate Source Repo → Load Config from Source Repo → Process Files → Copy to Targets -``` - -## Proposed Architecture - -### New Multi-Source Flow - -``` -Webhook Received → Identify Source Repo → Load Config for That Source → Process Files → Copy to Targets -``` - -### Key Design Decisions - -1. **Configuration Storage**: Support both centralized (single config file) and distributed (per-repo config) approaches -2. **Backward Compatibility**: Maintain support for existing single-source configurations -3. **GitHub App Installations**: Support multiple installation IDs for different organizations -4. **Config Discovery**: Allow configs to be stored in a central location or in each source repository - -## Implementation Tasks - -### 1. Configuration Schema Updates - -**Files to Modify:** -- `types/config.go` -- `configs/copier-config.example.yaml` - -**Changes:** - -#### Option A: Centralized Multi-Source Config (Recommended) -```yaml -# New schema supporting multiple sources -sources: - - repo: "mongodb/docs-code-examples" - branch: "main" - installation_id: "12345678" # Optional, falls back to default - copy_rules: - - name: "go-examples" - source_pattern: - type: "prefix" - pattern: "examples/go/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/${path}" - commit_strategy: - type: "direct" - - - repo: "mongodb/atlas-examples" - branch: "main" - installation_id: "87654321" # Different installation for different org - copy_rules: - - name: "atlas-cli-examples" - source_pattern: - type: "glob" - pattern: "cli/**/*.go" - targets: - - repo: "mongodb/atlas-cli" - branch: "main" - path_transform: "examples/${filename}" - commit_strategy: - type: "pull_request" - pr_title: "Update examples" - auto_merge: false - -# Global defaults (optional) -defaults: - commit_strategy: - type: "pull_request" - auto_merge: false - deprecation_check: - enabled: true - file: "deprecated_examples.json" -``` - -#### Option B: Backward Compatible (Single Source at Root) -```yaml -# Backward compatible - if source_repo exists at root, treat as single source -source_repo: "mongodb/docs-code-examples" -source_branch: "main" -copy_rules: - - name: "example" - # ... existing structure - -# OR use new multi-source structure -sources: - - repo: "mongodb/docs-code-examples" - # ... as above -``` - -**New Types:** -```go -// MultiSourceConfig represents configuration for multiple source repositories -type MultiSourceConfig struct { - Sources []SourceConfig `yaml:"sources" json:"sources"` - Defaults *DefaultsConfig `yaml:"defaults,omitempty" json:"defaults,omitempty"` -} - -// SourceConfig represents a single source repository configuration -type SourceConfig struct { - Repo string `yaml:"repo" json:"repo"` - Branch string `yaml:"branch" json:"branch"` - InstallationID string `yaml:"installation_id,omitempty" json:"installation_id,omitempty"` - ConfigFile string `yaml:"config_file,omitempty" json:"config_file,omitempty"` // For distributed configs - CopyRules []CopyRule `yaml:"copy_rules" json:"copy_rules"` -} - -// DefaultsConfig provides default values for all sources -type DefaultsConfig struct { - CommitStrategy *CommitStrategyConfig `yaml:"commit_strategy,omitempty" json:"commit_strategy,omitempty"` - DeprecationCheck *DeprecationConfig `yaml:"deprecation_check,omitempty" json:"deprecation_check,omitempty"` -} - -// Update YAMLConfig to support both formats -type YAMLConfig struct { - // Legacy single-source fields (for backward compatibility) - SourceRepo string `yaml:"source_repo,omitempty" json:"source_repo,omitempty"` - SourceBranch string `yaml:"source_branch,omitempty" json:"source_branch,omitempty"` - CopyRules []CopyRule `yaml:"copy_rules,omitempty" json:"copy_rules,omitempty"` - - // New multi-source fields - Sources []SourceConfig `yaml:"sources,omitempty" json:"sources,omitempty"` - Defaults *DefaultsConfig `yaml:"defaults,omitempty" json:"defaults,omitempty"` -} -``` - -### 2. Configuration Loading & Validation - -**Files to Modify:** -- `services/config_loader.go` - -**Changes:** - -1. **Add Config Discovery Method**: -```go -// ConfigDiscovery determines where to load config from -type ConfigDiscovery interface { - // DiscoverConfig finds the config for a given source repository - DiscoverConfig(ctx context.Context, repoOwner, repoName string) (*SourceConfig, error) -} -``` - -2. **Update LoadConfig Method**: -```go -// LoadConfigForSource loads configuration for a specific source repository -func (cl *DefaultConfigLoader) LoadConfigForSource(ctx context.Context, repoOwner, repoName string, config *configs.Config) (*SourceConfig, error) { - // Load the main config (centralized or from the source repo) - yamlConfig, err := cl.LoadConfig(ctx, config) - if err != nil { - return nil, err - } - - // Find the matching source configuration - sourceRepo := fmt.Sprintf("%s/%s", repoOwner, repoName) - sourceConfig := findSourceConfig(yamlConfig, sourceRepo) - if sourceConfig == nil { - return nil, fmt.Errorf("no configuration found for source repository: %s", sourceRepo) - } - - return sourceConfig, nil -} - -// findSourceConfig searches for a source repo in the config -func findSourceConfig(config *YAMLConfig, sourceRepo string) *SourceConfig { - // Check if using legacy single-source format - if config.SourceRepo != "" && config.SourceRepo == sourceRepo { - return &SourceConfig{ - Repo: config.SourceRepo, - Branch: config.SourceBranch, - CopyRules: config.CopyRules, - } - } - - // Search in multi-source format - for _, source := range config.Sources { - if source.Repo == sourceRepo { - return &source - } - } - - return nil -} -``` - -3. **Add Validation for Multi-Source**: -```go -func (c *YAMLConfig) Validate() error { - // Check if using legacy or new format - isLegacy := c.SourceRepo != "" - isMultiSource := len(c.Sources) > 0 - - if isLegacy && isMultiSource { - return fmt.Errorf("cannot use both legacy (source_repo) and new (sources) format") - } - - if !isLegacy && !isMultiSource { - return fmt.Errorf("must specify either source_repo or sources") - } - - if isLegacy { - return c.validateLegacyFormat() - } - - return c.validateMultiSourceFormat() -} - -func (c *YAMLConfig) validateMultiSourceFormat() error { - if len(c.Sources) == 0 { - return fmt.Errorf("at least one source repository is required") - } - - // Check for duplicate source repos - seen := make(map[string]bool) - for i, source := range c.Sources { - if source.Repo == "" { - return fmt.Errorf("sources[%d]: repo is required", i) - } - if seen[source.Repo] { - return fmt.Errorf("sources[%d]: duplicate source repository: %s", i, source.Repo) - } - seen[source.Repo] = true - - if err := validateSourceConfig(&source); err != nil { - return fmt.Errorf("sources[%d]: %w", i, err) - } - } - - return nil -} -``` - -### 3. Webhook Routing Logic - -**Files to Modify:** -- `services/webhook_handler_new.go` -- `services/github_auth.go` - -**Changes:** - -1. **Update Webhook Handler**: -```go -// handleMergedPRWithContainer processes a merged PR using the new pattern matching system -func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommitSHA string, repoOwner string, repoName string, config *configs.Config, container *ServiceContainer) { - startTime := time.Now() - - // Configure GitHub permissions for the source repository - if InstallationAccessToken == "" { - ConfigurePermissions() - } - - // Update config with actual repository from webhook - config.RepoOwner = repoOwner - config.RepoName = repoName - - // Load configuration for this specific source repository - sourceConfig, err := container.ConfigLoader.LoadConfigForSource(ctx, repoOwner, repoName, config) - if err != nil { - LogAndReturnError(ctx, "config_load", fmt.Sprintf("no configuration found for source repo %s/%s", repoOwner, repoName), err) - container.MetricsCollector.RecordWebhookFailed() - - container.SlackNotifier.NotifyError(ctx, &ErrorEvent{ - Operation: "config_load", - Error: err, - PRNumber: prNumber, - SourceRepo: fmt.Sprintf("%s/%s", repoOwner, repoName), - }) - return - } - - // Switch GitHub installation if needed - if sourceConfig.InstallationID != "" && sourceConfig.InstallationID != config.InstallationId { - if err := switchGitHubInstallation(sourceConfig.InstallationID); err != nil { - LogAndReturnError(ctx, "installation_switch", "failed to switch GitHub installation", err) - container.MetricsCollector.RecordWebhookFailed() - return - } - } - - // Continue with existing processing logic... - // Process files with pattern matching for this source - processFilesWithPatternMatching(ctx, prNumber, sourceCommitSHA, changedFiles, sourceConfig, config, container) -} -``` - -2. **Add Installation Switching**: -```go -// switchGitHubInstallation switches to a different GitHub App installation -func switchGitHubInstallation(installationID string) error { - // Save current installation ID - previousInstallationID := os.Getenv(configs.InstallationId) - - // Set new installation ID - os.Setenv(configs.InstallationId, installationID) - - // Clear cached token to force re-authentication - InstallationAccessToken = "" - - // Re-configure permissions with new installation - ConfigurePermissions() - - LogInfo(fmt.Sprintf("Switched GitHub installation from %s to %s", previousInstallationID, installationID)) - return nil -} -``` - -### 4. GitHub App Installation Support - -**Files to Modify:** -- `configs/environment.go` -- `services/github_auth.go` - -**Changes:** - -1. **Support Multiple Installation IDs**: -```go -// Config struct update -type Config struct { - // ... existing fields - - // Multi-installation support - InstallationId string // Default installation ID - InstallationMapping map[string]string // Map of repo -> installation_id -} - -// Load installation mapping from environment or config -func (c *Config) GetInstallationID(repo string) string { - if id, ok := c.InstallationMapping[repo]; ok { - return id - } - return c.InstallationId // fallback to default -} -``` - -2. **Update Authentication**: -```go -// ConfigurePermissionsForRepo configures GitHub permissions for a specific repository -func ConfigurePermissionsForRepo(installationID string) error { - if installationID == "" { - return fmt.Errorf("installation ID is required") - } - - // Use the provided installation ID - token, err := generateInstallationToken(installationID) - if err != nil { - return fmt.Errorf("failed to generate installation token: %w", err) - } - - InstallationAccessToken = token - return nil -} -``` - -### 5. Metrics & Audit Logging Updates - -**Files to Modify:** -- `services/health_metrics.go` -- `services/audit_logger.go` - -**Changes:** - -1. **Add Source Repository to Metrics**: -```go -// MetricsCollector update -type MetricsCollector struct { - // ... existing fields - - // Per-source metrics - webhooksBySource map[string]int64 - filesBySource map[string]int64 - uploadsBySource map[string]int64 - mu sync.RWMutex -} - -func (mc *MetricsCollector) RecordWebhookReceivedForSource(sourceRepo string) { - mc.mu.Lock() - defer mc.mu.Unlock() - mc.webhooksReceived++ - mc.webhooksBySource[sourceRepo]++ -} - -func (mc *MetricsCollector) GetMetricsBySource() map[string]SourceMetrics { - mc.mu.RLock() - defer mc.mu.RUnlock() - - result := make(map[string]SourceMetrics) - for source := range mc.webhooksBySource { - result[source] = SourceMetrics{ - Webhooks: mc.webhooksBySource[source], - Files: mc.filesBySource[source], - Uploads: mc.uploadsBySource[source], - } - } - return result -} -``` - -2. **Update Audit Events**: -```go -// AuditEvent already has SourceRepo field, just ensure it's populated correctly -// in all logging calls with the actual source repository -``` - -### 6. Documentation Updates - -**Files to Create/Modify:** -- `docs/MULTI-SOURCE-GUIDE.md` (new) -- `docs/CONFIGURATION-GUIDE.md` (update) -- `README.md` (update) -- `configs/copier-config.example.yaml` (update with multi-source example) - -### 7. Testing & Validation - -**Files to Create:** -- `services/config_loader_multi_test.go` -- `services/webhook_handler_multi_test.go` -- `test-payloads/multi-source-webhook.json` - -**Test Scenarios:** -1. Load multi-source configuration -2. Validate configuration with multiple sources -3. Route webhook to correct source configuration -4. Handle missing source repository gracefully -5. Switch between GitHub installations -6. Backward compatibility with single-source configs - -### 8. Migration Guide & Backward Compatibility - -**Backward Compatibility Strategy:** - -1. **Auto-detect Format**: Check if `source_repo` exists at root level -2. **Convert Legacy to New**: Internally convert single-source to multi-source format -3. **Validation**: Ensure both formats validate correctly -4. **Migration Tool**: Provide CLI command to convert configs - -```bash -# Convert legacy config to multi-source format -./config-validator convert-to-multi-source -input copier-config.yaml -output copier-config-multi.yaml -``` - -## Implementation Phases - -### Phase 1: Core Infrastructure (Week 1) -- [ ] Update configuration schema -- [ ] Implement config loading for multiple sources -- [ ] Add validation for multi-source configs -- [ ] Ensure backward compatibility - -### Phase 2: Webhook Routing (Week 2) -- [ ] Implement webhook routing logic -- [ ] Add GitHub installation switching -- [ ] Update authentication handling -- [ ] Test with multiple source repos - -### Phase 3: Observability (Week 3) -- [ ] Update metrics collection -- [ ] Enhance audit logging -- [ ] Add per-source monitoring -- [ ] Update health endpoints - -### Phase 4: Documentation & Testing (Week 4) -- [ ] Write comprehensive documentation -- [ ] Create migration guide -- [ ] Add unit and integration tests -- [ ] Perform end-to-end testing - -## Risks & Mitigation - -### Risk 1: Breaking Changes -**Mitigation**: Maintain full backward compatibility with legacy single-source format - -### Risk 2: GitHub Rate Limits -**Mitigation**: Implement per-source rate limiting and monitoring - -### Risk 3: Configuration Complexity -**Mitigation**: Provide clear examples, templates, and validation tools - -### Risk 4: Installation Token Management -**Mitigation**: Implement proper token caching and refresh logic per installation - -## Success Criteria - -1. ✅ Support multiple source repositories in a single deployment -2. ✅ Maintain 100% backward compatibility with existing configs -3. ✅ No performance degradation for single-source use cases -4. ✅ Clear documentation and migration path -5. ✅ Comprehensive test coverage (>80%) -6. ✅ Successful deployment with 2+ source repositories - -## Future Enhancements - -1. **Dynamic Config Reloading**: Reload configuration without restart -2. **Per-Source Webhooks**: Different webhook endpoints for different sources -3. **Source Repository Discovery**: Auto-discover repositories with copier configs -4. **Config Validation API**: REST API for validating configurations -5. **Multi-Tenant Support**: Support multiple organizations with isolated configs - diff --git a/examples-copier/docs/multi-source/MULTI-SOURCE-MIGRATION-GUIDE.md b/examples-copier/docs/multi-source/MULTI-SOURCE-MIGRATION-GUIDE.md deleted file mode 100644 index 94ac0a3..0000000 --- a/examples-copier/docs/multi-source/MULTI-SOURCE-MIGRATION-GUIDE.md +++ /dev/null @@ -1,435 +0,0 @@ -# Migration Guide: Single Source to Multi-Source Configuration - -This guide helps you migrate from the legacy single-source configuration format to the new multi-source format. - -## Table of Contents - -- [Overview](#overview) -- [Backward Compatibility](#backward-compatibility) -- [Migration Steps](#migration-steps) -- [Configuration Comparison](#configuration-comparison) -- [Testing Your Migration](#testing-your-migration) -- [Rollback Plan](#rollback-plan) -- [FAQ](#faq) - -## Overview - -The multi-source feature allows the examples-copier to monitor and process webhooks from multiple source repositories in a single deployment. This eliminates the need to run separate copier instances for different source repositories. - -### Benefits of Multi-Source - -- **Simplified Deployment**: One instance handles multiple source repositories -- **Centralized Configuration**: Manage all copy rules in one place -- **Better Resource Utilization**: Shared infrastructure for all sources -- **Consistent Monitoring**: Unified metrics and audit logging -- **Cross-Organization Support**: Handle repos from different GitHub organizations - -## Backward Compatibility - -**Good News**: The new multi-source format is 100% backward compatible with existing configurations. - -- ✅ Existing single-source configs continue to work without changes -- ✅ No breaking changes to the configuration schema -- ✅ Automatic detection of legacy vs. new format -- ✅ Gradual migration path available - -## Migration Steps - -### Step 1: Assess Your Current Setup - -First, identify all the source repositories you're currently monitoring: - -```bash -# List all your current copier deployments -# Each deployment typically monitors one source repository -``` - -**Example Current State:** -- Deployment 1: Monitors `mongodb/docs-code-examples` -- Deployment 2: Monitors `mongodb/atlas-examples` -- Deployment 3: Monitors `10gen/internal-examples` - -### Step 2: Backup Current Configuration - -```bash -# Backup your current configuration -cp copier-config.yaml copier-config.yaml.backup - -# Backup environment variables -cp .env .env.backup -``` - -### Step 3: Convert Configuration Format - -#### Option A: Manual Conversion - -**Before (Single Source):** -```yaml -source_repo: "mongodb/docs-code-examples" -source_branch: "main" - -copy_rules: - - name: "go-examples" - source_pattern: - type: "prefix" - pattern: "examples/go/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/go/${path}" - commit_strategy: - type: "pull_request" - pr_title: "Update Go examples" - auto_merge: false -``` - -**After (Multi-Source):** -```yaml -sources: - - repo: "mongodb/docs-code-examples" - branch: "main" - # Optional: Add installation_id if different from default - # installation_id: "12345678" - - copy_rules: - - name: "go-examples" - source_pattern: - type: "prefix" - pattern: "examples/go/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/go/${path}" - commit_strategy: - type: "pull_request" - pr_title: "Update Go examples" - auto_merge: false -``` - -#### Option B: Automated Conversion (Recommended) - -Use the config-validator tool to automatically convert your configuration: - -```bash -# Convert single-source to multi-source format -./config-validator convert-to-multi-source \ - -input copier-config.yaml \ - -output copier-config-multi.yaml - -# Validate the new configuration -./config-validator validate -config copier-config-multi.yaml -v -``` - -### Step 4: Consolidate Multiple Deployments - -If you have multiple copier deployments, consolidate them into one multi-source config: - -```yaml -sources: - # Source 1: From deployment 1 - - repo: "mongodb/docs-code-examples" - branch: "main" - installation_id: "12345678" - copy_rules: - # ... copy rules from deployment 1 - - # Source 2: From deployment 2 - - repo: "mongodb/atlas-examples" - branch: "main" - installation_id: "87654321" - copy_rules: - # ... copy rules from deployment 2 - - # Source 3: From deployment 3 - - repo: "10gen/internal-examples" - branch: "main" - installation_id: "11223344" - copy_rules: - # ... copy rules from deployment 3 -``` - -### Step 5: Update Environment Variables - -Update your `.env` file to support multiple installations: - -```bash -# Before (single installation) -INSTALLATION_ID=12345678 - -# After (default installation + optional per-source) -INSTALLATION_ID=12345678 # Default/fallback installation ID - -# Note: Per-source installation IDs are now in the config file -# under each source's installation_id field -``` - -### Step 6: Update GitHub App Installations - -Ensure your GitHub App is installed on all source repositories: - -1. Go to your GitHub App settings -2. Install the app on each source repository's organization -3. Note the installation ID for each organization -4. Add installation IDs to your config file - -```bash -# Get installation IDs -curl -H "Authorization: Bearer YOUR_JWT_TOKEN" \ - https://api.github.com/app/installations -``` - -### Step 7: Validate Configuration - -Before deploying, validate your new configuration: - -```bash -# Validate configuration syntax and logic -./config-validator validate -config copier-config-multi.yaml -v - -# Test pattern matching -./config-validator test-pattern \ - -config copier-config-multi.yaml \ - -source "mongodb/docs-code-examples" \ - -file "examples/go/main.go" - -# Dry-run test -./examples-copier -config copier-config-multi.yaml -dry-run -``` - -### Step 8: Deploy and Test - -1. **Deploy to staging first**: -```bash -# Deploy to staging environment -gcloud app deploy --project=your-staging-project -``` - -2. **Test with real webhooks**: -```bash -# Use the test-webhook tool -./test-webhook -config copier-config-multi.yaml \ - -payload test-payloads/example-pr-merged.json -``` - -3. **Monitor logs**: -```bash -# Watch application logs -gcloud app logs tail -s default -``` - -4. **Verify metrics**: -```bash -# Check health endpoint -curl https://your-app.appspot.com/health - -# Check metrics endpoint -curl https://your-app.appspot.com/metrics -``` - -### Step 9: Production Deployment - -Once validated in staging: - -```bash -# Deploy to production -gcloud app deploy --project=your-production-project - -# Monitor for issues -gcloud app logs tail -s default --project=your-production-project -``` - -### Step 10: Decommission Old Deployments - -After confirming the multi-source deployment works: - -1. Monitor for 24-48 hours -2. Verify all source repositories are being processed -3. Check audit logs for any errors -4. Decommission old single-source deployments - -## Configuration Comparison - -### Single Source (Legacy) - -```yaml -source_repo: "mongodb/docs-code-examples" -source_branch: "main" - -copy_rules: - - name: "example-rule" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/${path}" - commit_strategy: - type: "direct" -``` - -### Multi-Source (New) - -```yaml -sources: - - repo: "mongodb/docs-code-examples" - branch: "main" - installation_id: "12345678" # Optional - copy_rules: - - name: "example-rule" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/${path}" - commit_strategy: - type: "direct" - -# Optional: Global defaults -defaults: - commit_strategy: - type: "pull_request" - auto_merge: false - deprecation_check: - enabled: true -``` - -### Hybrid (Both Formats Supported) - -The application automatically detects which format you're using: - -```go -// Automatic detection logic -if config.SourceRepo != "" { - // Legacy single-source format - processSingleSource(config) -} else if len(config.Sources) > 0 { - // New multi-source format - processMultiSource(config) -} -``` - -## Testing Your Migration - -### Test Checklist - -- [ ] Configuration validates successfully -- [ ] Pattern matching works for all sources -- [ ] Path transformations are correct -- [ ] Webhooks route to correct source config -- [ ] GitHub authentication works for all installations -- [ ] Files are copied to correct target repositories -- [ ] Deprecation tracking works (if enabled) -- [ ] Metrics show data for all sources -- [ ] Audit logs contain source repository info -- [ ] Slack notifications work (if enabled) - -### Test Commands - -```bash -# 1. Validate configuration -./config-validator validate -config copier-config-multi.yaml -v - -# 2. Test pattern matching for each source -./config-validator test-pattern \ - -config copier-config-multi.yaml \ - -source "mongodb/docs-code-examples" \ - -file "examples/go/main.go" - -# 3. Dry-run mode -DRY_RUN=true ./examples-copier -config copier-config-multi.yaml - -# 4. Test with webhook payload -./test-webhook -config copier-config-multi.yaml \ - -payload test-payloads/multi-source-webhook.json - -# 5. Check health -curl http://localhost:8080/health - -# 6. Check metrics -curl http://localhost:8080/metrics -``` - -## Rollback Plan - -If you encounter issues after migration: - -### Quick Rollback - -```bash -# 1. Restore backup configuration -cp copier-config.yaml.backup copier-config.yaml -cp .env.backup .env - -# 2. Redeploy previous version -gcloud app deploy --version=previous-version - -# 3. Route traffic back -gcloud app services set-traffic default --splits=previous-version=1 -``` - -### Gradual Rollback - -```bash -# Route 50% traffic to old version -gcloud app services set-traffic default \ - --splits=new-version=0.5,previous-version=0.5 - -# Monitor and adjust as needed -``` - -## FAQ - -### Q: Do I need to migrate immediately? - -**A:** No. The legacy single-source format is fully supported and will continue to work. Migrate when you need to monitor multiple source repositories or want to consolidate deployments. - -### Q: Can I mix legacy and new formats? - -**A:** No. Each configuration file must use either the legacy format OR the new format, not both. However, you can have different deployments using different formats. - -### Q: What happens if I don't specify installation_id? - -**A:** The application will use the default `INSTALLATION_ID` from environment variables. This works fine if all your source repositories are in the same organization. - -### Q: Can I gradually migrate one source at a time? - -**A:** Yes. You can start with one source in the new format and add more sources over time. Keep your old deployments running until all sources are migrated. - -### Q: How do I test without affecting production? - -**A:** Use dry-run mode (`DRY_RUN=true`) to test configuration without making actual commits. Also test in a staging environment first. - -### Q: What if a webhook comes from an unknown source? - -**A:** The application will log a warning and return a 204 No Content response. No processing will occur. Check your configuration to ensure all expected sources are listed. - -### Q: Can different sources target the same repository? - -**A:** Yes! Multiple sources can target the same repository with different copy rules. The application handles this correctly. - -### Q: How are metrics tracked for multiple sources? - -**A:** Metrics are tracked both globally and per-source. Use the `/metrics` endpoint to see breakdown by source repository. - -## Support - -If you encounter issues during migration: - -1. Check the [Troubleshooting Guide](TROUBLESHOOTING.md) -2. Review application logs for errors -3. Use the config-validator tool to identify issues -4. Consult the [Multi-Source Implementation Plan](MULTI-SOURCE-IMPLEMENTATION-PLAN.md) - -## Next Steps - -After successful migration: - -1. Monitor metrics and audit logs -2. Optimize copy rules for performance -3. Consider enabling additional features (Slack notifications, etc.) -4. Document your specific configuration for your team -5. Set up alerts for failures - diff --git a/examples-copier/docs/multi-source/MULTI-SOURCE-QUICK-REFERENCE.md b/examples-copier/docs/multi-source/MULTI-SOURCE-QUICK-REFERENCE.md deleted file mode 100644 index d4de5a9..0000000 --- a/examples-copier/docs/multi-source/MULTI-SOURCE-QUICK-REFERENCE.md +++ /dev/null @@ -1,532 +0,0 @@ -# Multi-Source Support - Quick Reference Guide - -## Overview - -This guide provides quick reference information for working with multi-source repository configurations. - -## Configuration Format - -### Single Source (Legacy) - -```yaml -source_repo: "mongodb/docs-code-examples" -source_branch: "main" -copy_rules: - - name: "example" - # ... rules -``` - -### Multi-Source (New) - -```yaml -sources: - - repo: "mongodb/docs-code-examples" - branch: "main" - installation_id: "12345678" # Optional - copy_rules: - - name: "example" - # ... rules -``` - -## Key Concepts - -### Source Repository -- The repository being monitored for changes -- Identified by `owner/repo` format (e.g., `mongodb/docs-code-examples`) -- Each source can have its own copy rules - -### Installation ID -- GitHub App installation identifier -- Different organizations require different installation IDs -- Optional: defaults to `INSTALLATION_ID` environment variable - -### Copy Rules -- Define which files to copy and where -- Each source can have multiple copy rules -- Rules are evaluated independently per source - -## Common Tasks - -### Add a New Source Repository - -```yaml -sources: - # Existing sources... - - # Add new source - - repo: "mongodb/new-repo" - branch: "main" - installation_id: "99887766" - copy_rules: - - name: "new-rule" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "mongodb/target" - branch: "main" - path_transform: "code/${path}" - commit_strategy: - type: "pull_request" - pr_title: "Update examples" - auto_merge: false -``` - -### Configure Multiple Targets - -```yaml -sources: - - repo: "mongodb/source" - branch: "main" - copy_rules: - - name: "multi-target" - source_pattern: - type: "glob" - pattern: "**/*.go" - targets: - # Target 1 - - repo: "mongodb/target1" - branch: "main" - path_transform: "examples/${filename}" - commit_strategy: - type: "direct" - - # Target 2 - - repo: "mongodb/target2" - branch: "develop" - path_transform: "code/${filename}" - commit_strategy: - type: "pull_request" - pr_title: "Update examples" - auto_merge: false -``` - -### Set Global Defaults - -```yaml -sources: - - repo: "mongodb/source1" - # ... config - - repo: "mongodb/source2" - # ... config - -# Apply to all sources unless overridden -defaults: - commit_strategy: - type: "pull_request" - auto_merge: false - deprecation_check: - enabled: true - file: "deprecated_examples.json" -``` - -### Cross-Organization Copying - -```yaml -sources: - # Source from mongodb org - - repo: "mongodb/public-examples" - branch: "main" - installation_id: "11111111" - copy_rules: - - name: "to-internal" - source_pattern: - type: "prefix" - pattern: "public/" - targets: - # Target in 10gen org (requires different installation) - - repo: "10gen/internal-docs" - branch: "main" - path_transform: "examples/${path}" - commit_strategy: - type: "direct" -``` - -## Validation - -### Validate Configuration - -```bash -# Validate syntax and logic -./config-validator validate -config copier-config.yaml -v - -# Check specific source -./config-validator validate-source \ - -config copier-config.yaml \ - -source "mongodb/docs-code-examples" -``` - -### Test Pattern Matching - -```bash -# Test if a file matches patterns -./config-validator test-pattern \ - -config copier-config.yaml \ - -source "mongodb/docs-code-examples" \ - -file "examples/go/main.go" -``` - -### Test Path Transformation - -```bash -# Test path transformation -./config-validator test-transform \ - -config copier-config.yaml \ - -source "mongodb/docs-code-examples" \ - -file "examples/go/main.go" -``` - -## Monitoring - -### Health Check - -```bash -# Check application health -curl http://localhost:8080/health | jq - -# Check specific source -curl http://localhost:8080/health | jq '.sources["mongodb/docs-code-examples"]' -``` - -### Metrics - -```bash -# Get all metrics -curl http://localhost:8080/metrics | jq - -# Get metrics for specific source -curl http://localhost:8080/metrics | jq '.by_source["mongodb/docs-code-examples"]' -``` - -### Logs - -```bash -# Filter logs by source -gcloud app logs read --filter='jsonPayload.source_repo="mongodb/docs-code-examples"' - -# Filter by operation -gcloud app logs read --filter='jsonPayload.operation="webhook_received"' -``` - -## Troubleshooting - -### Webhook Not Processing - -**Check 1: Is source configured?** -```bash -./config-validator list-sources -config copier-config.yaml -``` - -**Check 2: Is webhook signature valid?** -```bash -# Check logs for signature validation errors -gcloud app logs read --filter='jsonPayload.error=~"signature"' -``` - -**Check 3: Is installation ID correct?** -```bash -# Verify installation ID -curl -H "Authorization: Bearer YOUR_JWT" \ - https://api.github.com/app/installations -``` - -### Files Not Copying - -**Check 1: Do files match patterns?** -```bash -./config-validator test-pattern \ - -config copier-config.yaml \ - -source "mongodb/source" \ - -file "path/to/file.go" -``` - -**Check 2: Is path transformation correct?** -```bash -./config-validator test-transform \ - -config copier-config.yaml \ - -source "mongodb/source" \ - -file "path/to/file.go" -``` - -**Check 3: Check audit logs** -```bash -# Query MongoDB audit logs -db.audit_events.find({ - source_repo: "mongodb/source", - success: false -}).sort({timestamp: -1}).limit(10) -``` - -### Installation Authentication Errors - -**Check 1: Verify installation ID** -```yaml -sources: - - repo: "mongodb/source" - installation_id: "12345678" # Verify this is correct -``` - -**Check 2: Check token expiry** -```bash -# Tokens are cached for 1 hour -# Check logs for token refresh -gcloud app logs read --filter='jsonPayload.operation="token_refresh"' -``` - -**Check 3: Verify app permissions** -- Go to GitHub App settings -- Check installation has required permissions -- Verify app is installed on the repository - -## Environment Variables - -### Required - -```bash -# GitHub App Configuration -GITHUB_APP_ID=123456 -INSTALLATION_ID=12345678 # Default installation ID - -# Google Cloud -GCP_PROJECT_ID=your-project -PEM_KEY_NAME=projects/123/secrets/pem/versions/latest -WEBHOOK_SECRET_NAME=projects/123/secrets/webhook/versions/latest - -# Application -PORT=8080 -CONFIG_FILE=copier-config.yaml -``` - -### Optional - -```bash -# Dry Run Mode -DRY_RUN=false - -# Audit Logging -AUDIT_ENABLED=true -MONGO_URI=mongodb+srv://... -AUDIT_DATABASE=copier_audit -AUDIT_COLLECTION=events - -# Metrics -METRICS_ENABLED=true - -# Slack Notifications -SLACK_WEBHOOK_URL=https://hooks.slack.com/... -SLACK_CHANNEL=#copier-alerts -``` - -## Best Practices - -### 1. Use Descriptive Rule Names - -```yaml -# Good -- name: "go-examples-to-docs" - -# Bad -- name: "rule1" -``` - -### 2. Test Before Deploying - -```bash -# Always validate -./config-validator validate -config copier-config.yaml -v - -# Test in dry-run mode -DRY_RUN=true ./examples-copier -``` - -### 3. Monitor Per Source - -```yaml -# Enable metrics for each source -sources: - - repo: "mongodb/source" - settings: - enabled: true - # Monitor this source specifically -``` - -### 4. Use Pull Requests for Production - -```yaml -# Safer for production -commit_strategy: - type: "pull_request" - auto_merge: false # Require review -``` - -### 5. Enable Deprecation Tracking - -```yaml -# Track deleted files -deprecation_check: - enabled: true - file: "deprecated_examples.json" -``` - -### 6. Set Appropriate Timeouts - -```yaml -sources: - - repo: "mongodb/large-repo" - settings: - timeout_seconds: 300 # 5 minutes for large repos -``` - -### 7. Use Rate Limiting - -```yaml -sources: - - repo: "mongodb/high-volume-repo" - settings: - rate_limit: - max_webhooks_per_minute: 10 - max_concurrent: 3 -``` - -## Migration Checklist - -- [ ] Backup current configuration -- [ ] Convert to multi-source format -- [ ] Validate new configuration -- [ ] Test in dry-run mode -- [ ] Deploy to staging -- [ ] Test with real webhooks -- [ ] Monitor metrics and logs -- [ ] Deploy to production -- [ ] Decommission old deployments - -## Quick Commands - -```bash -# Validate config -./config-validator validate -config copier-config.yaml -v - -# Convert legacy to multi-source -./config-validator convert-to-multi-source \ - -input copier-config.yaml \ - -output copier-config-multi.yaml - -# Test pattern matching -./config-validator test-pattern \ - -config copier-config.yaml \ - -source "mongodb/source" \ - -file "examples/go/main.go" - -# Dry run -DRY_RUN=true ./examples-copier - -# Check health -curl http://localhost:8080/health | jq - -# Get metrics -curl http://localhost:8080/metrics | jq - -# View logs -gcloud app logs tail -s default - -# Deploy -gcloud app deploy -``` - -## Support Resources - -- [Implementation Plan](MULTI-SOURCE-IMPLEMENTATION-PLAN.md) -- [Technical Specification](MULTI-SOURCE-TECHNICAL-SPEC.md) -- [Migration Guide](MULTI-SOURCE-MIGRATION-GUIDE.md) -- [Configuration Guide](CONFIGURATION-GUIDE.md) -- [Troubleshooting Guide](TROUBLESHOOTING.md) - -## Common Patterns - -### Pattern 1: Single Source, Multiple Targets - -```yaml -sources: - - repo: "mongodb/source" - branch: "main" - copy_rules: - - name: "to-multiple-targets" - source_pattern: - type: "glob" - pattern: "**/*.go" - targets: - - repo: "mongodb/target1" - # ... config - - repo: "mongodb/target2" - # ... config - - repo: "mongodb/target3" - # ... config -``` - -### Pattern 2: Multiple Sources, Single Target - -```yaml -sources: - - repo: "mongodb/source1" - branch: "main" - copy_rules: - - name: "from-source1" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "mongodb/target" - path_transform: "source1/${path}" - # ... config - - - repo: "mongodb/source2" - branch: "main" - copy_rules: - - name: "from-source2" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "mongodb/target" - path_transform: "source2/${path}" - # ... config -``` - -### Pattern 3: Cross-Organization with Different Strategies - -```yaml -sources: - # Public repo - use PRs - - repo: "mongodb/public-examples" - branch: "main" - installation_id: "11111111" - copy_rules: - - name: "public-to-docs" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/${path}" - commit_strategy: - type: "pull_request" - auto_merge: false - - # Internal repo - direct commits - - repo: "10gen/internal-examples" - branch: "main" - installation_id: "22222222" - copy_rules: - - name: "internal-to-docs" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "10gen/internal-docs" - branch: "main" - path_transform: "code/${path}" - commit_strategy: - type: "direct" -``` - diff --git a/examples-copier/docs/multi-source/MULTI-SOURCE-README.md b/examples-copier/docs/multi-source/MULTI-SOURCE-README.md deleted file mode 100644 index 37000f5..0000000 --- a/examples-copier/docs/multi-source/MULTI-SOURCE-README.md +++ /dev/null @@ -1,314 +0,0 @@ -# Multi-Source Repository Support - Documentation Index - -## 📋 Overview - -This directory contains comprehensive documentation for implementing multi-source repository support in the examples-copier application. This feature enables monitoring and processing webhooks from multiple source repositories in a single deployment. - -## 🎯 Quick Start - -**New to multi-source?** Start here: - -1. **[Summary](docs/MULTI-SOURCE-SUMMARY.md)** - High-level overview and benefits -2. **[Quick Reference](docs/MULTI-SOURCE-QUICK-REFERENCE.md)** - Common tasks and commands -3. **[Example Config](configs/copier-config.multi-source.example.yaml)** - Working configuration example - -**Ready to implement?** Follow this path: - -1. **[Implementation Plan](docs/MULTI-SOURCE-IMPLEMENTATION-PLAN.md)** - Detailed implementation guide -2. **[Technical Spec](docs/MULTI-SOURCE-TECHNICAL-SPEC.md)** - Technical specifications -3. **[Migration Guide](docs/MULTI-SOURCE-MIGRATION-GUIDE.md)** - Step-by-step migration - -## 📚 Documentation - -### Core Documents - -| Document | Purpose | Audience | -|----------|---------|----------| -| [**Summary**](docs/MULTI-SOURCE-SUMMARY.md) | Executive overview, benefits, and status | Everyone | -| [**Implementation Plan**](docs/MULTI-SOURCE-IMPLEMENTATION-PLAN.md) | Detailed implementation roadmap | Developers | -| [**Technical Spec**](docs/MULTI-SOURCE-TECHNICAL-SPEC.md) | Technical specifications and APIs | Developers | -| [**Migration Guide**](docs/MULTI-SOURCE-MIGRATION-GUIDE.md) | Migration from single to multi-source | DevOps, Developers | -| [**Quick Reference**](docs/MULTI-SOURCE-QUICK-REFERENCE.md) | Daily operations and troubleshooting | Everyone | - -### Configuration Examples - -| File | Description | -|------|-------------| -| [**Multi-Source Example**](configs/copier-config.multi-source.example.yaml) | Complete multi-source configuration | -| [**Single-Source Example**](configs/copier-config.example.yaml) | Legacy single-source format | - -### Visual Diagrams - -- **Architecture Diagram**: High-level system architecture with multiple sources -- **Sequence Diagram**: Webhook processing flow for multi-source setup - -## 🚀 What's New - -### Key Features - -✅ **Multiple Source Repositories** -- Monitor 3+ source repositories in one deployment -- Each source has independent copy rules -- Cross-organization support (mongodb, 10gen, etc.) - -✅ **Intelligent Webhook Routing** -- Automatic source repository detection -- Dynamic configuration loading -- Graceful handling of unknown sources - -✅ **Multi-Installation Support** -- Different GitHub App installations per organization -- Automatic token management and refresh -- Seamless installation switching - -✅ **Enhanced Observability** -- Per-source metrics and monitoring -- Source-specific audit logging -- Detailed health status per source - -✅ **100% Backward Compatible** -- Existing single-source configs work unchanged -- Automatic format detection -- Gradual migration path - -## 📖 Documentation Guide - -### For Product Managers - -**Start with:** -1. [Summary](docs/MULTI-SOURCE-SUMMARY.md) - Understand benefits and scope -2. [Implementation Plan](docs/MULTI-SOURCE-IMPLEMENTATION-PLAN.md) - Review timeline and phases - -**Key Questions Answered:** -- Why do we need this? → See "Key Benefits" in Summary -- What's the timeline? → 4 weeks (see Implementation Plan) -- What are the risks? → See "Risk Mitigation" in Summary -- How do we measure success? → See "Success Criteria" in Implementation Plan - -### For Developers - -**Start with:** -1. [Technical Spec](docs/MULTI-SOURCE-TECHNICAL-SPEC.md) - Understand architecture -2. [Implementation Plan](docs/MULTI-SOURCE-IMPLEMENTATION-PLAN.md) - See detailed tasks - -**Key Sections:** -- Data models and schemas → Technical Spec §3 -- Component specifications → Technical Spec §4 -- API specifications → Technical Spec §5 -- Implementation tasks → Implementation Plan §2-8 - -**Code Changes Required:** -- `types/config.go` - New configuration types -- `services/config_loader.go` - Enhanced config loading -- `services/webhook_handler_new.go` - Webhook routing -- `services/github_auth.go` - Installation management -- `services/health_metrics.go` - Per-source metrics - -### For DevOps/SRE - -**Start with:** -1. [Migration Guide](docs/MULTI-SOURCE-MIGRATION-GUIDE.md) - Migration steps -2. [Quick Reference](docs/MULTI-SOURCE-QUICK-REFERENCE.md) - Operations guide - -**Key Sections:** -- Deployment strategy → Implementation Plan §10 -- Monitoring and metrics → Quick Reference "Monitoring" -- Troubleshooting → Quick Reference "Troubleshooting" -- Rollback procedures → Migration Guide "Rollback Plan" - -**Operational Tasks:** -- Configuration validation -- Staging deployment -- Production rollout -- Monitoring and alerting -- Decommissioning old deployments - -### For QA/Testing - -**Start with:** -1. [Technical Spec](docs/MULTI-SOURCE-TECHNICAL-SPEC.md) §9 - Testing strategy -2. [Migration Guide](docs/MULTI-SOURCE-MIGRATION-GUIDE.md) - Testing checklist - -**Test Scenarios:** -- Multi-source webhook processing -- Installation switching -- Config format conversion -- Error handling -- Performance under load -- Cross-organization copying - -## 🔧 Configuration Examples - -### Single Source (Legacy - Still Supported) - -```yaml -source_repo: "mongodb/docs-code-examples" -source_branch: "main" -copy_rules: - - name: "go-examples" - source_pattern: - type: "prefix" - pattern: "examples/go/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/go/${path}" - commit_strategy: - type: "pull_request" -``` - -### Multi-Source (New) - -```yaml -sources: - - repo: "mongodb/docs-code-examples" - branch: "main" - installation_id: "12345678" - copy_rules: - - name: "go-examples" - source_pattern: - type: "prefix" - pattern: "examples/go/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/go/${path}" - commit_strategy: - type: "pull_request" - - - repo: "mongodb/atlas-examples" - branch: "main" - installation_id: "87654321" - copy_rules: - - name: "atlas-cli" - source_pattern: - type: "glob" - pattern: "cli/**/*.go" - targets: - - repo: "mongodb/atlas-cli" - branch: "main" - path_transform: "examples/${filename}" - commit_strategy: - type: "direct" -``` - -## 🎯 Implementation Roadmap - -### Phase 1: Core Infrastructure (Week 1) -- [ ] Update configuration schema -- [ ] Implement config loading for multiple sources -- [ ] Add validation for multi-source configs -- [ ] Ensure backward compatibility - -### Phase 2: Webhook Routing (Week 2) -- [ ] Implement webhook routing logic -- [ ] Add GitHub installation switching -- [ ] Update authentication handling -- [ ] Test with multiple source repos - -### Phase 3: Observability (Week 3) -- [ ] Update metrics collection -- [ ] Enhance audit logging -- [ ] Add per-source monitoring -- [ ] Update health endpoints - -### Phase 4: Documentation & Testing (Week 4) -- [x] Write comprehensive documentation -- [x] Create migration guide -- [ ] Add unit and integration tests -- [ ] Perform end-to-end testing - -## 📊 Success Metrics - -- ✅ Support 3+ source repositories in single deployment -- ✅ 100% backward compatibility -- ✅ No performance degradation -- ✅ Clear documentation (Complete) -- ⏳ Test coverage >80% -- ⏳ Successful production deployment - -## 🔗 Related Documentation - -### Existing Documentation -- [Main README](README.md) - Application overview -- [Architecture](docs/ARCHITECTURE.md) - Current architecture -- [Configuration Guide](docs/CONFIGURATION-GUIDE.md) - Configuration reference -- [Deployment Guide](docs/DEPLOYMENT.md) - Deployment instructions - -### New Documentation -- [Multi-Source Summary](docs/MULTI-SOURCE-SUMMARY.md) -- [Implementation Plan](docs/MULTI-SOURCE-IMPLEMENTATION-PLAN.md) -- [Technical Specification](docs/MULTI-SOURCE-TECHNICAL-SPEC.md) -- [Migration Guide](docs/MULTI-SOURCE-MIGRATION-GUIDE.md) -- [Quick Reference](docs/MULTI-SOURCE-QUICK-REFERENCE.md) - -## 💡 Quick Commands - -```bash -# Validate multi-source config -./config-validator validate -config copier-config.yaml -v - -# Convert legacy to multi-source -./config-validator convert-to-multi-source \ - -input copier-config.yaml \ - -output copier-config-multi.yaml - -# Test pattern matching -./config-validator test-pattern \ - -config copier-config.yaml \ - -source "mongodb/docs-code-examples" \ - -file "examples/go/main.go" - -# Dry run with multi-source -DRY_RUN=true ./examples-copier -config copier-config-multi.yaml - -# Check health (per-source status) -curl http://localhost:8080/health | jq '.sources' - -# Get metrics by source -curl http://localhost:8080/metrics | jq '.by_source' -``` - -## 🤝 Contributing - -When implementing multi-source support: - -1. Follow the implementation plan phases -2. Write tests for all new functionality -3. Update documentation as needed -4. Ensure backward compatibility -5. Test with multiple source repositories -6. Monitor metrics during rollout - -## 📞 Support - -For questions or issues: - -1. Check the [Quick Reference](docs/MULTI-SOURCE-QUICK-REFERENCE.md) for common tasks -2. Review the [Migration Guide](docs/MULTI-SOURCE-MIGRATION-GUIDE.md) FAQ -3. Consult the [Technical Spec](docs/MULTI-SOURCE-TECHNICAL-SPEC.md) for details -4. Check existing [Troubleshooting Guide](docs/TROUBLESHOOTING.md) - -## 📝 Status - -| Component | Status | -|-----------|--------| -| Documentation | ✅ Complete | -| Implementation Plan | ✅ Complete | -| Technical Spec | ✅ Complete | -| Migration Guide | ✅ Complete | -| Example Configs | ✅ Complete | -| Code Implementation | ⏳ Pending | -| Unit Tests | ⏳ Pending | -| Integration Tests | ⏳ Pending | -| Staging Deployment | ⏳ Pending | -| Production Deployment | ⏳ Pending | - -**Last Updated**: 2025-10-15 -**Version**: 1.0 -**Status**: Documentation Complete, Ready for Implementation - ---- - -**Next Steps**: Begin Phase 1 implementation (Core Infrastructure) - diff --git a/examples-copier/docs/multi-source/MULTI-SOURCE-SUMMARY.md b/examples-copier/docs/multi-source/MULTI-SOURCE-SUMMARY.md deleted file mode 100644 index ec44f51..0000000 --- a/examples-copier/docs/multi-source/MULTI-SOURCE-SUMMARY.md +++ /dev/null @@ -1,405 +0,0 @@ -# Multi-Source Repository Support - Implementation Summary - -## Executive Summary - -This document provides a comprehensive overview of the multi-source repository support implementation plan for the examples-copier application. - -## What's Being Built - -The multi-source feature enables the examples-copier to monitor and process webhooks from **multiple source repositories** in a single deployment, eliminating the need for separate copier instances. - -### Current State -- ✅ Single source repository per deployment -- ✅ Hardcoded repository configuration -- ✅ One GitHub App installation per instance -- ✅ Manual deployment for each source - -### Future State -- 🎯 Multiple source repositories per deployment -- 🎯 Dynamic webhook routing -- 🎯 Multiple GitHub App installations -- 🎯 Centralized configuration management -- 🎯 Per-source metrics and monitoring - -## Key Benefits - -1. **Simplified Operations**: One deployment handles all source repositories -2. **Cost Reduction**: Shared infrastructure reduces hosting costs -3. **Easier Maintenance**: Single codebase and configuration to manage -4. **Better Observability**: Unified metrics and audit logging -5. **Scalability**: Easy to add new source repositories - -## Documentation Deliverables - -### 1. Implementation Plan -**File**: `docs/MULTI-SOURCE-IMPLEMENTATION-PLAN.md` - -Comprehensive plan covering: -- Current architecture analysis -- Proposed architecture design -- Detailed implementation tasks (8 phases) -- Risk assessment and mitigation -- Success criteria -- Timeline (4 weeks) - -**Key Sections**: -- Configuration schema updates -- Webhook routing logic -- GitHub App installation support -- Metrics and audit logging -- Testing strategy -- Deployment phases - -### 2. Technical Specification -**File**: `docs/MULTI-SOURCE-TECHNICAL-SPEC.md` - -Detailed technical specifications including: -- Data models and schemas -- Component interfaces -- API specifications -- Error handling -- Performance considerations -- Security requirements - -**Key Components**: -- `WebhookRouter`: Routes webhooks to correct source config -- `InstallationManager`: Manages multiple GitHub App installations -- `ConfigLoader`: Enhanced to support multi-source configs -- `MetricsCollector`: Tracks per-source metrics - -### 3. Migration Guide -**File**: `docs/MULTI-SOURCE-MIGRATION-GUIDE.md` - -Step-by-step guide for migrating from single to multi-source: -- Backward compatibility assurance -- Manual and automated conversion options -- Consolidation of multiple deployments -- Testing and validation procedures -- Rollback plan -- FAQ section - -**Migration Steps**: -1. Assess current setup -2. Backup configuration -3. Convert format (manual or automated) -4. Consolidate deployments -5. Update environment variables -6. Validate configuration -7. Deploy to staging -8. Test thoroughly -9. Production deployment -10. Decommission old deployments - -### 4. Quick Reference Guide -**File**: `docs/MULTI-SOURCE-QUICK-REFERENCE.md` - -Quick reference for daily operations: -- Configuration format examples -- Common tasks and patterns -- Validation commands -- Monitoring and troubleshooting -- Best practices -- Quick command reference - -### 5. Example Configurations -**File**: `configs/copier-config.multi-source.example.yaml` - -Complete example showing: -- Multiple source repositories -- Different organizations (mongodb, 10gen) -- Various pattern types (prefix, glob, regex) -- Multiple targets per source -- Cross-organization copying -- Global defaults - -## Architecture Overview - -### High-Level Flow - -``` -Multiple Source Repos → Webhooks → Router → Config Loader → Pattern Matcher → Target Repos - ↓ - Installation Manager - ↓ - Metrics & Audit Logging -``` - -### Key Components - -1. **Webhook Router** (New) - - Routes incoming webhooks to correct source configuration - - Validates source repository against configured sources - - Returns 204 for unknown sources - -2. **Config Loader** (Enhanced) - - Supports both legacy and multi-source formats - - Auto-detects configuration format - - Validates multi-source configurations - - Converts legacy to multi-source format - -3. **Installation Manager** (New) - - Manages multiple GitHub App installations - - Caches installation tokens - - Handles token refresh automatically - - Switches between installations per source - -4. **Metrics Collector** (Enhanced) - - Tracks metrics per source repository - - Provides global and per-source statistics - - Monitors webhook processing times - - Tracks success/failure rates - -5. **Audit Logger** (Enhanced) - - Logs events with source repository context - - Enables per-source audit queries - - Tracks cross-organization operations - -## Configuration Schema - -### Multi-Source Format - -```yaml -sources: - - repo: "mongodb/docs-code-examples" - branch: "main" - installation_id: "12345678" # Optional - copy_rules: - - name: "go-examples" - source_pattern: - type: "prefix" - pattern: "examples/go/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "code/go/${path}" - commit_strategy: - type: "pull_request" - pr_title: "Update Go examples" - auto_merge: false - - - repo: "mongodb/atlas-examples" - branch: "main" - installation_id: "87654321" - copy_rules: - # ... additional rules - -defaults: - commit_strategy: - type: "pull_request" - auto_merge: false - deprecation_check: - enabled: true -``` - -### Backward Compatibility - -The system automatically detects and supports the legacy single-source format: - -```yaml -# Legacy format - still works! -source_repo: "mongodb/docs-code-examples" -source_branch: "main" -copy_rules: - - name: "example" - # ... rules -``` - -## Implementation Phases - -### Phase 1: Core Infrastructure (Week 1) -- Update configuration schema -- Implement config loading for multiple sources -- Add validation for multi-source configs -- Ensure backward compatibility - -### Phase 2: Webhook Routing (Week 2) -- Implement webhook routing logic -- Add GitHub installation switching -- Update authentication handling -- Test with multiple source repos - -### Phase 3: Observability (Week 3) -- Update metrics collection -- Enhance audit logging -- Add per-source monitoring -- Update health endpoints - -### Phase 4: Documentation & Testing (Week 4) -- Write comprehensive documentation ✅ (Complete) -- Create migration guide ✅ (Complete) -- Add unit and integration tests -- Perform end-to-end testing - -## Key Features - -### 1. Automatic Source Detection -The webhook router automatically identifies the source repository from incoming webhooks and routes to the appropriate configuration. - -### 2. Installation Management -Seamlessly switches between GitHub App installations for different organizations, with automatic token caching and refresh. - -### 3. Per-Source Metrics -Track webhooks, files, and operations separately for each source repository: - -```json -{ - "by_source": { - "mongodb/docs-code-examples": { - "webhooks": {"received": 100, "processed": 98}, - "files": {"matched": 200, "uploaded": 195} - }, - "mongodb/atlas-examples": { - "webhooks": {"received": 50, "processed": 47}, - "files": {"matched": 120, "uploaded": 115} - } - } -} -``` - -### 4. Flexible Configuration -Support for: -- Centralized configuration (all sources in one file) -- Distributed configuration (config per source repo) -- Global defaults with per-source overrides -- Cross-organization copying - -### 5. Enhanced Monitoring -- Health endpoint shows status per source -- Metrics endpoint provides per-source breakdown -- Audit logs include source repository context -- Slack notifications with source information - -## Testing Strategy - -### Unit Tests -- Configuration loading and validation -- Webhook routing logic -- Installation token management -- Metrics collection per source - -### Integration Tests -- Multi-source webhook processing -- Installation switching -- Config format conversion -- Error handling scenarios - -### End-to-End Tests -- Complete workflow with 3+ sources -- Cross-organization copying -- Failure recovery -- Performance under load - -## Deployment Strategy - -### Rollout Approach -1. Deploy with backward compatibility enabled -2. Test in staging with multi-source config -3. Gradual production rollout (canary deployment) -4. Monitor metrics and logs closely -5. Full production deployment -6. Decommission old single-source deployments - -### Monitoring During Rollout -- Track webhook success rates per source -- Monitor GitHub API rate limits -- Watch for authentication errors -- Verify file copying success rates -- Check audit logs for anomalies - -## Success Criteria - -- ✅ Support 3+ source repositories in single deployment -- ✅ 100% backward compatibility with existing configs -- ✅ No performance degradation for single-source use cases -- ✅ Clear documentation and migration path -- ✅ Comprehensive test coverage (target: >80%) -- ✅ Successful production deployment - -## Risk Mitigation - -### Risk 1: Breaking Changes -**Mitigation**: Full backward compatibility with automatic format detection - -### Risk 2: GitHub Rate Limits -**Mitigation**: Per-source rate limiting and monitoring - -### Risk 3: Configuration Complexity -**Mitigation**: Clear examples, templates, and validation tools - -### Risk 4: Installation Token Management -**Mitigation**: Robust caching and refresh logic with error handling - -## Next Steps - -### For Implementation Team -1. Review all documentation -2. Set up development environment -3. Begin Phase 1 implementation -4. Create feature branch -5. Implement core infrastructure -6. Write unit tests -7. Submit PR for review - -### For Stakeholders -1. Review implementation plan -2. Approve timeline and resources -3. Identify test repositories -4. Plan staging environment -5. Schedule deployment windows - -### For Operations Team -1. Review deployment strategy -2. Set up monitoring alerts -3. Prepare rollback procedures -4. Plan capacity for multi-source load - -## Resources - -### Documentation -- [Implementation Plan](MULTI-SOURCE-IMPLEMENTATION-PLAN.md) - Detailed implementation guide -- [Technical Spec](MULTI-SOURCE-TECHNICAL-SPEC.md) - Technical specifications -- [Migration Guide](MULTI-SOURCE-MIGRATION-GUIDE.md) - Migration instructions -- [Quick Reference](MULTI-SOURCE-QUICK-REFERENCE.md) - Daily operations guide - -### Configuration Examples -- [Multi-Source Example](../configs/copier-config.multi-source.example.yaml) - Complete example config - -### Diagrams -- Architecture diagram (Mermaid) -- Sequence diagram (Mermaid) -- Component interaction diagram - -## Questions & Answers - -### Q: When should we migrate? -**A**: Migrate when you need to monitor multiple source repositories or want to consolidate deployments. No rush - legacy format is fully supported. - -### Q: What's the effort estimate? -**A**: 4 weeks for full implementation, testing, and deployment. Documentation is complete. - -### Q: Will this affect existing deployments? -**A**: No. Existing single-source deployments continue to work without changes. - -### Q: Can we test without affecting production? -**A**: Yes. Use dry-run mode and staging environment for thorough testing. - -### Q: What if we need to rollback? -**A**: Simple rollback to previous version. Legacy format is always supported. - -## Conclusion - -The multi-source repository support is a significant enhancement that will: -- Simplify operations and reduce costs -- Improve scalability and flexibility -- Enhance monitoring and observability -- Maintain full backward compatibility - -All documentation is complete and ready for implementation. The plan provides a clear path forward with minimal risk and maximum benefit. - ---- - -**Status**: Documentation Complete ✅ -**Next Phase**: Implementation (Phase 1) -**Timeline**: 4 weeks -**Risk Level**: Low (backward compatible) - diff --git a/examples-copier/docs/multi-source/MULTI-SOURCE-TECHNICAL-SPEC.md b/examples-copier/docs/multi-source/MULTI-SOURCE-TECHNICAL-SPEC.md deleted file mode 100644 index 8435512..0000000 --- a/examples-copier/docs/multi-source/MULTI-SOURCE-TECHNICAL-SPEC.md +++ /dev/null @@ -1,646 +0,0 @@ -# Multi-Source Repository Support - Technical Specification - -## Document Information - -- **Version**: 1.0 -- **Status**: Draft -- **Last Updated**: 2025-10-15 -- **Author**: Examples Copier Team - -## 1. Overview - -### 1.1 Purpose - -This document provides detailed technical specifications for implementing multi-source repository support in the examples-copier application. - -### 1.2 Scope - -The implementation will enable the copier to: -- Monitor multiple source repositories simultaneously -- Route webhooks to appropriate source configurations -- Manage multiple GitHub App installations -- Maintain backward compatibility with existing single-source configurations - -### 1.3 Goals - -- **Primary**: Support multiple source repositories in a single deployment -- **Secondary**: Improve observability with per-source metrics -- **Tertiary**: Simplify deployment and reduce infrastructure costs - -## 2. System Architecture - -### 2.1 Current Architecture Limitations - -``` -Current Flow (Single Source): -┌─────────────────┐ -│ Source Repo │ -│ (hardcoded) │ -└────────┬────────┘ - │ Webhook - ▼ -┌─────────────────┐ -│ Webhook Handler │ -└────────┬────────┘ - │ - ▼ -┌─────────────────┐ -│ Load Config │ -│ (from source) │ -└────────┬────────┘ - │ - ▼ -┌─────────────────┐ -│ Process Files │ -└────────┬────────┘ - │ - ▼ -┌─────────────────┐ -│ Target Repos │ -└─────────────────┘ -``` - -### 2.2 Proposed Architecture - -``` -New Flow (Multi-Source): -┌──────────┐ ┌──────────┐ ┌──────────┐ -│ Source 1 │ │ Source 2 │ │ Source 3 │ -└────┬─────┘ └────┬─────┘ └────┬─────┘ - │ Webhook │ Webhook │ Webhook - └─────────────┴─────────────┘ - │ - ▼ - ┌─────────────────┐ - │ Webhook Router │ - │ (new component) │ - └────────┬────────┘ - │ - ▼ - ┌─────────────────┐ - │ Config Loader │ - │ (enhanced) │ - └────────┬────────┘ - │ - ┌────────┴────────┐ - │ │ - ▼ ▼ - ┌─────────┐ ┌─────────┐ - │Config 1 │ │Config 2 │ - └────┬────┘ └────┬────┘ - │ │ - └────────┬───────┘ - │ - ▼ - ┌─────────────────┐ - │ Process Files │ - └────────┬────────┘ - │ - ┌────────┴────────┐ - │ │ - ▼ ▼ - ┌─────────┐ ┌─────────┐ - │Target 1 │ │Target 2 │ - └─────────┘ └─────────┘ -``` - -## 3. Data Models - -### 3.1 Configuration Schema - -#### 3.1.1 MultiSourceConfig - -```go -// MultiSourceConfig represents the root configuration -type MultiSourceConfig struct { - // New multi-source format - Sources []SourceConfig `yaml:"sources,omitempty" json:"sources,omitempty"` - Defaults *DefaultsConfig `yaml:"defaults,omitempty" json:"defaults,omitempty"` - - // Legacy single-source format (for backward compatibility) - SourceRepo string `yaml:"source_repo,omitempty" json:"source_repo,omitempty"` - SourceBranch string `yaml:"source_branch,omitempty" json:"source_branch,omitempty"` - CopyRules []CopyRule `yaml:"copy_rules,omitempty" json:"copy_rules,omitempty"` -} -``` - -#### 3.1.2 SourceConfig - -```go -// SourceConfig represents a single source repository -type SourceConfig struct { - // Repository identifier (owner/repo format) - Repo string `yaml:"repo" json:"repo"` - - // Branch to monitor (default: "main") - Branch string `yaml:"branch" json:"branch"` - - // GitHub App installation ID for this repository - // Optional: falls back to default INSTALLATION_ID - InstallationID string `yaml:"installation_id,omitempty" json:"installation_id,omitempty"` - - // Path to config file in the repository - // Optional: for distributed config approach - ConfigFile string `yaml:"config_file,omitempty" json:"config_file,omitempty"` - - // Copy rules for this source - CopyRules []CopyRule `yaml:"copy_rules" json:"copy_rules"` - - // Source-specific settings - Settings *SourceSettings `yaml:"settings,omitempty" json:"settings,omitempty"` -} -``` - -#### 3.1.3 SourceSettings - -```go -// SourceSettings contains source-specific configuration -type SourceSettings struct { - // Enable/disable this source - Enabled bool `yaml:"enabled" json:"enabled"` - - // Timeout for processing webhooks from this source - TimeoutSeconds int `yaml:"timeout_seconds,omitempty" json:"timeout_seconds,omitempty"` - - // Rate limiting settings - RateLimit *RateLimitConfig `yaml:"rate_limit,omitempty" json:"rate_limit,omitempty"` -} - -// RateLimitConfig defines rate limiting per source -type RateLimitConfig struct { - // Maximum webhooks per minute - MaxWebhooksPerMinute int `yaml:"max_webhooks_per_minute" json:"max_webhooks_per_minute"` - - // Maximum concurrent processing - MaxConcurrent int `yaml:"max_concurrent" json:"max_concurrent"` -} -``` - -#### 3.1.4 DefaultsConfig - -```go -// DefaultsConfig provides default values for all sources -type DefaultsConfig struct { - CommitStrategy *CommitStrategyConfig `yaml:"commit_strategy,omitempty" json:"commit_strategy,omitempty"` - DeprecationCheck *DeprecationConfig `yaml:"deprecation_check,omitempty" json:"deprecation_check,omitempty"` - Settings *SourceSettings `yaml:"settings,omitempty" json:"settings,omitempty"` -} -``` - -### 3.2 Runtime Data Structures - -#### 3.2.1 SourceContext - -```go -// SourceContext holds runtime context for a source repository -type SourceContext struct { - // Source configuration - Config *SourceConfig - - // GitHub client for this source - GitHubClient *github.Client - - // Installation token - InstallationToken string - - // Token expiration time - TokenExpiry time.Time - - // Metrics for this source - Metrics *SourceMetrics - - // Last processed webhook timestamp - LastWebhook time.Time -} -``` - -#### 3.2.2 SourceMetrics - -```go -// SourceMetrics tracks metrics per source repository -type SourceMetrics struct { - SourceRepo string - - // Webhook metrics - WebhooksReceived int64 - WebhooksProcessed int64 - WebhooksFailed int64 - - // File metrics - FilesMatched int64 - FilesUploaded int64 - FilesUploadFailed int64 - FilesDeprecated int64 - - // Timing metrics - AvgProcessingTime time.Duration - MaxProcessingTime time.Duration - MinProcessingTime time.Duration - - // Last update - LastUpdated time.Time -} -``` - -## 4. Component Specifications - -### 4.1 Webhook Router - -**Purpose**: Route incoming webhooks to the correct source configuration - -**Interface**: -```go -type WebhookRouter interface { - // RouteWebhook routes a webhook to the appropriate source handler - RouteWebhook(ctx context.Context, event *github.PullRequestEvent) (*SourceConfig, error) - - // RegisterSource registers a source configuration - RegisterSource(config *SourceConfig) error - - // UnregisterSource removes a source configuration - UnregisterSource(repo string) error - - // GetSource retrieves a source configuration - GetSource(repo string) (*SourceConfig, error) - - // ListSources returns all registered sources - ListSources() []*SourceConfig -} -``` - -**Implementation**: -```go -type DefaultWebhookRouter struct { - sources map[string]*SourceConfig - mu sync.RWMutex -} - -func (r *DefaultWebhookRouter) RouteWebhook(ctx context.Context, event *github.PullRequestEvent) (*SourceConfig, error) { - repo := event.GetRepo() - if repo == nil { - return nil, fmt.Errorf("webhook missing repository info") - } - - repoFullName := repo.GetFullName() - - r.mu.RLock() - defer r.mu.RUnlock() - - source, ok := r.sources[repoFullName] - if !ok { - return nil, fmt.Errorf("no configuration found for repository: %s", repoFullName) - } - - // Check if source is enabled - if source.Settings != nil && !source.Settings.Enabled { - return nil, fmt.Errorf("source repository is disabled: %s", repoFullName) - } - - return source, nil -} -``` - -### 4.2 Config Loader (Enhanced) - -**Purpose**: Load and manage multi-source configurations - -**New Methods**: -```go -type ConfigLoader interface { - // Existing method - LoadConfig(ctx context.Context, config *configs.Config) (*types.YAMLConfig, error) - - // New methods for multi-source - LoadMultiSourceConfig(ctx context.Context, config *configs.Config) (*types.MultiSourceConfig, error) - LoadSourceConfig(ctx context.Context, repo string, config *configs.Config) (*types.SourceConfig, error) - ValidateMultiSourceConfig(config *types.MultiSourceConfig) error - ConvertLegacyToMultiSource(legacy *types.YAMLConfig) (*types.MultiSourceConfig, error) -} -``` - -**Implementation**: -```go -func (cl *DefaultConfigLoader) LoadMultiSourceConfig(ctx context.Context, config *configs.Config) (*types.MultiSourceConfig, error) { - // Load raw config - yamlConfig, err := cl.LoadConfig(ctx, config) - if err != nil { - return nil, err - } - - // Detect format - if yamlConfig.SourceRepo != "" { - // Legacy format - convert to multi-source - return cl.ConvertLegacyToMultiSource(yamlConfig) - } - - // Already multi-source format - multiConfig := &types.MultiSourceConfig{ - Sources: yamlConfig.Sources, - Defaults: yamlConfig.Defaults, - } - - // Validate - if err := cl.ValidateMultiSourceConfig(multiConfig); err != nil { - return nil, err - } - - return multiConfig, nil -} - -func (cl *DefaultConfigLoader) ConvertLegacyToMultiSource(legacy *types.YAMLConfig) (*types.MultiSourceConfig, error) { - source := types.SourceConfig{ - Repo: legacy.SourceRepo, - Branch: legacy.SourceBranch, - CopyRules: legacy.CopyRules, - } - - return &types.MultiSourceConfig{ - Sources: []types.SourceConfig{source}, - }, nil -} -``` - -### 4.3 Installation Manager - -**Purpose**: Manage multiple GitHub App installations - -**Interface**: -```go -type InstallationManager interface { - // GetInstallationToken gets or refreshes token for an installation - GetInstallationToken(ctx context.Context, installationID string) (string, error) - - // GetClientForInstallation gets a GitHub client for an installation - GetClientForInstallation(ctx context.Context, installationID string) (*github.Client, error) - - // RefreshToken refreshes an installation token - RefreshToken(ctx context.Context, installationID string) error - - // ClearCache clears cached tokens - ClearCache() -} -``` - -**Implementation**: -```go -type DefaultInstallationManager struct { - tokens map[string]*InstallationToken - mu sync.RWMutex -} - -type InstallationToken struct { - Token string - ExpiresAt time.Time -} - -func (im *DefaultInstallationManager) GetInstallationToken(ctx context.Context, installationID string) (string, error) { - im.mu.RLock() - token, ok := im.tokens[installationID] - im.mu.RUnlock() - - // Check if token exists and is not expired - if ok && time.Now().Before(token.ExpiresAt.Add(-5*time.Minute)) { - return token.Token, nil - } - - // Generate new token - newToken, err := generateInstallationToken(installationID) - if err != nil { - return "", err - } - - // Cache token - im.mu.Lock() - im.tokens[installationID] = &InstallationToken{ - Token: newToken, - ExpiresAt: time.Now().Add(1 * time.Hour), - } - im.mu.Unlock() - - return newToken, nil -} -``` - -### 4.4 Metrics Collector (Enhanced) - -**Purpose**: Track metrics per source repository - -**New Methods**: -```go -type MetricsCollector interface { - // Existing methods... - - // New methods for multi-source - RecordWebhookReceivedForSource(sourceRepo string) - RecordWebhookProcessedForSource(sourceRepo string, duration time.Duration) - RecordWebhookFailedForSource(sourceRepo string) - RecordFileMatchedForSource(sourceRepo string) - RecordFileUploadedForSource(sourceRepo string) - RecordFileUploadFailedForSource(sourceRepo string) - - GetMetricsBySource(sourceRepo string) *SourceMetrics - GetAllSourceMetrics() map[string]*SourceMetrics -} -``` - -## 5. API Specifications - -### 5.1 Enhanced Health Endpoint - -**Endpoint**: `GET /health` - -**Response**: -```json -{ - "status": "healthy", - "started": true, - "github": { - "status": "healthy", - "authenticated": true - }, - "sources": { - "mongodb/docs-code-examples": { - "status": "healthy", - "last_webhook": "2025-10-15T10:30:00Z", - "installation_id": "12345678" - }, - "mongodb/atlas-examples": { - "status": "healthy", - "last_webhook": "2025-10-15T10:25:00Z", - "installation_id": "87654321" - } - }, - "queues": { - "upload_count": 0, - "deprecation_count": 0 - }, - "uptime": "2h15m30s" -} -``` - -### 5.2 Enhanced Metrics Endpoint - -**Endpoint**: `GET /metrics` - -**Response**: -```json -{ - "global": { - "webhooks": { - "received": 150, - "processed": 145, - "failed": 5, - "success_rate": 96.67 - }, - "files": { - "matched": 320, - "uploaded": 310, - "upload_failed": 5, - "deprecated": 5 - } - }, - "by_source": { - "mongodb/docs-code-examples": { - "webhooks": { - "received": 100, - "processed": 98, - "failed": 2 - }, - "files": { - "matched": 200, - "uploaded": 195, - "upload_failed": 3 - }, - "last_webhook": "2025-10-15T10:30:00Z" - }, - "mongodb/atlas-examples": { - "webhooks": { - "received": 50, - "processed": 47, - "failed": 3 - }, - "files": { - "matched": 120, - "uploaded": 115, - "upload_failed": 2 - }, - "last_webhook": "2025-10-15T10:25:00Z" - } - } -} -``` - -## 6. Error Handling - -### 6.1 Error Scenarios - -| Scenario | HTTP Status | Response | Action | -|----------|-------------|----------|--------| -| Unknown source repo | 204 No Content | Empty | Log warning, ignore webhook | -| Disabled source | 204 No Content | Empty | Log info, ignore webhook | -| Config load failure | 500 Internal Server Error | Error message | Alert, retry | -| Installation auth failure | 500 Internal Server Error | Error message | Alert, retry | -| Pattern match failure | 200 OK | Success (no files matched) | Log info | -| Upload failure | 200 OK | Success (logged as failed) | Log error, alert | - -### 6.2 Error Response Format - -```json -{ - "error": "configuration error", - "message": "no configuration found for repository: mongodb/unknown-repo", - "source_repo": "mongodb/unknown-repo", - "timestamp": "2025-10-15T10:30:00Z", - "request_id": "abc123" -} -``` - -## 7. Performance Considerations - -### 7.1 Scalability - -- **Concurrent Processing**: Support up to 10 concurrent webhook processing -- **Config Caching**: Cache loaded configurations for 5 minutes -- **Token Caching**: Cache installation tokens until 5 minutes before expiry -- **Rate Limiting**: Per-source rate limiting to prevent abuse - -### 7.2 Resource Limits - -- **Max Sources**: 50 source repositories per deployment -- **Max Copy Rules**: 100 copy rules per source -- **Max Targets**: 20 targets per copy rule -- **Config Size**: 1 MB maximum config file size - -## 8. Security Considerations - -### 8.1 Authentication - -- Each source repository requires valid GitHub App installation -- Installation tokens are cached securely in memory -- Tokens are refreshed automatically before expiry - -### 8.2 Authorization - -- Verify webhook signatures for all incoming requests -- Validate source repository against configured sources -- Ensure installation has required permissions - -### 8.3 Data Protection - -- No sensitive data in logs -- Installation tokens never logged -- Audit logs contain only necessary information - -## 9. Testing Strategy - -### 9.1 Unit Tests - -- Config loading and validation -- Webhook routing logic -- Installation token management -- Metrics collection - -### 9.2 Integration Tests - -- Multi-source webhook processing -- Installation switching -- Config format conversion -- Error handling - -### 9.3 End-to-End Tests - -- Complete workflow with multiple sources -- Cross-organization copying -- Failure recovery -- Performance under load - -## 10. Deployment Strategy - -### 10.1 Rollout Plan - -1. **Phase 1**: Deploy with backward compatibility (Week 1) -2. **Phase 2**: Enable multi-source for staging (Week 2) -3. **Phase 3**: Gradual production rollout (Week 3) -4. **Phase 4**: Full production deployment (Week 4) - -### 10.2 Monitoring - -- Track metrics per source repository -- Alert on failures -- Monitor GitHub API rate limits -- Track installation token refresh - -## 11. Appendix - -### 11.1 Configuration Examples - -See `configs/copier-config.multi-source.example.yaml` - -### 11.2 Migration Guide - -See `docs/MULTI-SOURCE-MIGRATION-GUIDE.md` - -### 11.3 Implementation Plan - -See `docs/MULTI-SOURCE-IMPLEMENTATION-PLAN.md` - diff --git a/examples-copier/docs/multi-source/README.md b/examples-copier/docs/multi-source/README.md deleted file mode 100644 index 6570a35..0000000 --- a/examples-copier/docs/multi-source/README.md +++ /dev/null @@ -1,217 +0,0 @@ -# Multi-Source Repository Support - -## Overview - -This feature enables the examples-copier to monitor and process webhooks from **multiple source repositories** across **multiple GitHub organizations** using a **centralized configuration** approach. - -### Use Case - -Perfect for teams managing code examples across multiple repositories and organizations: - -``` -Sources (monitored repos): -├── 10gen/docs-mongodb-internal -├── mongodb/docs-sample-apps -└── mongodb/docs-code-examples - -Targets (destination repos): -├── mongodb/docs -├── mongodb/docs-realm -├── mongodb/developer-hub -└── 10gen/docs-mongodb-internal -``` - -### Key Features - -✅ **Centralized Configuration** - One config file manages all sources -✅ **Multi-Organization Support** - Works across mongodb, 10gen, mongodb-university orgs -✅ **Cross-Org Copying** - Copy from mongodb → 10gen or vice versa -✅ **Single Deployment** - One app instance handles all sources -✅ **100% Backward Compatible** - Existing single-source configs still work - -## Quick Start - -### 1. Configuration Repository Setup - -Store your config in a dedicated repository: - -``` -Repository: mongodb-university/code-example-tooling -File: copier-config.yaml -``` - -### 2. Environment Variables - -```bash -# Config Repository -CONFIG_REPO_OWNER=mongodb-university -CONFIG_REPO_NAME=code-example-tooling -CONFIG_FILE=copier-config.yaml - -# GitHub App Installations (one per org) -MONGODB_INSTALLATION_ID= -TENGEN_INSTALLATION_ID= -MONGODB_UNIVERSITY_INSTALLATION_ID= -``` - -### 3. Example Configuration - -```yaml -# File: mongodb-university/code-example-tooling/copier-config.yaml - -sources: - # Source from 10gen org - - repo: "10gen/docs-mongodb-internal" - branch: "main" - installation_id: "${TENGEN_INSTALLATION_ID}" - copy_rules: - - name: "internal-to-public" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "mongodb/docs" - branch: "main" - path_transform: "source/code/${relative_path}" - commit_strategy: - type: "pull_request" - pr_title: "Update examples from internal docs" - - # Source from mongodb org - - repo: "mongodb/docs-code-examples" - branch: "main" - installation_id: "${MONGODB_INSTALLATION_ID}" - copy_rules: - - name: "examples-to-internal" - source_pattern: - type: "prefix" - pattern: "public/" - targets: - - repo: "10gen/docs-mongodb-internal" - branch: "main" - path_transform: "external-examples/${relative_path}" - commit_strategy: - type: "direct" -``` - -### 4. GitHub App Installation - -Install the GitHub App in **all three organizations**: - -1. **mongodb** - for mongodb/* repos (source and target) -2. **10gen** - for 10gen/* repos (source and target) -3. **mongodb-university** - for the config repo - -## Documentation - -| Document | Purpose | -|----------|---------| -| **[Implementation Plan](MULTI-SOURCE-IMPLEMENTATION-PLAN.md)** | Detailed implementation guide for developers | -| **[Technical Spec](MULTI-SOURCE-TECHNICAL-SPEC.md)** | Technical specifications and architecture | -| **[Migration Guide](MULTI-SOURCE-MIGRATION-GUIDE.md)** | How to migrate from single-source to multi-source | -| **[Quick Reference](MULTI-SOURCE-QUICK-REFERENCE.md)** | Common tasks and troubleshooting | - -## Architecture - -### Centralized Configuration Approach - -``` -Config Repo (mongodb-university/code-example-tooling) - │ - ├─ copier-config.yaml (manages all sources) - │ - ├─ Sources: - │ ├─ 10gen/docs-mongodb-internal - │ ├─ mongodb/docs-sample-apps - │ └─ mongodb/docs-code-examples - │ - └─ Targets: - ├─ mongodb/docs - ├─ mongodb/docs-realm - ├─ mongodb/developer-hub - └─ 10gen/docs-mongodb-internal -``` - -### Webhook Flow - -``` -1. Webhook arrives from mongodb/docs-code-examples - ↓ -2. App loads config from mongodb-university/code-example-tooling - ↓ -3. Router identifies source repo in config - ↓ -4. Switches to MONGODB_INSTALLATION_ID - ↓ -5. Reads changed files from source - ↓ -6. For each target: - - Switches to target org's installation ID - - Writes files to target repo -``` - -## Key Differences from Original Plan - -This implementation focuses on **centralized configuration** for a **single team** managing multiple repos across organizations: - -| Feature | This Implementation | Original Plan | -|---------|-------------------|---------------| -| **Config Storage** | Centralized (one file) | Centralized OR distributed | -| **Config Location** | Dedicated repo (3rd org) | Source repo or central | -| **Use Case** | Single team, multi-org | General purpose | -| **Complexity** | Simplified | Full-featured | -| **Multi-Tenant** | No (not needed) | Future enhancement | - -## Benefits - -### For MongoDB Docs Team - -1. **Single Source of Truth** - All copy rules in one config file -2. **Easy to Understand** - See all flows at a glance -3. **Centralized Management** - No need to update multiple repos -4. **Cross-Org Support** - Built-in support for mongodb ↔ 10gen flows -5. **Simple Deployment** - One app instance for everything - -### Operational - -1. **Reduced Infrastructure** - One deployment instead of multiple -2. **Unified Monitoring** - All metrics and logs in one place -3. **Easier Debugging** - Single config to check -4. **Better Visibility** - See all copy operations together - -## Implementation Status - -| Component | Status | -|-----------|--------| -| Documentation | ✅ Complete | -| Implementation Plan | ✅ Complete | -| Technical Spec | ✅ Complete | -| Migration Guide | ✅ Complete | -| Code Implementation | ⏳ Pending | -| Testing | ⏳ Pending | -| Deployment | ⏳ Pending | - -## Next Steps - -1. Review the [Implementation Plan](MULTI-SOURCE-IMPLEMENTATION-PLAN.md) -2. Set up GitHub App installations in all three orgs -3. Create config repository structure -4. Begin implementation (Phase 1: Core Infrastructure) -5. Test with staging environment -6. Deploy to production - -## Support - -For questions or issues: - -1. Check the [Quick Reference](MULTI-SOURCE-QUICK-REFERENCE.md) -2. Review the [Migration Guide](MULTI-SOURCE-MIGRATION-GUIDE.md) FAQ -3. Consult the [Technical Spec](MULTI-SOURCE-TECHNICAL-SPEC.md) - ---- - -**Configuration Approach**: Centralized -**Target Use Case**: MongoDB Docs Team (mongodb, 10gen, mongodb-university orgs) -**Status**: Ready for Implementation -**Last Updated**: 2025-10-15 - diff --git a/examples-copier/go.mod b/examples-copier/go.mod index d27b379..aeb69fa 100644 --- a/examples-copier/go.mod +++ b/examples-copier/go.mod @@ -1,10 +1,11 @@ module github.com/mongodb/code-example-tooling/code-copier -go 1.23.4 +go 1.24.0 require ( cloud.google.com/go/logging v1.13.0 cloud.google.com/go/secretmanager v1.14.6 + github.com/bmatcuk/doublestar/v4 v4.9.1 github.com/golang-jwt/jwt/v5 v5.2.2 github.com/google/go-github/v48 v48.2.0 github.com/jarcoal/httpmock v1.4.0 @@ -25,7 +26,6 @@ require ( cloud.google.com/go/compute/metadata v0.6.0 // indirect cloud.google.com/go/iam v1.4.1 // indirect cloud.google.com/go/longrunning v0.6.4 // indirect - github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -48,14 +48,14 @@ require ( go.opentelemetry.io/otel v1.34.0 // indirect go.opentelemetry.io/otel/metric v1.34.0 // indirect go.opentelemetry.io/otel/trace v1.34.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sync v0.15.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.10.0 // indirect google.golang.org/api v0.224.0 // indirect - google.golang.org/genproto v0.0.0-20250313205543-e70fdf4c4cb4 // indirect + google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250227231956-55c901821b1e // indirect google.golang.org/grpc v1.71.0 // indirect diff --git a/examples-copier/go.sum b/examples-copier/go.sum index 93db559..2e3e5ab 100644 --- a/examples-copier/go.sum +++ b/examples-copier/go.sum @@ -105,35 +105,35 @@ go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -143,8 +143,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.224.0 h1:Ir4UPtDsNiwIOHdExr3fAj4xZ42QjK7uQte3lORLJwU= google.golang.org/api v0.224.0/go.mod h1:3V39my2xAGkodXy0vEqcEtkqgw2GtrFL5WuBZlCTCOQ= -google.golang.org/genproto v0.0.0-20250313205543-e70fdf4c4cb4 h1:kCjWYliqPA8g5z87mbjnf/cdgQqMzBfp9xYre5qKu2A= -google.golang.org/genproto v0.0.0-20250313205543-e70fdf4c4cb4/go.mod h1:SqIx1NV9hcvqdLHo7uNZDS5lrUJybQ3evo3+z/WBfA0= +google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE= +google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE= google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= google.golang.org/genproto/googleapis/rpc v0.0.0-20250227231956-55c901821b1e h1:YA5lmSs3zc/5w+xsRcHqpETkaYyK63ivEPzNTcUUlSA= diff --git a/examples-copier/services/batch_pr_config_test.go b/examples-copier/services/batch_pr_config_test.go deleted file mode 100644 index 19340cc..0000000 --- a/examples-copier/services/batch_pr_config_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package services_test - -import ( - "testing" - - "github.com/mongodb/code-example-tooling/code-copier/services" - "github.com/mongodb/code-example-tooling/code-copier/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestBatchPRConfig_LoadsCorrectly(t *testing.T) { - loader := services.NewConfigLoader() - - yamlContent := ` -source_repo: "org/source-repo" -source_branch: "main" -batch_by_repo: true - -batch_pr_config: - pr_title: "Custom batch PR title" - pr_body: "Custom batch PR body with ${file_count} files" - commit_message: "Batch commit from ${source_repo}" - -copy_rules: - - name: "test-rule" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "org/target-repo" - branch: "main" - path_transform: "docs/${relative_path}" - commit_strategy: - type: "pull_request" -` - - config, err := loader.LoadConfigFromContent(yamlContent, "config.yaml") - require.NoError(t, err) - require.NotNil(t, config) - - assert.True(t, config.BatchByRepo) - require.NotNil(t, config.BatchPRConfig) - assert.Equal(t, "Custom batch PR title", config.BatchPRConfig.PRTitle) - assert.Equal(t, "Custom batch PR body with ${file_count} files", config.BatchPRConfig.PRBody) - assert.Equal(t, "Batch commit from ${source_repo}", config.BatchPRConfig.CommitMessage) -} - -func TestBatchPRConfig_OptionalField(t *testing.T) { - loader := services.NewConfigLoader() - - yamlContent := ` -source_repo: "org/source-repo" -source_branch: "main" -batch_by_repo: true - -copy_rules: - - name: "test-rule" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "org/target-repo" - branch: "main" - path_transform: "docs/${relative_path}" - commit_strategy: - type: "pull_request" -` - - config, err := loader.LoadConfigFromContent(yamlContent, "config.yaml") - require.NoError(t, err) - require.NotNil(t, config) - - assert.True(t, config.BatchByRepo) - assert.Nil(t, config.BatchPRConfig) // Should be nil when not specified -} - -func TestBatchPRConfig_StructureValidation(t *testing.T) { - // Test that the BatchPRConfig struct is properly defined - yamlConfig := &types.YAMLConfig{ - SourceRepo: "owner/source-repo", - SourceBranch: "main", - BatchByRepo: true, - BatchPRConfig: &types.BatchPRConfig{ - PRTitle: "Batch update from ${source_repo}", - PRBody: "Updated ${file_count} files from PR #${pr_number}", - CommitMessage: "Batch commit", - }, - } - - // Verify the config structure - assert.NotNil(t, yamlConfig.BatchPRConfig) - assert.Equal(t, "Batch update from ${source_repo}", yamlConfig.BatchPRConfig.PRTitle) - assert.Equal(t, "Updated ${file_count} files from PR #${pr_number}", yamlConfig.BatchPRConfig.PRBody) - assert.Equal(t, "Batch commit", yamlConfig.BatchPRConfig.CommitMessage) -} - -func TestMessageTemplater_RendersFileCount(t *testing.T) { - templater := services.NewMessageTemplater() - - ctx := types.NewMessageContext() - ctx.SourceRepo = "owner/source-repo" - ctx.FileCount = 42 - ctx.PRNumber = 123 - - template := "Updated ${file_count} files from ${source_repo} PR #${pr_number}" - result := templater.RenderPRBody(template, ctx) - - assert.Equal(t, "Updated 42 files from owner/source-repo PR #123", result) -} - -func TestMessageTemplater_DefaultBatchPRBody(t *testing.T) { - templater := services.NewMessageTemplater() - - ctx := types.NewMessageContext() - ctx.SourceRepo = "owner/source-repo" - ctx.FileCount = 15 - ctx.PRNumber = 456 - - // Empty template should use default - result := templater.RenderPRBody("", ctx) - - assert.Contains(t, result, "15 file(s)") - assert.Contains(t, result, "owner/source-repo") - assert.Contains(t, result, "#456") -} - diff --git a/examples-copier/services/config_loader.go b/examples-copier/services/config_loader.go index d5e42b2..32177b4 100644 --- a/examples-copier/services/config_loader.go +++ b/examples-copier/services/config_loader.go @@ -2,7 +2,6 @@ package services import ( "context" - "encoding/json" "fmt" "os" @@ -75,17 +74,20 @@ func (cl *DefaultConfigLoader) LoadConfigFromContent(content string, filename st // retrieveConfigFileContent fetches the config file content from the repository func retrieveConfigFileContent(ctx context.Context, filePath string, config *configs.Config) (string, error) { - // Get GitHub client - client := GetRestClient() + // Get GitHub client for the config repo's org (auto-discovers installation ID) + client, err := GetRestClientForOrg(config.ConfigRepoOwner) + if err != nil { + return "", fmt.Errorf("failed to get GitHub client for org %s: %w", config.ConfigRepoOwner, err) + } // Fetch file content fileContent, _, _, err := client.Repositories.GetContents( ctx, - config.RepoOwner, - config.RepoName, + config.ConfigRepoOwner, + config.ConfigRepoName, filePath, &github.RepositoryContentGetOptions{ - Ref: config.SrcBranch, + Ref: config.ConfigRepoBranch, }, ) if err != nil { @@ -144,103 +146,6 @@ func (cv *ConfigValidator) TestTransform(sourcePath string, template string, var return transformer.Transform(sourcePath, template, variables) } -// ExportConfigAsYAML exports a config as YAML string -func ExportConfigAsYAML(config *types.YAMLConfig) (string, error) { - data, err := yaml.Marshal(config) - if err != nil { - return "", fmt.Errorf("failed to marshal config to YAML: %w", err) - } - return string(data), nil -} - -// ExportConfigAsJSON exports a config as JSON string -func ExportConfigAsJSON(config *types.YAMLConfig) (string, error) { - data, err := json.MarshalIndent(config, "", " ") - if err != nil { - return "", fmt.Errorf("failed to marshal config to JSON: %w", err) - } - return string(data), nil -} - -// ConfigTemplate represents a configuration template -type ConfigTemplate struct { - Name string - Description string - Content string -} - -// GetConfigTemplates returns available configuration templates -func GetConfigTemplates() []ConfigTemplate { - return []ConfigTemplate{ - { - Name: "basic", - Description: "Basic configuration with prefix pattern matching", - Content: `source_repo: "owner/source-repo" -source_branch: "main" - -copy_rules: - - name: "example-rule" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "owner/target-repo" - branch: "main" - path_transform: "code-examples/${relative_path}" - commit_strategy: - type: "direct" - commit_message: "Update code examples" -`, - }, - { - Name: "glob", - Description: "Configuration with glob pattern matching", - Content: `source_repo: "owner/source-repo" -source_branch: "main" - -copy_rules: - - name: "go-examples" - source_pattern: - type: "glob" - pattern: "examples/**/*.go" - targets: - - repo: "owner/target-repo" - branch: "main" - path_transform: "go-examples/${filename}" - commit_strategy: - type: "pull_request" - pr_title: "Update Go examples" - pr_body: "Automated update from source repository" - auto_merge: false -`, - }, - { - Name: "regex", - Description: "Advanced configuration with regex pattern matching", - Content: `source_repo: "owner/source-repo" -source_branch: "main" - -copy_rules: - - name: "language-examples" - source_pattern: - type: "regex" - pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$" - targets: - - repo: "owner/docs-repo" - branch: "main" - path_transform: "source/code-examples/${lang}/${category}/${file}" - commit_strategy: - type: "pull_request" - pr_title: "Update ${lang} examples" - pr_body: "Automated update of ${lang} examples from source repository" - deprecation_check: - enabled: true - file: "deprecated_examples.json" -`, - }, - } -} - // loadLocalConfigFile attempts to load config from a local file // This is useful for local testing and development func loadLocalConfigFile(filename string) (string, error) { diff --git a/examples-copier/services/config_loader_test.go b/examples-copier/services/config_loader_test.go deleted file mode 100644 index 2274571..0000000 --- a/examples-copier/services/config_loader_test.go +++ /dev/null @@ -1,381 +0,0 @@ -package services_test - -import ( - "testing" - - "github.com/mongodb/code-example-tooling/code-copier/services" - "github.com/mongodb/code-example-tooling/code-copier/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestConfigLoader_LoadYAML(t *testing.T) { - loader := services.NewConfigLoader() - - yamlContent := ` -source_repo: "org/source-repo" -source_branch: "main" - -copy_rules: - - name: "Copy Go examples" - source_pattern: - type: "prefix" - pattern: "examples/go/" - targets: - - repo: "org/target-repo" - branch: "main" - path_transform: "docs/${path}" - commit_strategy: - type: "direct" - commit_message: "Update examples" - deprecation_check: - enabled: true - file: "deprecated.json" -` - - config, err := loader.LoadConfigFromContent(yamlContent, "config.yaml") - require.NoError(t, err) - require.NotNil(t, config) - - assert.Equal(t, "org/source-repo", config.SourceRepo) - assert.Equal(t, "main", config.SourceBranch) - assert.Len(t, config.CopyRules, 1) - - rule := config.CopyRules[0] - assert.Equal(t, "Copy Go examples", rule.Name) - assert.Equal(t, types.PatternTypePrefix, rule.SourcePattern.Type) - assert.Equal(t, "examples/go/", rule.SourcePattern.Pattern) - assert.Len(t, rule.Targets, 1) - - target := rule.Targets[0] - assert.Equal(t, "org/target-repo", target.Repo) - assert.Equal(t, "main", target.Branch) - assert.Equal(t, "docs/${path}", target.PathTransform) - assert.Equal(t, "direct", target.CommitStrategy.Type) - assert.Equal(t, "Update examples", target.CommitStrategy.CommitMessage) - assert.True(t, target.DeprecationCheck.Enabled) - assert.Equal(t, "deprecated.json", target.DeprecationCheck.File) -} - -func TestConfigLoader_LoadJSON(t *testing.T) { - loader := services.NewConfigLoader() - - jsonContent := `{ - "source_repo": "org/source-repo", - "source_branch": "main", - "copy_rules": [ - { - "name": "Copy Python examples", - "source_pattern": { - "type": "glob", - "pattern": "examples/**/*.py" - }, - "targets": [ - { - "repo": "org/target-repo", - "branch": "main", - "path_transform": "${path}", - "commit_strategy": { - "type": "pull_request", - "pr_title": "Update Python examples", - "commit_message": "Sync examples", - "auto_merge": false - } - } - ] - } - ] -}` - - config, err := loader.LoadConfigFromContent(jsonContent, "config.json") - require.NoError(t, err) - require.NotNil(t, config) - - assert.Equal(t, "org/source-repo", config.SourceRepo) - assert.Len(t, config.CopyRules, 1) - - rule := config.CopyRules[0] - assert.Equal(t, "Copy Python examples", rule.Name) - assert.Equal(t, types.PatternTypeGlob, rule.SourcePattern.Type) - assert.Equal(t, "examples/**/*.py", rule.SourcePattern.Pattern) - - target := rule.Targets[0] - assert.Equal(t, "pull_request", target.CommitStrategy.Type) - assert.Equal(t, "Update Python examples", target.CommitStrategy.PRTitle) - assert.False(t, target.CommitStrategy.AutoMerge) -} - -func TestConfigLoader_LoadLegacyJSON(t *testing.T) { - t.Skip("Legacy JSON format conversion not implemented - backward compatibility not required") - - loader := services.NewConfigLoader() - - legacyJSON := `[ - { - "source_directory": "examples", - "target_repo": "org/target", - "target_branch": "main", - "target_directory": "docs", - "recursive_copy": true, - "copier_commit_strategy": "pr", - "pr_title": "Update docs", - "commit_message": "Sync from source", - "merge_without_review": false - } -]` - - config, err := loader.LoadConfigFromContent(legacyJSON, "config.json") - require.NoError(t, err) - require.NotNil(t, config) - - // Should be converted to new format - assert.Len(t, config.CopyRules, 1) - - rule := config.CopyRules[0] - assert.Contains(t, rule.Name, "legacy-rule") - assert.Equal(t, types.PatternTypePrefix, rule.SourcePattern.Type) - assert.Equal(t, "examples", rule.SourcePattern.Pattern) - - target := rule.Targets[0] - assert.Equal(t, "org/target", target.Repo) - assert.Equal(t, "main", target.Branch) - assert.Equal(t, "pr", target.CommitStrategy.Type) - assert.Equal(t, "Update docs", target.CommitStrategy.PRTitle) - assert.Equal(t, "Sync from source", target.CommitStrategy.CommitMessage) - assert.False(t, target.CommitStrategy.AutoMerge) -} - -func TestConfigLoader_InvalidYAML(t *testing.T) { - loader := services.NewConfigLoader() - - invalidYAML := ` -source_repo: "org/repo" -copy_rules: - - name: "Test" - invalid_field: [ - unclosed bracket -` - - _, err := loader.LoadConfigFromContent(invalidYAML, "config.yaml") - assert.Error(t, err) -} - -func TestConfigLoader_InvalidJSON(t *testing.T) { - loader := services.NewConfigLoader() - - invalidJSON := `{ - "source_repo": "org/repo", - "copy_rules": [ - { - "name": "Test" - } - ] - // missing closing brace -` - - _, err := loader.LoadConfigFromContent(invalidJSON, "config.json") - assert.Error(t, err) -} - -func TestConfigLoader_ValidationErrors(t *testing.T) { - loader := services.NewConfigLoader() - - tests := []struct { - name string - content string - wantErr string - }{ - { - name: "missing source_repo", - content: ` -source_branch: "main" -copy_rules: - - name: "Test" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "org/target" - branch: "main" -`, - wantErr: "source_repo", - }, - { - name: "missing copy_rules", - content: ` -source_repo: "org/source" -source_branch: "main" -`, - wantErr: "copy rule", - }, - { - name: "invalid pattern type", - content: ` -source_repo: "org/source" -copy_rules: - - name: "Test" - source_pattern: - type: "invalid_type" - pattern: "examples/" - targets: - - repo: "org/target" - branch: "main" -`, - wantErr: "pattern type", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := loader.LoadConfigFromContent(tt.content, "config.yaml") - require.Error(t, err) - assert.Contains(t, err.Error(), tt.wantErr) - }) - } -} - -func TestConfigLoader_SetDefaults(t *testing.T) { - loader := services.NewConfigLoader() - - minimalYAML := ` -source_repo: "org/source" -copy_rules: - - name: "Test" - source_pattern: - type: "prefix" - pattern: "examples/" - targets: - - repo: "org/target" - path_transform: "${path}" -` - - config, err := loader.LoadConfigFromContent(minimalYAML, "config.yaml") - require.NoError(t, err) - - // Check defaults are set - assert.Equal(t, "main", config.SourceBranch, "default source branch") - - target := config.CopyRules[0].Targets[0] - assert.Equal(t, "main", target.Branch, "default target branch") - assert.Equal(t, "${path}", target.PathTransform, "default path transform") - assert.Equal(t, "direct", target.CommitStrategy.Type, "default commit strategy") -} - -func TestConfigValidator_ValidatePattern(t *testing.T) { - validator := services.NewConfigValidator() - - tests := []struct { - name string - pattern types.SourcePattern - wantErr bool - }{ - { - name: "valid prefix pattern", - pattern: types.SourcePattern{ - Type: types.PatternTypePrefix, - Pattern: "examples/", - }, - wantErr: false, - }, - { - name: "valid glob pattern", - pattern: types.SourcePattern{ - Type: types.PatternTypeGlob, - Pattern: "examples/**/*.go", - }, - wantErr: false, - }, - { - name: "valid regex pattern", - pattern: types.SourcePattern{ - Type: types.PatternTypeRegex, - Pattern: "^examples/(?P[^/]+)/.*$", - }, - wantErr: false, - }, - { - name: "invalid regex pattern", - pattern: types.SourcePattern{ - Type: types.PatternTypeRegex, - Pattern: "^examples/(?P[^/]+)/(?P[^/]+)/(?P.+)$", - }, - }, - target: types.TargetConfig{ - PathTransform: "docs/${lang}/${category}/${file}", - }, - wantMatch: true, - wantPath: "docs/go/database/connect.go", - wantVariables: nil, // Don't check - includes built-in variables too - }, - { - name: "no match", - filePath: "src/main.go", - rule: types.CopyRule{ - Name: "test", - SourcePattern: types.SourcePattern{ - Type: types.PatternTypePrefix, - Pattern: "examples/", - }, - }, - target: types.TargetConfig{ - PathTransform: "docs/${path}", - }, - wantMatch: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - transformedPath, variables, matched := services.MatchAndTransform( - tt.filePath, - tt.rule, - tt.target, - ) - - if !tt.wantMatch { - assert.False(t, matched) - return - } - - assert.True(t, matched) - assert.Equal(t, tt.wantPath, transformedPath) - if tt.wantVariables != nil { - assert.Equal(t, tt.wantVariables, variables) - } - }) - } -} - diff --git a/examples-copier/services/service_container.go b/examples-copier/services/service_container.go index 90fed06..a112eab 100644 --- a/examples-copier/services/service_container.go +++ b/examples-copier/services/service_container.go @@ -14,17 +14,17 @@ type ServiceContainer struct { FileStateService FileStateService // New services - ConfigLoader ConfigLoader - PatternMatcher PatternMatcher - PathTransformer PathTransformer - MessageTemplater MessageTemplater + ConfigLoader ConfigLoader + PatternMatcher PatternMatcher + PathTransformer PathTransformer + MessageTemplater MessageTemplater PRTemplateFetcher PRTemplateFetcher - AuditLogger AuditLogger - MetricsCollector *MetricsCollector - SlackNotifier SlackNotifier + AuditLogger AuditLogger + MetricsCollector *MetricsCollector + SlackNotifier SlackNotifier // Server state - StartTime time.Time + StartTime time.Time } // NewServiceContainer creates and initializes all services @@ -32,8 +32,16 @@ func NewServiceContainer(config *configs.Config) (*ServiceContainer, error) { // Initialize file state service fileStateService := NewFileStateService() - // Initialize new services - configLoader := NewConfigLoader() + // Initialize config loader based on configuration + var configLoader ConfigLoader + if config.UseMainConfig && config.MainConfigFile != "" { + // Use main config loader for new format with workflow references (when USE_MAIN_CONFIG=true) + configLoader = NewMainConfigLoader() + } else { + // Use default config loader for singular config file (when USE_MAIN_CONFIG=false) + configLoader = NewConfigLoader() + } + patternMatcher := NewPatternMatcher() pathTransformer := NewPathTransformer() messageTemplater := NewMessageTemplater() @@ -83,4 +91,3 @@ func (sc *ServiceContainer) Close(ctx context.Context) error { } return nil } - diff --git a/examples-copier/services/service_container_test.go b/examples-copier/services/service_container_test.go index 3fcf29f..b230bfb 100644 --- a/examples-copier/services/service_container_test.go +++ b/examples-copier/services/service_container_test.go @@ -18,9 +18,9 @@ func TestNewServiceContainer(t *testing.T) { { name: "valid config with audit disabled", config: &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", - AuditEnabled: false, + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + AuditEnabled: false, SlackWebhookURL: "", }, wantErr: false, @@ -29,8 +29,8 @@ func TestNewServiceContainer(t *testing.T) { { name: "valid config with Slack enabled", config: &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", AuditEnabled: false, SlackWebhookURL: "https://hooks.slack.com/services/TEST", SlackChannel: "#test", @@ -43,9 +43,9 @@ func TestNewServiceContainer(t *testing.T) { { name: "audit enabled without URI", config: &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", - AuditEnabled: true, + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + AuditEnabled: true, MongoURI: "", }, wantErr: true, @@ -133,9 +133,9 @@ func TestServiceContainer_Close(t *testing.T) { { name: "close with NoOp audit logger", config: &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", - AuditEnabled: false, + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + AuditEnabled: false, }, wantErr: false, }, @@ -166,8 +166,8 @@ func TestServiceContainer_Close(t *testing.T) { func TestServiceContainer_ConfigPropagation(t *testing.T) { config := &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", AuditEnabled: false, SlackWebhookURL: "https://hooks.slack.com/services/TEST", SlackChannel: "#test-channel", @@ -185,8 +185,8 @@ func TestServiceContainer_ConfigPropagation(t *testing.T) { t.Error("Config not stored correctly in container") } - if container.Config.RepoOwner != "test-owner" { - t.Errorf("RepoOwner = %v, want test-owner", container.Config.RepoOwner) + if container.Config.ConfigRepoOwner != "test-owner" { + t.Errorf("ConfigRepoOwner = %v, want test-owner", container.Config.ConfigRepoOwner) } if container.Config.SlackChannel != "#test-channel" { @@ -224,8 +224,8 @@ func TestServiceContainer_SlackNotifierConfiguration(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { config := &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", AuditEnabled: false, SlackWebhookURL: tt.webhookURL, SlackChannel: tt.channel, @@ -273,9 +273,9 @@ func TestServiceContainer_AuditLoggerConfiguration(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { config := &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", - AuditEnabled: tt.auditEnabled, + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + AuditEnabled: tt.auditEnabled, MongoURI: tt.mongoURI, AuditDatabase: "test-db", AuditCollection: "test-coll", @@ -305,9 +305,9 @@ func TestServiceContainer_AuditLoggerConfiguration(t *testing.T) { func TestServiceContainer_MetricsCollectorInitialization(t *testing.T) { config := &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", - AuditEnabled: false, + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + AuditEnabled: false, } container, err := NewServiceContainer(config) @@ -336,9 +336,9 @@ func TestServiceContainer_MetricsCollectorInitialization(t *testing.T) { func TestServiceContainer_StartTimeTracking(t *testing.T) { config := &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", - AuditEnabled: false, + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + AuditEnabled: false, } beforeCreate := time.Now() diff --git a/examples-copier/services/webhook_handler_new.go b/examples-copier/services/webhook_handler_new.go index dba0318..f75ad97 100644 --- a/examples-copier/services/webhook_handler_new.go +++ b/examples-copier/services/webhook_handler_new.go @@ -44,13 +44,13 @@ func simpleVerifySignature(sigHeader string, body, secret []byte) bool { } // RetrieveFileContentsWithConfigAndBranch fetches file contents from a specific branch -func RetrieveFileContentsWithConfigAndBranch(ctx context.Context, filePath string, branch string, config *configs.Config) (*github.RepositoryContent, error) { +func RetrieveFileContentsWithConfigAndBranch(ctx context.Context, filePath string, branch string, repoOwner string, repoName string) (*github.RepositoryContent, error) { client := GetRestClient() fileContent, _, _, err := client.Repositories.GetContents( ctx, - config.RepoOwner, - config.RepoName, + repoOwner, + repoName, filePath, &github.RepositoryContentGetOptions{ Ref: branch, @@ -209,11 +209,9 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit ConfigurePermissions() } - // Update config with actual repository from webhook - config.RepoOwner = repoOwner - config.RepoName = repoName - // Load configuration using new loader + // Note: config.ConfigRepoOwner and config.ConfigRepoName are already set from env.yaml + // The webhook repoOwner/repoName are used for matching workflows, not for loading config yamlConfig, err := container.ConfigLoader.LoadConfig(ctx, config) if err != nil { LogAndReturnError(ctx, "config_load", "failed to load config", err) @@ -229,24 +227,34 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit return } - // Set source repo in config if not set - if yamlConfig.SourceRepo == "" { - yamlConfig.SourceRepo = fmt.Sprintf("%s/%s", config.RepoOwner, config.RepoName) + // Find workflows matching this source repo + webhookRepo := fmt.Sprintf("%s/%s", repoOwner, repoName) + matchingWorkflows := []types.Workflow{} + for _, workflow := range yamlConfig.Workflows { + if workflow.Source.Repo == webhookRepo { + matchingWorkflows = append(matchingWorkflows, workflow) + } } - // Validate webhook is from expected source repository - webhookRepo := fmt.Sprintf("%s/%s", repoOwner, repoName) - if webhookRepo != yamlConfig.SourceRepo { - LogWarningCtx(ctx, "webhook from unexpected repository", map[string]interface{}{ - "webhook_repo": webhookRepo, - "expected_repo": yamlConfig.SourceRepo, + if len(matchingWorkflows) == 0 { + LogWarningCtx(ctx, "no workflows configured for source repository", map[string]interface{}{ + "webhook_repo": webhookRepo, + "workflow_count": len(yamlConfig.Workflows), }) container.MetricsCollector.RecordWebhookFailed() return } - // Get changed files from PR - changedFiles, err := GetFilesChangedInPr(prNumber) + LogInfoCtx(ctx, "found matching workflows", map[string]interface{}{ + "webhook_repo": webhookRepo, + "matching_count": len(matchingWorkflows), + }) + + // Store matching workflows for processing + yamlConfig.Workflows = matchingWorkflows + + // Get changed files from PR (from the source repository that triggered the webhook) + changedFiles, err := GetFilesChangedInPr(repoOwner, repoName, prNumber) if err != nil { LogAndReturnError(ctx, "get_files", "failed to get changed files", err) container.MetricsCollector.RecordWebhookFailed() @@ -256,7 +264,7 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit Operation: "get_files", Error: err, PRNumber: prNumber, - SourceRepo: yamlConfig.SourceRepo, + SourceRepo: webhookRepo, }) return } @@ -270,17 +278,12 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit filesUploadedBefore := container.MetricsCollector.GetFilesUploaded() filesFailedBefore := container.MetricsCollector.GetFilesUploadFailed() - // Process files with new pattern matching - processFilesWithPatternMatching(ctx, prNumber, sourceCommitSHA, changedFiles, yamlConfig, config, container) - - // Finalize PR metadata for batched PRs with accurate file counts - if yamlConfig.BatchByRepo { - finalizeBatchPRMetadata(yamlConfig, config, prNumber, sourceCommitSHA, container) - } + // Process files with workflow processor + processFilesWithWorkflows(ctx, prNumber, sourceCommitSHA, changedFiles, yamlConfig, container) // Upload queued files FilesToUpload = container.FileStateService.GetFilesToUpload() - AddFilesToTargetRepoBranchWithFetcher(container.PRTemplateFetcher) + AddFilesToTargetRepoBranchWithFetcher(container.PRTemplateFetcher, container.MetricsCollector) container.FileStateService.ClearFilesToUpload() // Update deprecation file - copy from FileStateService to global map for legacy function @@ -310,8 +313,8 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit container.SlackNotifier.NotifyPRProcessed(ctx, &PRProcessedEvent{ PRNumber: prNumber, PRTitle: fmt.Sprintf("PR #%d", prNumber), // TODO: Get actual PR title from GitHub - PRURL: fmt.Sprintf("https://github.com/%s/pull/%d", yamlConfig.SourceRepo, prNumber), - SourceRepo: yamlConfig.SourceRepo, + PRURL: fmt.Sprintf("https://github.com/%s/pull/%d", webhookRepo, prNumber), + SourceRepo: webhookRepo, FilesMatched: filesMatched, FilesCopied: filesUploaded, FilesFailed: filesFailed, @@ -319,345 +322,44 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit }) } -// processFilesWithPatternMatching processes changed files using the new pattern matching system -func processFilesWithPatternMatching(ctx context.Context, prNumber int, sourceCommitSHA string, - changedFiles []types.ChangedFile, yamlConfig *types.YAMLConfig, config *configs.Config, container *ServiceContainer) { +// processFilesWithWorkflows processes changed files using the workflow system +func processFilesWithWorkflows(ctx context.Context, prNumber int, sourceCommitSHA string, + changedFiles []types.ChangedFile, yamlConfig *types.YAMLConfig, container *ServiceContainer) { - LogInfoCtx(ctx, "processing files with pattern matching", map[string]interface{}{ - "file_count": len(changedFiles), - "rule_count": len(yamlConfig.CopyRules), + LogInfoCtx(ctx, "processing files with workflows", map[string]interface{}{ + "file_count": len(changedFiles), + "workflow_count": len(yamlConfig.Workflows), }) - // Log first few files for debugging - for i, file := range changedFiles { - if i < 3 { - LogInfoCtx(ctx, "sample file path", map[string]interface{}{ - "index": i, - "path": file.Path, - }) - } - } + // Create workflow processor + workflowProcessor := NewWorkflowProcessor( + container.PatternMatcher, + container.PathTransformer, + container.FileStateService, + container.MetricsCollector, + container.MessageTemplater, + ) - for _, file := range changedFiles { + // Process each workflow + for _, workflow := range yamlConfig.Workflows { if err := ctx.Err(); err != nil { - LogWebhookOperation(ctx, "file_iteration", "file iteration cancelled", err) + LogWebhookOperation(ctx, "workflow_processing", "workflow processing cancelled", err) return } - // Try to match file against each rule - for _, rule := range yamlConfig.CopyRules { - if err := ctx.Err(); err != nil { - LogWebhookOperation(ctx, "file_iteration", "file iteration cancelled", err) - return - } - - // Match file against pattern - matchResult := container.PatternMatcher.Match(file.Path, rule.SourcePattern) - if !matchResult.Matched { - continue - } - - // Record matched file - container.MetricsCollector.RecordFileMatched() - - LogInfoCtx(ctx, "file matched pattern", map[string]interface{}{ - "file": file.Path, - "rule": rule.Name, - "pattern": rule.SourcePattern.Pattern, - "variables": matchResult.Variables, + err := workflowProcessor.ProcessWorkflow(ctx, workflow, changedFiles, prNumber, sourceCommitSHA) + if err != nil { + LogErrorCtx(ctx, "failed to process workflow", err, map[string]interface{}{ + "workflow_name": workflow.Name, }) - - // Process each target - for _, target := range rule.Targets { - processFileForTarget(ctx, prNumber, sourceCommitSHA, file, rule, target, matchResult.Variables, yamlConfig, config, container) - } + // Continue processing other workflows + continue } } -} - -// processFileForTarget processes a single file for a specific target -func processFileForTarget(ctx context.Context, prNumber int, sourceCommitSHA string, file types.ChangedFile, - rule types.CopyRule, target types.TargetConfig, variables map[string]string, yamlConfig *types.YAMLConfig, config *configs.Config, container *ServiceContainer) { - - // Transform path - targetPath, err := container.PathTransformer.Transform(file.Path, target.PathTransform, variables) - if err != nil { - LogErrorCtx(ctx, "failed to transform path", err, - map[string]interface{}{ - "operation": "path_transform", - "source_path": file.Path, - "template": target.PathTransform, - }) - return - } - - // Handle deleted files - if file.Status == statusDeleted { - LogInfoCtx(ctx, "file marked as deleted, handling deprecation", map[string]interface{}{ - "file": file.Path, - "status": file.Status, - "target": targetPath, - }) - handleFileDeprecation(ctx, prNumber, sourceCommitSHA, file, rule, target, targetPath, yamlConfig.SourceBranch, config, container) - return - } - - // Handle file copy - LogInfoCtx(ctx, "file marked for copy", map[string]interface{}{ - "file": file.Path, - "status": file.Status, - "target": targetPath, - }) - handleFileCopyWithAudit(ctx, prNumber, sourceCommitSHA, file, rule, target, targetPath, variables, yamlConfig, config, container) -} - -// handleFileCopyWithAudit handles file copying with audit logging -func handleFileCopyWithAudit(ctx context.Context, prNumber int, sourceCommitSHA string, file types.ChangedFile, - rule types.CopyRule, target types.TargetConfig, targetPath string, variables map[string]string, yamlConfig *types.YAMLConfig, - config *configs.Config, container *ServiceContainer) { - - startTime := time.Now() - sourceRepo := fmt.Sprintf("%s/%s", config.RepoOwner, config.RepoName) - - // Retrieve file content from the source commit SHA (the merge commit) - // This ensures we fetch the exact version of the file that was merged - fc, err := RetrieveFileContentsWithConfigAndBranch(ctx, file.Path, sourceCommitSHA, config) - if err != nil { - // Log error event - container.AuditLogger.LogErrorEvent(ctx, &AuditEvent{ - RuleName: rule.Name, - SourceRepo: sourceRepo, - SourcePath: file.Path, - TargetRepo: target.Repo, - TargetPath: targetPath, - CommitSHA: sourceCommitSHA, - PRNumber: prNumber, - Success: false, - ErrorMessage: err.Error(), - DurationMs: time.Since(startTime).Milliseconds(), - }) - container.MetricsCollector.RecordFileUploadFailed() - LogFileOperation(ctx, "retrieve", file.Path, target.Repo, "failed to retrieve file", err) - return - } - - // Update file name to target path - fc.Name = github.String(targetPath) - - // Queue file for upload - queueFileForUploadWithStrategy(target, *fc, rule, variables, prNumber, sourceCommitSHA, yamlConfig, config, container) - - // Log successful copy event - fileSize := int64(0) - if fc.Content != nil { - fileSize = int64(len(*fc.Content)) - } - container.AuditLogger.LogCopyEvent(ctx, &AuditEvent{ - RuleName: rule.Name, - SourceRepo: sourceRepo, - SourcePath: file.Path, - TargetRepo: target.Repo, - TargetPath: targetPath, - CommitSHA: sourceCommitSHA, - PRNumber: prNumber, - Success: true, - DurationMs: time.Since(startTime).Milliseconds(), - FileSize: fileSize, - AdditionalData: map[string]any{ - "variables": variables, - }, - }) - - container.MetricsCollector.RecordFileUploaded(time.Since(startTime)) - - LogFileOperation(ctx, "queue_copy", file.Path, target.Repo, "file queued for copy", nil, - map[string]interface{}{ - "target_path": targetPath, - "rule": rule.Name, - }) -} - -// handleFileDeprecation handles file deprecation with audit logging -func handleFileDeprecation(ctx context.Context, prNumber int, sourceCommitSHA string, file types.ChangedFile, - rule types.CopyRule, target types.TargetConfig, targetPath string, sourceBranch string, config *configs.Config, container *ServiceContainer) { - - sourceRepo := fmt.Sprintf("%s/%s", config.RepoOwner, config.RepoName) - - // Check if deprecation is enabled for this target - if target.DeprecationCheck == nil || !target.DeprecationCheck.Enabled { - return - } - - // Add to deprecation queue - addToDeprecationMapForTarget(targetPath, target, container.FileStateService) - - // Log deprecation event - container.AuditLogger.LogDeprecationEvent(ctx, &AuditEvent{ - RuleName: rule.Name, - SourceRepo: sourceRepo, - SourcePath: file.Path, - TargetRepo: target.Repo, - TargetPath: targetPath, - CommitSHA: sourceCommitSHA, - PRNumber: prNumber, - Success: true, + LogInfoCtx(ctx, "workflow processing complete", map[string]interface{}{ + "workflow_count": len(yamlConfig.Workflows), }) - - container.MetricsCollector.RecordFileDeprecated() - - LogFileOperation(ctx, "deprecate", file.Path, target.Repo, "file marked for deprecation", nil, - map[string]interface{}{ - "target_path": targetPath, - "rule": rule.Name, - }) -} - -// queueFileForUploadWithStrategy queues a file for upload with the appropriate strategy -func queueFileForUploadWithStrategy(target types.TargetConfig, file github.RepositoryContent, - rule types.CopyRule, variables map[string]string, prNumber int, sourceCommitSHA string, yamlConfig *types.YAMLConfig, config *configs.Config, container *ServiceContainer) { - - // Determine commit strategy - commitStrategy := string(target.CommitStrategy.Type) - if commitStrategy == "" { - commitStrategy = "direct" // default - } - - // Create upload key - // If batch_by_repo is true, exclude rule name to batch all changes into one PR per repo - // Otherwise, include rule name to create separate PRs per rule - key := types.UploadKey{ - RepoName: target.Repo, - BranchPath: "refs/heads/" + target.Branch, - CommitStrategy: commitStrategy, - } - - if !yamlConfig.BatchByRepo { - // Include rule name to create separate PRs per rule (default behavior) - key.RuleName = rule.Name - } - - // Get existing entry or create new - filesToUpload := container.FileStateService.GetFilesToUpload() - entry, exists := filesToUpload[key] - if !exists { - entry = types.UploadFileContent{ - TargetBranch: target.Branch, - } - } - - // Set commit strategy - entry.CommitStrategy = types.CommitStrategy(target.CommitStrategy.Type) - entry.AutoMergePR = target.CommitStrategy.AutoMerge - entry.UsePRTemplate = target.CommitStrategy.UsePRTemplate - - // Add file to content first so we can get accurate file count - entry.Content = append(entry.Content, file) - - // Render commit message, PR title, and PR body using templates - msgCtx := types.NewMessageContext() - msgCtx.RuleName = rule.Name - msgCtx.SourceRepo = fmt.Sprintf("%s/%s", config.RepoOwner, config.RepoName) - msgCtx.SourceBranch = yamlConfig.SourceBranch - msgCtx.TargetRepo = target.Repo - msgCtx.TargetBranch = target.Branch - msgCtx.FileCount = len(entry.Content) - msgCtx.PRNumber = prNumber - msgCtx.CommitSHA = sourceCommitSHA - msgCtx.Variables = variables - - // For batched PRs, skip setting PR metadata here - it will be set later with accurate file counts - // For non-batched PRs, always update with current rule's messages - if yamlConfig.BatchByRepo { - // Batching by repo - PR metadata will be set in finalizeBatchPRMetadata() - // Only set commit message if not already set - if entry.CommitMessage == "" && target.CommitStrategy.CommitMessage != "" { - entry.CommitMessage = container.MessageTemplater.RenderCommitMessage(target.CommitStrategy.CommitMessage, msgCtx) - } - // Leave PRTitle and PRBody empty - will be set with accurate file count later - } else { - // Not batching - update messages for each rule (last one wins) - if target.CommitStrategy.CommitMessage != "" { - entry.CommitMessage = container.MessageTemplater.RenderCommitMessage(target.CommitStrategy.CommitMessage, msgCtx) - } - if target.CommitStrategy.PRTitle != "" { - entry.PRTitle = container.MessageTemplater.RenderPRTitle(target.CommitStrategy.PRTitle, msgCtx) - } - if target.CommitStrategy.PRBody != "" { - entry.PRBody = container.MessageTemplater.RenderPRBody(target.CommitStrategy.PRBody, msgCtx) - } - } - - container.FileStateService.AddFileToUpload(key, entry) -} - -// addToDeprecationMapForTarget adds a file to the deprecation map -func addToDeprecationMapForTarget(targetPath string, target types.TargetConfig, fileStateService FileStateService) { - entry := types.DeprecatedFileEntry{ - FileName: targetPath, - Repo: target.Repo, - Branch: target.Branch, - } - - // Use a composite key to ensure uniqueness: repo + targetPath - // This allows multiple files to be deprecated to the same deprecation file - key := target.Repo + ":" + targetPath - fileStateService.AddFileToDeprecate(key, entry) } -// finalizeBatchPRMetadata sets PR metadata for batched PRs with accurate file counts -// This is called after all files have been collected -func finalizeBatchPRMetadata(yamlConfig *types.YAMLConfig, config *configs.Config, prNumber int, sourceCommitSHA string, container *ServiceContainer) { - filesToUpload := container.FileStateService.GetFilesToUpload() - - for key, entry := range filesToUpload { - // Create message context with accurate file count - msgCtx := types.NewMessageContext() - msgCtx.SourceRepo = fmt.Sprintf("%s/%s", config.RepoOwner, config.RepoName) - msgCtx.SourceBranch = yamlConfig.SourceBranch - msgCtx.TargetRepo = key.RepoName - msgCtx.TargetBranch = entry.TargetBranch - msgCtx.FileCount = len(entry.Content) // Accurate file count! - msgCtx.PRNumber = prNumber - msgCtx.CommitSHA = sourceCommitSHA - - // Use batch_pr_config if available, otherwise use defaults - if yamlConfig.BatchPRConfig != nil { - // Use dedicated batch PR config - if yamlConfig.BatchPRConfig.PRTitle != "" { - entry.PRTitle = container.MessageTemplater.RenderPRTitle(yamlConfig.BatchPRConfig.PRTitle, msgCtx) - } else { - // Default title - entry.PRTitle = fmt.Sprintf("Update files from %s PR #%d", msgCtx.SourceRepo, prNumber) - } - - if yamlConfig.BatchPRConfig.PRBody != "" { - entry.PRBody = container.MessageTemplater.RenderPRBody(yamlConfig.BatchPRConfig.PRBody, msgCtx) - } else { - // Default body - entry.PRBody = fmt.Sprintf("Automated update from %s\n\nSource PR: #%d\nCommit: %s\nFiles: %d", - msgCtx.SourceRepo, prNumber, sourceCommitSHA[:7], len(entry.Content)) - } - - // Override commit message if specified in batch config - if yamlConfig.BatchPRConfig.CommitMessage != "" && entry.CommitMessage == "" { - entry.CommitMessage = container.MessageTemplater.RenderCommitMessage(yamlConfig.BatchPRConfig.CommitMessage, msgCtx) - } - - // Set UsePRTemplate from batch config - entry.UsePRTemplate = yamlConfig.BatchPRConfig.UsePRTemplate - } else { - // No batch_pr_config - use generic defaults - if entry.PRTitle == "" { - entry.PRTitle = fmt.Sprintf("Update files from %s PR #%d", msgCtx.SourceRepo, prNumber) - } - if entry.PRBody == "" { - entry.PRBody = fmt.Sprintf("Automated update from %s\n\nSource PR: #%d\nCommit: %s\nFiles: %d", - msgCtx.SourceRepo, prNumber, sourceCommitSHA[:7], len(entry.Content)) - } - } - // Update the entry in the map - container.FileStateService.AddFileToUpload(key, entry) - } -} diff --git a/examples-copier/services/webhook_handler_new_test.go b/examples-copier/services/webhook_handler_new_test.go index 92dfaab..381313d 100644 --- a/examples-copier/services/webhook_handler_new_test.go +++ b/examples-copier/services/webhook_handler_new_test.go @@ -85,8 +85,8 @@ func TestSimpleVerifySignature(t *testing.T) { func TestHandleWebhookWithContainer_MissingEventType(t *testing.T) { config := &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", AuditEnabled: false, } @@ -115,8 +115,8 @@ func TestHandleWebhookWithContainer_MissingEventType(t *testing.T) { func TestHandleWebhookWithContainer_InvalidSignature(t *testing.T) { config := &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", WebhookSecret: "test-secret", AuditEnabled: false, @@ -144,8 +144,8 @@ func TestHandleWebhookWithContainer_InvalidSignature(t *testing.T) { func TestHandleWebhookWithContainer_ValidSignature(t *testing.T) { secret := "test-secret" config := &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", WebhookSecret: secret, AuditEnabled: false, @@ -188,8 +188,8 @@ func TestHandleWebhookWithContainer_ValidSignature(t *testing.T) { func TestHandleWebhookWithContainer_NonPREvent(t *testing.T) { config := &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", AuditEnabled: false, } @@ -220,8 +220,8 @@ func TestHandleWebhookWithContainer_NonPREvent(t *testing.T) { func TestHandleWebhookWithContainer_NonMergedPR(t *testing.T) { config := &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", AuditEnabled: false, } @@ -267,8 +267,8 @@ func TestHandleWebhookWithContainer_MergedPR(t *testing.T) { // 3. These are test values that won't affect other tests os.Setenv(configs.AppId, "123456") os.Setenv(configs.InstallationId, "789012") - os.Setenv(configs.RepoOwner, "test-owner") - os.Setenv(configs.RepoName, "test-repo") + os.Setenv(configs.ConfigRepoOwner, "test-owner") + os.Setenv(configs.ConfigRepoName, "test-repo") os.Setenv("SKIP_SECRET_MANAGER", "true") // Generate a valid RSA private key for testing @@ -283,10 +283,10 @@ func TestHandleWebhookWithContainer_MergedPR(t *testing.T) { InstallationAccessToken = "test-token" config := &configs.Config{ - RepoOwner: "test-owner", - RepoName: "test-repo", - ConfigFile: "nonexistent-config.yaml", // Use nonexistent file to prevent actual config loading - AuditEnabled: false, + ConfigRepoOwner: "test-owner", + ConfigRepoName: "test-repo", + ConfigFile: "nonexistent-config.yaml", // Use nonexistent file to prevent actual config loading + AuditEnabled: false, } container, err := NewServiceContainer(config) diff --git a/examples-copier/services/workflow_processor.go b/examples-copier/services/workflow_processor.go new file mode 100644 index 0000000..86cf222 --- /dev/null +++ b/examples-copier/services/workflow_processor.go @@ -0,0 +1,442 @@ +package services + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/bmatcuk/doublestar/v4" + "github.com/google/go-github/v48/github" + . "github.com/mongodb/code-example-tooling/code-copier/types" +) + +// WorkflowProcessor processes workflows and applies transformations +type WorkflowProcessor interface { + ProcessWorkflow(ctx context.Context, workflow Workflow, changedFiles []ChangedFile, prNumber int, sourceCommitSHA string) error +} + +// workflowProcessor implements WorkflowProcessor +type workflowProcessor struct { + patternMatcher PatternMatcher + pathTransformer PathTransformer + fileStateService FileStateService + metricsCollector *MetricsCollector + messageTemplater MessageTemplater +} + +// NewWorkflowProcessor creates a new workflow processor +func NewWorkflowProcessor( + patternMatcher PatternMatcher, + pathTransformer PathTransformer, + fileStateService FileStateService, + metricsCollector *MetricsCollector, + messageTemplater MessageTemplater, +) WorkflowProcessor { + return &workflowProcessor{ + patternMatcher: patternMatcher, + pathTransformer: pathTransformer, + fileStateService: fileStateService, + metricsCollector: metricsCollector, + messageTemplater: messageTemplater, + } +} + +// ProcessWorkflow processes a single workflow +func (wp *workflowProcessor) ProcessWorkflow( + ctx context.Context, + workflow Workflow, + changedFiles []ChangedFile, + prNumber int, + sourceCommitSHA string, +) error { + LogInfoCtx(ctx, "Processing workflow", map[string]interface{}{ + "workflow_name": workflow.Name, + "source_repo": workflow.Source.Repo, + "destination_repo": workflow.Destination.Repo, + "file_count": len(changedFiles), + }) + + // Track files matched and skipped + filesMatched := 0 + filesSkipped := 0 + + // Process each changed file + for _, file := range changedFiles { + matched, err := wp.processFileForWorkflow(ctx, workflow, file, prNumber, sourceCommitSHA) + if err != nil { + LogErrorCtx(ctx, "Failed to process file for workflow", err, map[string]interface{}{ + "workflow_name": workflow.Name, + "file_path": file.Path, + }) + continue + } + + if matched { + filesMatched++ + } else { + filesSkipped++ + } + } + + LogInfoCtx(ctx, "Workflow processing complete", map[string]interface{}{ + "workflow_name": workflow.Name, + "files_matched": filesMatched, + "files_skipped": filesSkipped, + }) + + return nil +} + +// processFileForWorkflow processes a single file for a workflow +func (wp *workflowProcessor) processFileForWorkflow( + ctx context.Context, + workflow Workflow, + file ChangedFile, + prNumber int, + sourceCommitSHA string, +) (bool, error) { + // Check if file is excluded + if wp.isExcluded(file.Path, workflow.Exclude) { + LogInfoCtx(ctx, "File excluded by workflow exclude patterns", map[string]interface{}{ + "workflow_name": workflow.Name, + "file_path": file.Path, + }) + return false, nil + } + + // Try each transformation until one matches + for i, transformation := range workflow.Transformations { + matched, targetPath, err := wp.applyTransformation(ctx, workflow, transformation, file.Path) + if err != nil { + return false, fmt.Errorf("transformation[%d]: %w", i, err) + } + + if !matched { + continue + } + + // File matched this transformation + LogInfoCtx(ctx, "File matched transformation", map[string]interface{}{ + "workflow_name": workflow.Name, + "transformation_idx": i, + "transformation_type": transformation.GetType(), + "source_path": file.Path, + "target_path": targetPath, + }) + + // Handle file based on status + if file.Status == "removed" { + // Add to deprecation map + wp.addToDeprecationMap(workflow, targetPath) + } else { + // Add to upload queue + err := wp.addToUploadQueue(ctx, workflow, file, targetPath, prNumber, sourceCommitSHA) + if err != nil { + return false, fmt.Errorf("failed to queue file for upload: %w", err) + } + } + + return true, nil + } + + // No transformation matched + LogInfoCtx(ctx, "File did not match any transformation", map[string]interface{}{ + "workflow_name": workflow.Name, + "file_path": file.Path, + }) + + return false, nil +} + +// applyTransformation applies a transformation to a file path +func (wp *workflowProcessor) applyTransformation( + ctx context.Context, + workflow Workflow, + transformation Transformation, + sourcePath string, +) (matched bool, targetPath string, err error) { + switch transformation.GetType() { + case TransformationTypeMove: + return wp.applyMoveTransformation(transformation.Move, sourcePath) + case TransformationTypeCopy: + return wp.applyCopyTransformation(transformation.Copy, sourcePath) + case TransformationTypeGlob: + return wp.applyGlobTransformation(transformation.Glob, sourcePath) + case TransformationTypeRegex: + return wp.applyRegexTransformation(transformation.Regex, sourcePath) + default: + return false, "", fmt.Errorf("unknown transformation type: %s", transformation.GetType()) + } +} + +// applyMoveTransformation applies a move transformation +func (wp *workflowProcessor) applyMoveTransformation( + move *MoveTransform, + sourcePath string, +) (matched bool, targetPath string, err error) { + // Check if source path starts with the "from" prefix + from := strings.TrimSuffix(move.From, "/") + + if sourcePath == from { + // Exact match - move the file to the "to" path + return true, move.To, nil + } + + if strings.HasPrefix(sourcePath, from+"/") { + // Path is under the "from" directory - preserve relative path + relativePath := strings.TrimPrefix(sourcePath, from+"/") + targetPath = filepath.Join(move.To, relativePath) + return true, targetPath, nil + } + + return false, "", nil +} + +// applyCopyTransformation applies a copy transformation +func (wp *workflowProcessor) applyCopyTransformation( + copy *CopyTransform, + sourcePath string, +) (matched bool, targetPath string, err error) { + // Copy only matches exact file path + if sourcePath == copy.From { + return true, copy.To, nil + } + return false, "", nil +} + +// applyGlobTransformation applies a glob transformation +func (wp *workflowProcessor) applyGlobTransformation( + glob *GlobTransform, + sourcePath string, +) (matched bool, targetPath string, err error) { + // Use doublestar for glob matching + matched, err = doublestar.Match(glob.Pattern, sourcePath) + if err != nil { + return false, "", fmt.Errorf("invalid glob pattern: %w", err) + } + if !matched { + return false, "", nil + } + + // Extract variables for path transformation + variables := wp.extractGlobVariables(glob.Pattern, sourcePath) + + // Apply path transformation using the correct signature + targetPath, err = wp.pathTransformer.Transform(sourcePath, glob.Transform, variables) + if err != nil { + return false, "", fmt.Errorf("path transformation failed: %w", err) + } + + return true, targetPath, nil +} + +// applyRegexTransformation applies a regex transformation +func (wp *workflowProcessor) applyRegexTransformation( + regex *RegexTransform, + sourcePath string, +) (matched bool, targetPath string, err error) { + // Use existing pattern matcher for regex + sourcePattern := SourcePattern{ + Type: PatternTypeRegex, + Pattern: regex.Pattern, + } + + matchResult := wp.patternMatcher.Match(sourcePath, sourcePattern) + if !matchResult.Matched { + return false, "", nil + } + + // Apply path transformation with captured variables + targetPath, err = wp.pathTransformer.Transform(sourcePath, regex.Transform, matchResult.Variables) + if err != nil { + return false, "", fmt.Errorf("path transformation failed: %w", err) + } + + return true, targetPath, nil +} + +// extractGlobVariables extracts variables from a glob pattern match +func (wp *workflowProcessor) extractGlobVariables(pattern, path string) map[string]string { + variables := make(map[string]string) + + // Extract common variables + // For pattern "mflix/server/**" matching "mflix/server/java-spring/src/main.java" + // Extract relative_path = "java-spring/src/main.java" + + // Find the ** in the pattern + starStarIdx := strings.Index(pattern, "**") + if starStarIdx >= 0 { + prefix := pattern[:starStarIdx] + if strings.HasPrefix(path, prefix) { + relativePath := strings.TrimPrefix(path, prefix) + relativePath = strings.TrimPrefix(relativePath, "/") + variables["relative_path"] = relativePath + } + } + + return variables +} + +// isExcluded checks if a file path matches any exclude pattern +func (wp *workflowProcessor) isExcluded(path string, excludePatterns []string) bool { + for _, pattern := range excludePatterns { + matched, err := doublestar.Match(pattern, path) + if err != nil { + LogWarning(fmt.Sprintf("Invalid exclude pattern: %s: %v", pattern, err)) + continue + } + if matched { + return true + } + } + return false +} + +// addToDeprecationMap adds a file to the deprecation map +func (wp *workflowProcessor) addToDeprecationMap(workflow Workflow, targetPath string) { + deprecationFile := "deprecated_examples.json" + if workflow.DeprecationCheck != nil && workflow.DeprecationCheck.File != "" { + deprecationFile = workflow.DeprecationCheck.File + } + + entry := DeprecatedFileEntry{ + FileName: targetPath, + Repo: workflow.Destination.Repo, + Branch: workflow.Destination.Branch, + } + + wp.fileStateService.AddFileToDeprecate(deprecationFile, entry) +} + +// addToUploadQueue adds a file to the upload queue +func (wp *workflowProcessor) addToUploadQueue( + ctx context.Context, + workflow Workflow, + file ChangedFile, + targetPath string, + prNumber int, + sourceCommitSHA string, +) error { + // Parse source repo owner/name + parts := strings.Split(workflow.Source.Repo, "/") + if len(parts) != 2 { + return fmt.Errorf("invalid source repo format: expected owner/repo, got: %s", workflow.Source.Repo) + } + sourceRepoOwner := parts[0] + sourceRepoName := parts[1] + + // Fetch file content from source repository + fileContent, err := RetrieveFileContentsWithConfigAndBranch(ctx, file.Path, sourceCommitSHA, sourceRepoOwner, sourceRepoName) + if err != nil { + return fmt.Errorf("failed to retrieve file content: %w", err) + } + + // Update file name to target path + fileContent.Name = github.String(targetPath) + + // Create upload key + key := UploadKey{ + RepoName: workflow.Destination.Repo, + BranchPath: workflow.Destination.Branch, + } + + // Get existing entries from FileStateService + filesToUpload := wp.fileStateService.GetFilesToUpload() + content, exists := filesToUpload[key] + if !exists { + content = UploadFileContent{ + Content: []github.RepositoryContent{}, + CommitStrategy: CommitStrategy(getCommitStrategyType(workflow)), + UsePRTemplate: getUsePRTemplate(workflow), + AutoMergePR: getAutoMerge(workflow), + } + } + + // Add file to content + content.Content = append(content.Content, *fileContent) + + // Render templates with message context + msgCtx := NewMessageContext() + msgCtx.SourceRepo = workflow.Source.Repo + msgCtx.SourceBranch = workflow.Source.Branch + msgCtx.TargetRepo = workflow.Destination.Repo + msgCtx.TargetBranch = workflow.Destination.Branch + msgCtx.PRNumber = prNumber + msgCtx.CommitSHA = sourceCommitSHA + msgCtx.FileCount = len(content.Content) + + // Render commit message + if workflow.CommitStrategy != nil && workflow.CommitStrategy.CommitMessage != "" { + content.CommitMessage = wp.messageTemplater.RenderCommitMessage(workflow.CommitStrategy.CommitMessage, msgCtx) + } else { + content.CommitMessage = fmt.Sprintf("Update from workflow: %s", workflow.Name) + } + + // Render PR title + if workflow.CommitStrategy != nil && workflow.CommitStrategy.PRTitle != "" { + content.PRTitle = wp.messageTemplater.RenderPRTitle(workflow.CommitStrategy.PRTitle, msgCtx) + } else { + content.PRTitle = content.CommitMessage + } + + // Render PR body + if workflow.CommitStrategy != nil && workflow.CommitStrategy.PRBody != "" { + content.PRBody = wp.messageTemplater.RenderPRBody(workflow.CommitStrategy.PRBody, msgCtx) + } + + // Add back to FileStateService + wp.fileStateService.AddFileToUpload(key, content) + + // Record metric (with zero duration since we're just queuing) + if wp.metricsCollector != nil { + wp.metricsCollector.RecordFileUploaded(0 * time.Second) + } + + return nil +} + +// Helper functions to extract config values + +func getCommitStrategyType(workflow Workflow) string { + if workflow.CommitStrategy != nil && workflow.CommitStrategy.Type != "" { + return workflow.CommitStrategy.Type + } + return "pull_request" // default +} + +func getCommitMessage(workflow Workflow) string { + if workflow.CommitStrategy != nil && workflow.CommitStrategy.CommitMessage != "" { + return workflow.CommitStrategy.CommitMessage + } + return fmt.Sprintf("Update from workflow: %s", workflow.Name) +} + +func getPRTitle(workflow Workflow) string { + if workflow.CommitStrategy != nil && workflow.CommitStrategy.PRTitle != "" { + return workflow.CommitStrategy.PRTitle + } + return getCommitMessage(workflow) +} + +func getPRBody(workflow Workflow) string { + if workflow.CommitStrategy != nil && workflow.CommitStrategy.PRBody != "" { + return workflow.CommitStrategy.PRBody + } + return "" +} + +func getUsePRTemplate(workflow Workflow) bool { + if workflow.CommitStrategy != nil { + return workflow.CommitStrategy.UsePRTemplate + } + return false +} + +func getAutoMerge(workflow Workflow) bool { + if workflow.CommitStrategy != nil { + return workflow.CommitStrategy.AutoMerge + } + return false +} diff --git a/examples-copier/tests/utils.go b/examples-copier/tests/utils.go index 4886dcd..ddf2b33 100644 --- a/examples-copier/tests/utils.go +++ b/examples-copier/tests/utils.go @@ -19,13 +19,13 @@ import ( // Environment helpers // -// EnvOwnerRepo returns owner/repo from env and fails the test if either is missing. +// EnvOwnerRepo returns config repo owner/name from env and fails the test if either is missing. func EnvOwnerRepo(t testing.TB) (string, string) { t.Helper() - owner := os.Getenv(configs.RepoOwner) - repo := os.Getenv(configs.RepoName) + owner := os.Getenv(configs.ConfigRepoOwner) + repo := os.Getenv(configs.ConfigRepoName) if owner == "" || repo == "" { - t.Fatalf("REPO_OWNER/REPO_NAME not set") + t.Fatalf("CONFIG_REPO_OWNER/CONFIG_REPO_NAME not set") } return owner, repo } diff --git a/examples-copier/types/config.go b/examples-copier/types/config.go index cae1b80..940e50f 100644 --- a/examples-copier/types/config.go +++ b/examples-copier/types/config.go @@ -25,55 +25,242 @@ func (p PatternType) String() string { return string(p) } -// YAMLConfig represents the new YAML-based configuration structure +// SourcePattern defines how to match source files (used by pattern matcher) +type SourcePattern struct { + Type PatternType `yaml:"type" json:"type"` + Pattern string `yaml:"pattern" json:"pattern"` + ExcludePatterns []string `yaml:"exclude_patterns,omitempty" json:"exclude_patterns,omitempty"` // Optional: regex patterns to exclude from matches +} + +// Validate validates a source pattern +func (sp *SourcePattern) Validate() error { + if !sp.Type.IsValid() { + return fmt.Errorf("invalid pattern type: %s (must be prefix, glob, or regex)", sp.Type) + } + if sp.Pattern == "" { + return fmt.Errorf("pattern is required") + } + + // Validate exclude patterns if provided + if len(sp.ExcludePatterns) > 0 { + for i, excludePattern := range sp.ExcludePatterns { + if excludePattern == "" { + return fmt.Errorf("exclude_patterns[%d] is empty", i) + } + // Validate that it's a valid regex pattern + if _, err := regexp.Compile(excludePattern); err != nil { + return fmt.Errorf("exclude_patterns[%d] is not a valid regex: %w", i, err) + } + } + } + + return nil +} + +// YAMLConfig represents the YAML-based configuration structure type YAMLConfig struct { - SourceRepo string `yaml:"source_repo" json:"source_repo"` - SourceBranch string `yaml:"source_branch" json:"source_branch"` - BatchByRepo bool `yaml:"batch_by_repo,omitempty" json:"batch_by_repo,omitempty"` // If true, batch all changes into one PR per target repo - BatchPRConfig *BatchPRConfig `yaml:"batch_pr_config,omitempty" json:"batch_pr_config,omitempty"` // PR config used when batch_by_repo is true - CopyRules []CopyRule `yaml:"copy_rules" json:"copy_rules"` + Workflows []Workflow `yaml:"workflows" json:"workflows"` + Defaults *Defaults `yaml:"defaults,omitempty" json:"defaults,omitempty"` } -// BatchPRConfig defines PR metadata for batched PRs -type BatchPRConfig struct { - PRTitle string `yaml:"pr_title,omitempty" json:"pr_title,omitempty"` - PRBody string `yaml:"pr_body,omitempty" json:"pr_body,omitempty"` - CommitMessage string `yaml:"commit_message,omitempty" json:"commit_message,omitempty"` - UsePRTemplate bool `yaml:"use_pr_template,omitempty" json:"use_pr_template,omitempty"` +// ============================================================================ +// Main Config types (central configuration with workflow references) +// ============================================================================ + +// MainConfig represents the central configuration file that references workflow configs +type MainConfig struct { + Defaults *Defaults `yaml:"defaults,omitempty" json:"defaults,omitempty"` + WorkflowConfigs []WorkflowConfigRef `yaml:"workflow_configs" json:"workflow_configs"` } -// CopyRule defines a single rule for copying files with pattern matching -type CopyRule struct { - Name string `yaml:"name" json:"name"` - SourcePattern SourcePattern `yaml:"source_pattern" json:"source_pattern"` - Targets []TargetConfig `yaml:"targets" json:"targets"` +// WorkflowConfigRef references a workflow configuration file +type WorkflowConfigRef struct { + Source string `yaml:"source" json:"source"` // "local", "repo", or "inline" + Path string `yaml:"path,omitempty" json:"path,omitempty"` // Path to config file + Repo string `yaml:"repo,omitempty" json:"repo,omitempty"` // Repository (for source="repo") + Branch string `yaml:"branch,omitempty" json:"branch,omitempty"` // Branch (for source="repo") + Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` // Whether this workflow config is enabled (default: true) + Workflows []Workflow `yaml:"workflows,omitempty" json:"workflows,omitempty"` // Inline workflows (for source="inline") } -// SourcePattern defines how to match source files -type SourcePattern struct { - Type PatternType `yaml:"type" json:"type"` - Pattern string `yaml:"pattern" json:"pattern"` - ExcludePatterns []string `yaml:"exclude_patterns,omitempty" json:"exclude_patterns,omitempty"` // Optional: regex patterns to exclude from matches +// WorkflowConfig represents a workflow configuration file (can be in source repos) +type WorkflowConfig struct { + Defaults *Defaults `yaml:"defaults,omitempty" json:"defaults,omitempty"` + Workflows []Workflow `yaml:"workflows" json:"workflows"` + + // Context information (not from YAML, set by loader) + SourceRepo string `yaml:"-" json:"-"` // Source repo this config came from (for context inference) + SourceBranch string `yaml:"-" json:"-"` // Source branch this config came from (for context inference) +} + +// ============================================================================ +// Reference types (for $ref support) +// ============================================================================ + +// Ref represents a reference to another file +type Ref struct { + Ref string `yaml:"$ref,omitempty" json:"$ref,omitempty"` +} + +// RefOrValue is a generic type that can be either a reference or an inline value +// This is used for fields that support $ref +type RefOrValue[T any] struct { + Ref string `yaml:"$ref,omitempty" json:"$ref,omitempty"` + Value *T `yaml:",inline" json:",inline"` +} + +// IsRef returns true if this is a reference +func (r *RefOrValue[T]) IsRef() bool { + return r.Ref != "" +} + +// GetValue returns the inline value (nil if this is a reference) +func (r *RefOrValue[T]) GetValue() *T { + return r.Value +} + +// TransformationsOrRef can be either inline transformations or a $ref +type TransformationsOrRef struct { + Ref string `yaml:"-" json:"-"` + Transformations []Transformation `yaml:"-" json:"-"` + isRef bool +} + +// UnmarshalYAML implements custom YAML unmarshaling for transformations +func (t *TransformationsOrRef) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Try to unmarshal as a reference first + var refMap map[string]string + if err := unmarshal(&refMap); err == nil { + if ref, ok := refMap["$ref"]; ok { + t.Ref = ref + t.isRef = true + return nil + } + } + + // Not a reference, unmarshal as array of transformations + var transformations []Transformation + if err := unmarshal(&transformations); err != nil { + return err + } + t.Transformations = transformations + t.isRef = false + return nil +} + +// MarshalYAML implements custom YAML marshaling for transformations +func (t TransformationsOrRef) MarshalYAML() (interface{}, error) { + if t.isRef { + return map[string]string{"$ref": t.Ref}, nil + } + return t.Transformations, nil +} + +// IsRef returns true if this is a reference +func (t *TransformationsOrRef) IsRef() bool { + return t.isRef +} + +// CommitStrategyOrRef can be either inline commit strategy or a $ref +type CommitStrategyOrRef struct { + Ref string `yaml:"-" json:"-"` + CommitStrategy *CommitStrategyConfig `yaml:"-" json:"-"` + isRef bool +} + +// UnmarshalYAML implements custom YAML unmarshaling for commit strategy +func (c *CommitStrategyOrRef) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Try to unmarshal as a reference first + var refMap map[string]string + if err := unmarshal(&refMap); err == nil { + if ref, ok := refMap["$ref"]; ok { + c.Ref = ref + c.isRef = true + return nil + } + } + + // Not a reference, unmarshal as commit strategy + var strategy CommitStrategyConfig + if err := unmarshal(&strategy); err != nil { + return err + } + c.CommitStrategy = &strategy + c.isRef = false + return nil +} + +// MarshalYAML implements custom YAML marshaling for commit strategy +func (c CommitStrategyOrRef) MarshalYAML() (interface{}, error) { + if c.isRef { + return map[string]string{"$ref": c.Ref}, nil + } + return c.CommitStrategy, nil +} + +// IsRef returns true if this is a reference +func (c *CommitStrategyOrRef) IsRef() bool { + return c.isRef +} + +// ExcludeOrRef can be either inline exclude patterns or a $ref +type ExcludeOrRef struct { + Ref string `yaml:"-" json:"-"` + Exclude []string `yaml:"-" json:"-"` + isRef bool +} + +// UnmarshalYAML implements custom YAML unmarshaling for exclude patterns +func (e *ExcludeOrRef) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Try to unmarshal as a reference first + var refMap map[string]string + if err := unmarshal(&refMap); err == nil { + if ref, ok := refMap["$ref"]; ok { + e.Ref = ref + e.isRef = true + return nil + } + } + + // Not a reference, unmarshal as array of strings + var exclude []string + if err := unmarshal(&exclude); err != nil { + return err + } + e.Exclude = exclude + e.isRef = false + return nil } -// TargetConfig defines where and how to copy matched files -type TargetConfig struct { - Repo string `yaml:"repo" json:"repo"` - Branch string `yaml:"branch" json:"branch"` - PathTransform string `yaml:"path_transform" json:"path_transform"` - CommitStrategy CommitStrategyConfig `yaml:"commit_strategy,omitempty" json:"commit_strategy,omitempty"` - DeprecationCheck *DeprecationConfig `yaml:"deprecation_check,omitempty" json:"deprecation_check,omitempty"` +// MarshalYAML implements custom YAML marshaling for exclude patterns +func (e ExcludeOrRef) MarshalYAML() (interface{}, error) { + if e.isRef { + return map[string]string{"$ref": e.Ref}, nil + } + return e.Exclude, nil } -// CommitStrategyConfig defines how to commit changes +// IsRef returns true if this is a reference +func (e *ExcludeOrRef) IsRef() bool { + return e.isRef +} + +// CommitStrategyConfig defines commit strategy settings type CommitStrategyConfig struct { - Type string `yaml:"type" json:"type"` // "direct", "pull_request", or "batch" + Type string `yaml:"type" json:"type"` // "direct" or "pull_request" CommitMessage string `yaml:"commit_message,omitempty" json:"commit_message,omitempty"` PRTitle string `yaml:"pr_title,omitempty" json:"pr_title,omitempty"` PRBody string `yaml:"pr_body,omitempty" json:"pr_body,omitempty"` UsePRTemplate bool `yaml:"use_pr_template,omitempty" json:"use_pr_template,omitempty"` // If true, fetch and use PR template from target repo AutoMerge bool `yaml:"auto_merge,omitempty" json:"auto_merge,omitempty"` - BatchSize int `yaml:"batch_size,omitempty" json:"batch_size,omitempty"` +} + +// Validate validates the commit strategy configuration +func (c *CommitStrategyConfig) Validate() error { + if c.Type != "" && c.Type != "direct" && c.Type != "pull_request" { + return fmt.Errorf("invalid type: %s (must be direct or pull_request)", c.Type) + } + return nil } // DeprecationConfig defines deprecation tracking settings @@ -82,136 +269,312 @@ type DeprecationConfig struct { File string `yaml:"file,omitempty" json:"file,omitempty"` // defaults to deprecated_examples.json } +// ============================================================================ +// Workflow-based configuration types +// ============================================================================ + +// Defaults defines default settings for all workflows +type Defaults struct { + CommitStrategy *CommitStrategyConfig `yaml:"commit_strategy,omitempty" json:"commit_strategy,omitempty"` + DeprecationCheck *DeprecationConfig `yaml:"deprecation_check,omitempty" json:"deprecation_check,omitempty"` + Exclude []string `yaml:"exclude,omitempty" json:"exclude,omitempty"` +} + +// Workflow defines a complete source → destination mapping with transformations +type Workflow struct { + Name string `yaml:"name" json:"name"` + Source Source `yaml:"source" json:"source"` + Destination Destination `yaml:"destination" json:"destination"` + Transformations []Transformation `yaml:"transformations" json:"transformations"` + Exclude []string `yaml:"exclude,omitempty" json:"exclude,omitempty"` + CommitStrategy *CommitStrategyConfig `yaml:"commit_strategy,omitempty" json:"commit_strategy,omitempty"` + DeprecationCheck *DeprecationConfig `yaml:"deprecation_check,omitempty" json:"deprecation_check,omitempty"` + + // Internal fields for $ref support (not serialized) + TransformationsRef string `yaml:"-" json:"-"` + ExcludeRef string `yaml:"-" json:"-"` + CommitStrategyRef string `yaml:"-" json:"-"` +} + +// Source defines the source repository and branch +type Source struct { + Repo string `yaml:"repo" json:"repo"` + Branch string `yaml:"branch,omitempty" json:"branch,omitempty"` // defaults to "main" + InstallationID string `yaml:"installation_id,omitempty" json:"installation_id,omitempty"` // optional override +} + +// Destination defines the destination repository and branch +type Destination struct { + Repo string `yaml:"repo" json:"repo"` + Branch string `yaml:"branch,omitempty" json:"branch,omitempty"` // defaults to "main" + InstallationID string `yaml:"installation_id,omitempty" json:"installation_id,omitempty"` // optional override +} + +// TransformationType defines the type of transformation +type TransformationType string + +const ( + TransformationTypeMove TransformationType = "move" + TransformationTypeCopy TransformationType = "copy" + TransformationTypeGlob TransformationType = "glob" + TransformationTypeRegex TransformationType = "regex" +) + +// Transformation defines how to transform file paths from source to destination +type Transformation struct { + // Type is inferred from which field is set (move, copy, glob, regex) + Move *MoveTransform `yaml:"move,omitempty" json:"move,omitempty"` + Copy *CopyTransform `yaml:"copy,omitempty" json:"copy,omitempty"` + Glob *GlobTransform `yaml:"glob,omitempty" json:"glob,omitempty"` + Regex *RegexTransform `yaml:"regex,omitempty" json:"regex,omitempty"` +} + +// MoveTransform moves files from one directory to another +type MoveTransform struct { + From string `yaml:"from" json:"from"` // Source path (can be directory or file) + To string `yaml:"to" json:"to"` // Destination path +} + +// CopyTransform copies a single file to a new location +type CopyTransform struct { + From string `yaml:"from" json:"from"` // Source file path + To string `yaml:"to" json:"to"` // Destination file path +} + +// GlobTransform uses glob patterns with path transformation +type GlobTransform struct { + Pattern string `yaml:"pattern" json:"pattern"` // Glob pattern (e.g., "mflix/server/**/*.js") + Transform string `yaml:"transform" json:"transform"` // Path transform template (e.g., "server/${relative_path}") +} + +// RegexTransform uses regex patterns with named capture groups +type RegexTransform struct { + Pattern string `yaml:"pattern" json:"pattern"` // Regex pattern with named groups + Transform string `yaml:"transform" json:"transform"` // Path transform template using captured groups +} + // Validate validates the YAML configuration func (c *YAMLConfig) Validate() error { - if c.SourceRepo == "" { - return fmt.Errorf("source_repo is required") - } - if c.SourceBranch == "" { - c.SourceBranch = "main" // default - } - if len(c.CopyRules) == 0 { - return fmt.Errorf("at least one copy rule is required") + if len(c.Workflows) == 0 { + return fmt.Errorf("at least one workflow is required") } - for i, rule := range c.CopyRules { - if err := rule.Validate(); err != nil { - return fmt.Errorf("copy_rules[%d]: %w", i, err) + for i, workflow := range c.Workflows { + if err := workflow.Validate(); err != nil { + return fmt.Errorf("workflows[%d]: %w", i, err) } } return nil } -// Validate validates a copy rule -func (r *CopyRule) Validate() error { - if r.Name == "" { - return fmt.Errorf("name is required") - } - if err := r.SourcePattern.Validate(); err != nil { - return fmt.Errorf("source_pattern: %w", err) - } - if len(r.Targets) == 0 { - return fmt.Errorf("at least one target is required") +// Validate validates the main configuration +func (m *MainConfig) Validate() error { + if len(m.WorkflowConfigs) == 0 { + return fmt.Errorf("at least one workflow config reference is required") } - for i, target := range r.Targets { - if err := target.Validate(); err != nil { - return fmt.Errorf("targets[%d]: %w", i, err) + for i, ref := range m.WorkflowConfigs { + if err := ref.Validate(); err != nil { + return fmt.Errorf("workflow_configs[%d]: %w", i, err) } } return nil } -// Validate validates a source pattern -func (p *SourcePattern) Validate() error { - if !p.Type.IsValid() { - return fmt.Errorf("invalid pattern type: %s (must be prefix, glob, or regex)", p.Type) +// Validate validates a workflow config reference +func (w *WorkflowConfigRef) Validate() error { + // Skip validation for disabled workflow configs + if w.Enabled != nil && !*w.Enabled { + return nil } - if p.Pattern == "" { - return fmt.Errorf("pattern is required") + + if w.Source == "" { + return fmt.Errorf("source is required (must be 'local', 'repo', or 'inline')") } - // Validate exclude patterns if provided - if len(p.ExcludePatterns) > 0 { - for i, excludePattern := range p.ExcludePatterns { - if excludePattern == "" { - return fmt.Errorf("exclude_patterns[%d] is empty", i) - } - // Validate that it's a valid regex pattern - if _, err := regexp.Compile(excludePattern); err != nil { - return fmt.Errorf("exclude_patterns[%d] is not a valid regex: %w", i, err) + switch w.Source { + case "local": + if w.Path == "" { + return fmt.Errorf("path is required for source='local'") + } + case "repo": + if w.Repo == "" { + return fmt.Errorf("repo is required for source='repo'") + } + if w.Path == "" { + return fmt.Errorf("path is required for source='repo'") + } + if w.Branch == "" { + w.Branch = "main" // default + } + case "inline": + if len(w.Workflows) == 0 { + return fmt.Errorf("workflows are required for source='inline'") + } + for i, workflow := range w.Workflows { + if err := workflow.Validate(); err != nil { + return fmt.Errorf("workflows[%d]: %w", i, err) } } + default: + return fmt.Errorf("invalid source: %s (must be 'local', 'repo', or 'inline')", w.Source) } return nil } -// Validate validates a target config -func (t *TargetConfig) Validate() error { - if t.Repo == "" { - return fmt.Errorf("repo is required") - } - if t.Branch == "" { - t.Branch = "main" // default - } - if t.PathTransform == "" { - return fmt.Errorf("path_transform is required") +// Validate validates a workflow config +func (w *WorkflowConfig) Validate() error { + if len(w.Workflows) == 0 { + return fmt.Errorf("at least one workflow is required") } - // Validate commit strategy if provided - if t.CommitStrategy.Type != "" { - if err := t.CommitStrategy.Validate(); err != nil { - return fmt.Errorf("commit_strategy: %w", err) + for i, workflow := range w.Workflows { + if err := workflow.Validate(); err != nil { + return fmt.Errorf("workflows[%d]: %w", i, err) } } return nil } -// Validate validates a commit strategy config -func (c *CommitStrategyConfig) Validate() error { - validTypes := map[string]bool{ - "direct": true, - "pull_request": true, - "batch": true, - } +// SetDefaults sets default values for the configuration +func (c *YAMLConfig) SetDefaults() { + // Set defaults for workflow format + for i := range c.Workflows { + workflow := &c.Workflows[i] - if c.Type != "" && !validTypes[c.Type] { - return fmt.Errorf("invalid type: %s (must be direct, pull_request, or batch)", c.Type) + // Set source defaults + if workflow.Source.Branch == "" { + workflow.Source.Branch = "main" + } + + // Set destination defaults + if workflow.Destination.Branch == "" { + workflow.Destination.Branch = "main" + } + + // Apply global defaults if not overridden + if workflow.CommitStrategy == nil && c.Defaults != nil && c.Defaults.CommitStrategy != nil { + workflow.CommitStrategy = c.Defaults.CommitStrategy + } + + if workflow.DeprecationCheck == nil && c.Defaults != nil && c.Defaults.DeprecationCheck != nil { + workflow.DeprecationCheck = c.Defaults.DeprecationCheck + } + + if len(workflow.Exclude) == 0 && c.Defaults != nil && len(c.Defaults.Exclude) > 0 { + workflow.Exclude = c.Defaults.Exclude + } + + // Set commit strategy defaults + if workflow.CommitStrategy != nil && workflow.CommitStrategy.Type == "" { + workflow.CommitStrategy.Type = "pull_request" + } + + // Set deprecation check defaults + if workflow.DeprecationCheck != nil && workflow.DeprecationCheck.File == "" { + workflow.DeprecationCheck.File = "deprecated_examples.json" + } } +} - if c.Type == "batch" && c.BatchSize <= 0 { - c.BatchSize = 100 // default batch size +// SetDefaults sets default values for the main configuration +func (m *MainConfig) SetDefaults() { + // Set defaults for workflow config references + for i := range m.WorkflowConfigs { + ref := &m.WorkflowConfigs[i] + if ref.Source == "repo" && ref.Branch == "" { + ref.Branch = "main" + } } +} - return nil +// SetDefaults sets default values for a workflow config +func (w *WorkflowConfig) SetDefaults() { + // Set defaults for each workflow + for i := range w.Workflows { + workflow := &w.Workflows[i] + + // Set source defaults + if workflow.Source.Branch == "" { + workflow.Source.Branch = "main" + } + + // Set destination defaults + if workflow.Destination.Branch == "" { + workflow.Destination.Branch = "main" + } + + // Apply local defaults if not overridden + if workflow.CommitStrategy == nil && w.Defaults != nil && w.Defaults.CommitStrategy != nil { + workflow.CommitStrategy = w.Defaults.CommitStrategy + } + + if workflow.DeprecationCheck == nil && w.Defaults != nil && w.Defaults.DeprecationCheck != nil { + workflow.DeprecationCheck = w.Defaults.DeprecationCheck + } + + if len(workflow.Exclude) == 0 && w.Defaults != nil && len(w.Defaults.Exclude) > 0 { + workflow.Exclude = w.Defaults.Exclude + } + + // Set commit strategy defaults + if workflow.CommitStrategy != nil && workflow.CommitStrategy.Type == "" { + workflow.CommitStrategy.Type = "pull_request" + } + + // Set deprecation check defaults + if workflow.DeprecationCheck != nil && workflow.DeprecationCheck.File == "" { + workflow.DeprecationCheck.File = "deprecated_examples.json" + } + } } -// SetDefaults sets default values for the configuration -func (c *YAMLConfig) SetDefaults() { - if c.SourceBranch == "" { - c.SourceBranch = "main" +// ApplySourceContext applies source repo/branch context to workflows that don't specify them +// This allows workflows in source repos to omit the source.repo and source.branch fields +func (w *WorkflowConfig) ApplySourceContext() { + if w.SourceRepo == "" { + return // No context to apply } - for i := range c.CopyRules { - for j := range c.CopyRules[i].Targets { - target := &c.CopyRules[i].Targets[j] - if target.Branch == "" { - target.Branch = "main" - } - if target.CommitStrategy.Type == "" { - target.CommitStrategy.Type = "direct" - } - if target.DeprecationCheck != nil && target.DeprecationCheck.File == "" { - target.DeprecationCheck.File = "deprecated_examples.json" - } + for i := range w.Workflows { + workflow := &w.Workflows[i] + + // Apply source repo if not specified + if workflow.Source.Repo == "" { + workflow.Source.Repo = w.SourceRepo + } + + // Apply source branch if not specified + if workflow.Source.Branch == "" && w.SourceBranch != "" { + workflow.Source.Branch = w.SourceBranch } } } +// ApplyGlobalDefaults applies global defaults from main config to workflow config +func (w *WorkflowConfig) ApplyGlobalDefaults(globalDefaults *Defaults) { + // Apply global defaults to local defaults if not set + if w.Defaults == nil { + w.Defaults = &Defaults{} + } + + if w.Defaults.CommitStrategy == nil && globalDefaults != nil && globalDefaults.CommitStrategy != nil { + w.Defaults.CommitStrategy = globalDefaults.CommitStrategy + } + + if w.Defaults.DeprecationCheck == nil && globalDefaults != nil && globalDefaults.DeprecationCheck != nil { + w.Defaults.DeprecationCheck = globalDefaults.DeprecationCheck + } + + if len(w.Defaults.Exclude) == 0 && globalDefaults != nil && len(globalDefaults.Exclude) > 0 { + w.Defaults.Exclude = globalDefaults.Exclude + } +} + // MatchResult represents the result of a pattern match type MatchResult struct { Matched bool // Whether the pattern matched @@ -290,3 +653,214 @@ func NewMessageContext() *MessageContext { } } +// ============================================================================ +// Custom YAML unmarshaling for Workflow (to support $ref) +// ============================================================================ + +// UnmarshalYAML implements custom YAML unmarshaling for Workflow to support $ref +func (w *Workflow) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Create a temporary struct with the same fields but using OrRef types + type workflowAlias struct { + Name string `yaml:"name"` + Source Source `yaml:"source"` + Destination Destination `yaml:"destination"` + Transformations TransformationsOrRef `yaml:"transformations"` + Exclude ExcludeOrRef `yaml:"exclude,omitempty"` + CommitStrategy CommitStrategyOrRef `yaml:"commit_strategy,omitempty"` + DeprecationCheck *DeprecationConfig `yaml:"deprecation_check,omitempty"` + } + + var alias workflowAlias + if err := unmarshal(&alias); err != nil { + return err + } + + // Copy simple fields + w.Name = alias.Name + w.Source = alias.Source + w.Destination = alias.Destination + w.DeprecationCheck = alias.DeprecationCheck + + // Handle transformations (inline or $ref) + if alias.Transformations.IsRef() { + w.TransformationsRef = alias.Transformations.Ref + } else { + w.Transformations = alias.Transformations.Transformations + } + + // Handle exclude (inline or $ref) + if alias.Exclude.IsRef() { + w.ExcludeRef = alias.Exclude.Ref + } else { + w.Exclude = alias.Exclude.Exclude + } + + // Handle commit strategy (inline or $ref) + if alias.CommitStrategy.IsRef() { + w.CommitStrategyRef = alias.CommitStrategy.Ref + } else { + w.CommitStrategy = alias.CommitStrategy.CommitStrategy + } + + return nil +} + +// ============================================================================ +// Validation methods for workflow types +// ============================================================================ + +// Validate validates a workflow +func (w *Workflow) Validate() error { + if w.Name == "" { + return fmt.Errorf("name is required") + } + if err := w.Source.Validate(); err != nil { + return fmt.Errorf("source: %w", err) + } + if err := w.Destination.Validate(); err != nil { + return fmt.Errorf("destination: %w", err) + } + if len(w.Transformations) == 0 { + return fmt.Errorf("at least one transformation is required") + } + + for i, transform := range w.Transformations { + if err := transform.Validate(); err != nil { + return fmt.Errorf("transformations[%d]: %w", i, err) + } + } + + // Validate commit strategy if provided + if w.CommitStrategy != nil { + if err := w.CommitStrategy.Validate(); err != nil { + return fmt.Errorf("commit_strategy: %w", err) + } + } + + return nil +} + +// Validate validates a source +func (s *Source) Validate() error { + if s.Repo == "" { + return fmt.Errorf("repo is required") + } + if s.Branch == "" { + s.Branch = "main" // default + } + return nil +} + +// Validate validates a destination +func (d *Destination) Validate() error { + if d.Repo == "" { + return fmt.Errorf("repo is required") + } + if d.Branch == "" { + d.Branch = "main" // default + } + return nil +} + +// Validate validates a transformation +func (t *Transformation) Validate() error { + // Count how many transformation types are set + count := 0 + if t.Move != nil { + count++ + } + if t.Copy != nil { + count++ + } + if t.Glob != nil { + count++ + } + if t.Regex != nil { + count++ + } + + if count == 0 { + return fmt.Errorf("one of move, copy, glob, or regex must be specified") + } + if count > 1 { + return fmt.Errorf("only one of move, copy, glob, or regex can be specified") + } + + // Validate the specific transformation type + if t.Move != nil { + return t.Move.Validate() + } + if t.Copy != nil { + return t.Copy.Validate() + } + if t.Glob != nil { + return t.Glob.Validate() + } + if t.Regex != nil { + return t.Regex.Validate() + } + + return nil +} + +// Validate validates a move transformation +func (m *MoveTransform) Validate() error { + if m.From == "" { + return fmt.Errorf("from is required") + } + if m.To == "" { + return fmt.Errorf("to is required") + } + return nil +} + +// Validate validates a copy transformation +func (c *CopyTransform) Validate() error { + if c.From == "" { + return fmt.Errorf("from is required") + } + if c.To == "" { + return fmt.Errorf("to is required") + } + return nil +} + +// Validate validates a glob transformation +func (g *GlobTransform) Validate() error { + if g.Pattern == "" { + return fmt.Errorf("pattern is required") + } + if g.Transform == "" { + return fmt.Errorf("transform is required") + } + return nil +} + +// Validate validates a regex transformation +func (r *RegexTransform) Validate() error { + if r.Pattern == "" { + return fmt.Errorf("pattern is required") + } + if r.Transform == "" { + return fmt.Errorf("transform is required") + } + return nil +} + +// GetType returns the type of transformation +func (t *Transformation) GetType() TransformationType { + if t.Move != nil { + return TransformationTypeMove + } + if t.Copy != nil { + return TransformationTypeCopy + } + if t.Glob != nil { + return TransformationTypeGlob + } + if t.Regex != nil { + return TransformationTypeRegex + } + return "" +} + diff --git a/examples-copier/types/types.go b/examples-copier/types/types.go index 9fffc5c..9cb346f 100644 --- a/examples-copier/types/types.go +++ b/examples-copier/types/types.go @@ -11,6 +11,10 @@ type PullRequestQuery struct { Repository struct { PullRequest struct { Files struct { + PageInfo struct { + EndCursor githubv4.String + HasNextPage bool + } Edges []struct { Node struct { Path githubv4.String @@ -19,7 +23,7 @@ type PullRequestQuery struct { ChangeType githubv4.String } } - } `graphql:"files(first: 100)"` + } `graphql:"files(first: 100, after: $cursor)"` } `graphql:"pullRequest(number: $number)"` } `graphql:"repository(owner: $owner, name: $name)"` }