Wire module expander into CI, remove old generator

.github/workflows/commit-terraform.yml

@@ -60,14 +60,15 @@ jobs:
           sparse-checkout: scripts
           persist-credentials: false
 
-      - name: Generate Terraform from app.yaml
+      - name: Expand modules from app.yaml
         if: steps.check.outputs.has_yaml == 'true'
         env:
           APP_SERVICE: ${{ github.event.repository.name }}
           AWS_ACCOUNT_ID: ${{ inputs.aws_account_id }}
           AWS_REGION: ${{ inputs.aws_region }}
           TF_ROOT: ${{ inputs.tf_root }}
-        run: sh .platform/scripts/generate-terraform.sh
+          PLATFORM_ROOT: .platform
+        run: python3 .platform/scripts/expand-modules.py
 
       - name: Commit and push generated files
         if: steps.check.outputs.has_yaml == 'true'

.github/workflows/expand-terraform.yml

@@ -0,0 +1,62 @@
+name: Expand Terraform
+
+# Expands app.yaml into raw Terraform resource definitions using the module
+# expander. Commits the generated .tf files back to the repo so that
+# tf-plan and tf-apply work on committed files — no repeated expansion.
+
+on:
+  workflow_call:
+    inputs:
+      aws_account_id:
+        description: "AWS account ID"
+        type: string
+        default: "553637109631"
+      aws_region:
+        description: "AWS region"
+        type: string
+        default: "eu-central-1"
+      tf_root:
+        description: "Terraform root directory"
+        type: string
+        default: "terraform"
+
+permissions:
+  contents: write
+
+jobs:
+  expand:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Generate GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.PLATFORM_APP_ID }}
+          private-key: ${{ secrets.PLATFORM_APP_PRIVATE_KEY }}
+          owner: javaBin
+
+      - name: Checkout platform scripts
+        uses: actions/checkout@v4
+        with:
+          repository: javaBin/platform
+          token: ${{ steps.app-token.outputs.token }}
+          path: .platform
+          sparse-checkout: |
+            scripts
+            terraform/modules
+
+      - name: Expand modules from app.yaml
+        env:
+          APP_SERVICE: ${{ github.event.repository.name }}
+          AWS_ACCOUNT_ID: ${{ inputs.aws_account_id }}
+          AWS_REGION: ${{ inputs.aws_region }}
+          TF_ROOT: ${{ inputs.tf_root }}
+          PLATFORM_ROOT: .platform
+        run: python3 .platform/scripts/expand-modules.py
+
+      - name: Commit and push generated files
+        run: sh .platform/scripts/commit-generated-tf.sh "${{ inputs.tf_root }}" "${{ github.ref_name }}"

.github/workflows/javabin.yml

@@ -58,11 +58,23 @@ jobs:
     secrets: inherit
 
   # --------------------------------------------------------------------------
-  # 4. Terraform plan (if terraform/ exists or app.yaml needs generation)
+  # 4. Expand app.yaml → Terraform files (commit to repo, run once)
   # --------------------------------------------------------------------------
-  tf-plan:
+  expand:
     needs: detect
-    if: needs.detect.outputs.has_tf == 'true' || needs.detect.outputs.has_yaml == 'true'
+    if: needs.detect.outputs.has_yaml == 'true'
+    uses: javaBin/platform/.github/workflows/expand-terraform.yml@main
+    secrets: inherit
+
+  # --------------------------------------------------------------------------
+  # 5. Terraform plan (after expand commits TF files, or if terraform/ already exists)
+  # --------------------------------------------------------------------------
+  tf-plan:
+    needs: [detect, expand]
+    if: |
+      always() &&
+      (needs.detect.outputs.has_tf == 'true' || needs.detect.outputs.has_yaml == 'true') &&
+      (needs.expand.result == 'success' || needs.expand.result == 'skipped')
     uses: javaBin/platform/.github/workflows/tf-plan.yml@main
     secrets: inherit
 
@@ -108,17 +120,7 @@ jobs:
       image_tag: ${{ needs.docker-build.outputs.image_tag }}
     secrets: inherit
 
-  # --------------------------------------------------------------------------
-  # 8. Commit generated Terraform back to app repo
-  # --------------------------------------------------------------------------
-  commit-terraform:
-    needs: [detect, tf-apply]
-    if: >-
-      github.ref == 'refs/heads/main' &&
-      needs.detect.outputs.has_yaml == 'true' &&
-      needs.tf-apply.result == 'success'
-    uses: javaBin/platform/.github/workflows/commit-terraform.yml@main
-    secrets: inherit
+  # (commit-terraform removed — expand-terraform.yml handles it)
 
   # --------------------------------------------------------------------------
   # 9. EB deploy (transitional — for repos with .elasticbeanstalk/)

.github/workflows/tf-apply.yml

@@ -71,19 +71,6 @@ jobs:
       - name: Check risk level
         run: sh .platform/scripts/check-risk-gate.sh "${{ inputs.risk_level }}" "${{ github.repository }}" "${{ github.sha }}" /javabin/slack/platform-override-alerts-webhook
 
-      - name: Generate Terraform from app.yaml
-        if: hashFiles('app.yaml') != ''
-        env:
-          APP_SERVICE: ${{ github.event.repository.name }}
-          AWS_ACCOUNT_ID: ${{ inputs.aws_account_id }}
-          AWS_REGION: ${{ inputs.aws_region }}
-          TF_ROOT: ${{ inputs.tf_root }}
-        run: sh .platform/scripts/generate-terraform.sh
-
-      - name: Configure git credentials for module downloads
-        if: hashFiles('app.yaml') != ''
-        run: git config --global url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/".insteadOf "https://github.com/"
-
       - name: Download plan from S3
         working-directory: ${{ inputs.tf_root }}
         run: aws s3 cp "s3://${PLAN_BUCKET}/${{ inputs.plan_key }}" tfplan

.github/workflows/tf-plan.yml

@@ -70,19 +70,6 @@ jobs:
           path: .platform
           sparse-checkout: scripts
 
-      - name: Generate Terraform from app.yaml
-        if: hashFiles('app.yaml') != ''
-        env:
-          APP_SERVICE: ${{ github.event.repository.name }}
-          AWS_ACCOUNT_ID: ${{ inputs.aws_account_id }}
-          AWS_REGION: ${{ inputs.aws_region }}
-          TF_ROOT: ${{ inputs.tf_root }}
-        run: sh .platform/scripts/generate-terraform.sh
-
-      - name: Configure git credentials for module downloads
-        if: hashFiles('app.yaml') != ''
-        run: git config --global url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/".insteadOf "https://github.com/"
-
       - name: Terraform Init
         working-directory: ${{ inputs.tf_root }}
         run: terraform init -input=false

.gitignore

@@ -8,3 +8,4 @@
 *.zip
 .DS_Store
 phases/
+scripts/__pycache__/

CLAUDE.md

@@ -151,7 +151,7 @@ terraform/state/
 | `terraform/modules/service-secret/` | Secrets Manager secret with IAM policy output |
 | `terraform/modules/service-queue/` | SQS queue + DLQ with IAM policy output |
 | `terraform/modules/service-alarm/` | CloudWatch alarms for ECS service |
-| `terraform/modules/app-stack/` | Golden path — reads `app.yaml`, creates all infra |
+| ~~`terraform/modules/app-stack/`~~ | Removed — replaced by `scripts/expand-modules.py` + `scripts/registry.py` |
 | `terraform/modules/cognito-app-client/` | Cognito app client registration (code exists, no pools deployed yet) |
 
 ### GitHub Actions Workflows
@@ -187,7 +187,8 @@ terraform/state/
 | Script | What |
 |--------|------|
 | `scripts/bootstrap.sh` | One-time: create state bucket + lock tables |
-| `scripts/generate-terraform.sh` | CI: app.yaml → Terraform files |
+| `scripts/expand-modules.py` | CI: reads app.yaml + module sources, generates expanded .tf files |
+| `scripts/registry.py` | Module registry — maps app.yaml sections to platform modules |
 | `scripts/provision-teams.py` | CI: fetch team YAMLs from registry, invoke team-provisioner Lambda |
 | `scripts/review-plan.py` | CI: LLM plan review via Bedrock |
 | `scripts/notify-slack.py` | CI: generic Slack webhook notification |
@@ -268,11 +269,11 @@ The SA JSON key is at `/javabin/platform/google-admin-sa`, the impersonation tar
 | 2c | IAM / OIDC | **Deployed** — 5 CI roles (infra, per-app, deploy, override-approver, registry) |
 | 2d | Compute | **Deployed** — ECS cluster + ECR repos |
 | 2e | Monitoring | **Deployed** — GuardDuty, Security Hub, Config, SNS |
-| 2f | Lambda Functions | **Deployed** — 6 working (Google/GitHub/Budget sync live, Cognito/IdC sync not yet implemented in Lambda) |
+| 2f | Lambda Functions | **Deployed** — 6 working (Google/GitHub/Budget/Cognito/Identity Center sync live) |
 | 2g | Platform CI | **Done** — plan → LLM review → apply pipeline working |
 | 3a | Reusable Terraform Modules | **Code done** — 12 modules in repo |
 | 3b | GitHub Actions Workflows | **Code done** — 14 reusable workflows |
-| 3c | app.yaml Schema + Generation | **Done** — generate-terraform.sh + docs |
+| 3c | app.yaml Schema + Generation | **Done** — expand-modules.py + registry.py (expanded raw resources) |
 | 3d | Registry Repo | **Working** — repo exists, dispatch uses GitHub App token, team provisioner invoked |
 | 3e | javabin CLI | **Code done** — 4 commands (register, init, status, whoami) in javaBin/javabin-cli |
 | 3f | CI Images + Supporting Repos | Not started |

scripts/expand-modules.py

@@ -19,11 +19,11 @@
 Optional env vars: PLATFORM_ROOT (defaults to .platform)
 """
 
+import json
 import os
 import re
 import subprocess
 import sys
-import yaml
 
 # Import registry from the same directory
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
@@ -40,8 +40,20 @@
 # ---------------------------------------------------------------------------
 
 def load_yaml(path):
-    with open(path) as f:
-        return yaml.safe_load(f)
+    """Load YAML via yq (converts to JSON) — no pyyaml dependency."""
+    result = subprocess.run(
+        ["yq", "-o", "json", path],
+        capture_output=True, text=True,
+    )
+    if result.returncode != 0:
+        # Fallback: try python yaml module if available
+        try:
+            import yaml
+            with open(path) as f:
+                return yaml.safe_load(f)
+        except ImportError:
+            raise RuntimeError(f"Failed to parse {path}: yq not found and pyyaml not installed")
+    return json.loads(result.stdout)
 
 
 def yaml_get(data, dot_path, default=None):