feat(seo): GSC verification, robots.txt, IndexNow on deploy

- infra: optional google_site_verification TXT on the apex (Route 53)
- docs: robots.txt (allows AI crawlers) + IndexNow key file
- ci: ping IndexNow after CloudFront invalidation; serve *.txt as must-revalidate instead of immutable

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: cdskill

.github/workflows/deploy.yml

@@ -90,6 +90,9 @@ jobs:
               echo "bucket=${{ vars.DOCS_S3_BUCKET }}" >> "$GITHUB_OUTPUT"
               echo "distribution=${{ vars.DOCS_CF_DISTRIBUTION_ID }}" >> "$GITHUB_OUTPUT"
               echo "role=${{ vars.DOCS_DEPLOY_ROLE_ARN }}" >> "$GITHUB_OUTPUT"
+              # IndexNow: public domain + key (key file lives at apps/docs/public/<key>.txt).
+              echo "host=qalma.dev" >> "$GITHUB_OUTPUT"
+              echo "indexnow_key=57ae097f37a81fa242039798bfdb3226" >> "$GITHUB_OUTPUT"
               ;;
             *)
               echo "::error::No deploy config for '${{ matrix.app }}'"; exit 1 ;;
@@ -102,21 +105,55 @@ jobs:
           aws-region: ${{ vars.AWS_REGION }}
 
       # Pass 1 — hashed, content-addressed assets: cache forever, prune removed.
+      # .txt (robots.txt, IndexNow key) is excluded so it isn't pinned immutable for a year.
       - name: Upload immutable assets
         run: |
           aws s3 sync "${{ steps.cfg.outputs.dist }}" "s3://${{ steps.cfg.outputs.bucket }}" \
-            --delete --exclude "*.html" --exclude "*.xml" \
+            --delete --exclude "*.html" --exclude "*.xml" --exclude "*.txt" \
             --cache-control "public, max-age=31536000, immutable"
 
-      # Pass 2 — HTML + sitemap: always revalidate so deploys show up instantly.
-      - name: Upload HTML + sitemap (revalidate)
+      # Pass 2 — HTML, sitemap, robots.txt + IndexNow key: always revalidate so
+      # deploys show up instantly.
+      - name: Upload HTML, sitemap, txt (revalidate)
         run: |
           aws s3 sync "${{ steps.cfg.outputs.dist }}" "s3://${{ steps.cfg.outputs.bucket }}" \
-            --exclude "*" --include "*.html" --include "*.xml" \
+            --exclude "*" --include "*.html" --include "*.xml" --include "*.txt" \
             --cache-control "public, max-age=0, must-revalidate"
 
       # "/*" = one invalidation path (free up to 1000/month), fresh HTML at every edge.
       - name: Invalidate CloudFront
         run: |
           aws cloudfront create-invalidation \
             --distribution-id "${{ steps.cfg.outputs.distribution }}" --paths "/*"
+
+      # Tell IndexNow (Bing, Yandex, Seznam…; Google is trialing it) about the current
+      # pages so they get crawled in minutes, not days. URLs come from the freshly built
+      # sitemap. Non-blocking — a failed ping never fails the deploy. Skipped for apps
+      # that don't set an IndexNow `host` in the case above.
+      - name: Ping IndexNow
+        if: steps.cfg.outputs.host != ''
+        shell: bash
+        run: |
+          host="${{ steps.cfg.outputs.host }}"
+          key="${{ steps.cfg.outputs.indexnow_key }}"
+          sitemap="${{ steps.cfg.outputs.dist }}/sitemap.xml"
+          url_json=$(grep -oE '<loc>[^<]+</loc>' "$sitemap" | sed -E 's#</?loc>##g' | jq -R . | jq -s 'map(select(length > 0))')
+          count=$(echo "$url_json" | jq 'length')
+          if [ "$count" -eq 0 ]; then
+            echo "IndexNow: no URLs found in $sitemap, skipping."
+            exit 0
+          fi
+          payload=$(jq -n \
+            --arg host "$host" \
+            --arg key "$key" \
+            --arg keyLocation "https://$host/$key.txt" \
+            --argjson urlList "$url_json" \
+            '{host: $host, key: $key, keyLocation: $keyLocation, urlList: $urlList}')
+          code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST 'https://api.indexnow.org/indexnow' \
+            -H 'Content-Type: application/json; charset=utf-8' \
+            -d "$payload")
+          echo "IndexNow → HTTP $code for $count URL(s)"
+          case "$code" in
+            200|202) ;;
+            *) echo "::warning::IndexNow ping returned $code (non-blocking)" ;;
+          esac

apps/docs/public/57ae097f37a81fa242039798bfdb3226.txt

@@ -0,0 +1 @@
+57ae097f37a81fa242039798bfdb3226

apps/docs/public/robots.txt

@@ -0,0 +1,7 @@
+# https://qalma.dev/robots.txt
+# Allow everything, including AI crawlers (GPTBot, ClaudeBot, PerplexityBot, …)
+# so the docs can be cited in AI answers — see GEO / AI-visibility goal.
+User-agent: *
+Allow: /
+
+Sitemap: https://qalma.dev/sitemap.xml

infra/terraform/dns_cert.tf

@@ -63,3 +63,17 @@ resource "aws_route53_record" "aaaa" {
     evaluate_target_health = false
   }
 }
+
+# Google Search Console — domain-property ownership (apex TXT).
+# Route 53 stores every TXT value for a name as ONE record set: any future apex
+# TXT (SPF, other site verifications) must be ADDED to this resource's `records`
+# list, not declared as a separate aws_route53_record (that would conflict).
+resource "aws_route53_record" "google_site_verification" {
+  count = var.google_site_verification != "" ? 1 : 0
+
+  zone_id = data.aws_route53_zone.this.zone_id
+  name    = var.domain_name
+  type    = "TXT"
+  records = [var.google_site_verification]
+  ttl     = 300
+}

infra/terraform/variables.tf

@@ -14,6 +14,12 @@ variable "route53_zone_name" {
   type        = string
 }
 
+variable "google_site_verification" {
+  description = "Google Search Console domain-property TXT verification token (the full 'google-site-verification=...' string). Empty = no record created."
+  type        = string
+  default     = ""
+}
+
 variable "github_repo" {
   description = "owner/repo allowed to assume the deploy role via GitHub OIDC."
   type        = string