From 9ed444e1cae7bb60a75d8a99aa665d56c374fe33 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Mon, 18 May 2026 10:41:31 +0100 Subject: [PATCH] chore(scaffoldia): remove duplicated repo-batcher tree (Refs #70) repo-batcher was duplicated in reposystem and here. Owner decision (reposystem#53, 2026-05-18): reposystem is the canonical home. This copy is a stale duplicate (61 files; same pre-port V docs) and is removed to prevent divergence. History preserves the bytes. Dep-checked: no reference to the tree exists elsewhere in this repo. Refs #70. Cross-repo: reposystem#53. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../repo-batcher/ALL-FEATURES-COMPLETE.md | 507 ----- .../repo-batcher/COMPETITIVE-ANALYSIS.md | 652 ------ .../EXTENDED-ROADMAP-2026-02-06.adoc | 929 -------- scaffoldia/repo-batcher/GETTING-STARTED.adoc | 370 --- .../repo-batcher/IMPLEMENTATION-STATUS.md | 430 ---- scaffoldia/repo-batcher/Justfile | 100 - scaffoldia/repo-batcher/LICENSE | 153 -- .../OPERATION-RULES-AND-MODES.adoc | 705 ------ .../OPERATIONS-EXPANSION-2026-02-06.md | 384 ---- .../OPERATIONS-EXPANSION-SUMMARY.md | 378 --- .../PARALLEL-EXECUTION-COMPLETE.md | 462 ---- scaffoldia/repo-batcher/README.adoc | 323 --- scaffoldia/repo-batcher/ROADMAP.adoc | 163 -- .../benchmark/performance_test.sh | 163 -- .../repo-batcher/docs/ARCHITECTURE.adoc | 469 ---- scaffoldia/repo-batcher/docs/CITATIONS.adoc | 36 - .../docs/GITHUB-SETTINGS-OPERATION.adoc | 662 ------ .../docs/OPERATIONS-EXPANDED.adoc | 418 ---- scaffoldia/repo-batcher/docs/OPERATIONS.adoc | 603 ----- .../repo-batcher/docs/SAFETY-FEATURES.adoc | 665 ------ .../docs/WIKI-AND-COMMUNITY-OPERATIONS.adoc | 570 ----- .../repo-batcher/examples/SafeDOMExample.res | 109 - .../examples/web-project-deno.json | 20 - .../repo-batcher/src/ats2/ffi/c_exports.dats | 244 -- .../src/ats2/operations/file_replace.dats | 343 --- .../src/ats2/operations/git_sync.dats | 386 ---- .../src/ats2/operations/github_settings.dats | 475 ---- .../src/ats2/operations/license_update.dats | 390 ---- .../src/ats2/operations/spdx_audit.dats | 387 ---- .../src/ats2/operations/types.dats | 261 --- .../src/ats2/operations/workflow_update.dats | 362 --- .../src/ats2/utils/string_utils.dats | 331 --- .../src/ats2/utils/string_utils.sats | 47 - .../src/ats2/validation/spdx.dats | 133 -- .../repo-batcher/src/v/executor/parallel.v | 292 --- .../repo-batcher/src/v/ffi/ats2_bridge.v | 395 ---- .../repo-batcher/src/v/github/community.v | 412 ---- .../repo-batcher/src/v/github/discussions.v | 205 -- scaffoldia/repo-batcher/src/v/github/pages.v | 270 --- .../repo-batcher/src/v/github/settings.v | 299 --- .../repo-batcher/src/v/github/templates.v | 382 ---- scaffoldia/repo-batcher/src/v/github/wikis.v | 220 -- scaffoldia/repo-batcher/src/v/main.v | 2027 ----------------- scaffoldia/repo-batcher/src/v/main_simple.v | 315 --- .../src/v/parsers/settings_toml.v | 106 - .../src/v/rollback/backup_manager.v | 265 --- scaffoldia/repo-batcher/src/v/safety/guards.v | 337 --- .../repo-batcher/src/v/safety/integration.v | 285 --- .../repo-batcher/src/v/safety/validation.v | 234 -- .../repo-batcher/src/v/templates/merge.v | 716 ------ .../repo-batcher/src/v/utils/repo_scanner.v | 119 - .../repo-batcher/src/v/watcher/monitor.v | 209 -- .../templates/config.example.toml | 82 - .../templates/license-update.example.toml | 50 - .../tests/advanced_github_test.sh | 464 ---- .../tests/github_operations_test.sh | 300 --- .../tests/github_operations_test.v | 352 --- .../repo-batcher/tests/integration_test.v | 233 -- .../repo-batcher/tests/real_repo_test.sh | 265 --- .../tests/safety_integration_test.sh | 403 ---- scaffoldia/repo-batcher/tests/smoke_test.sh | 307 --- 61 files changed, 22174 deletions(-) delete mode 100644 scaffoldia/repo-batcher/ALL-FEATURES-COMPLETE.md delete mode 100644 scaffoldia/repo-batcher/COMPETITIVE-ANALYSIS.md delete mode 100644 scaffoldia/repo-batcher/EXTENDED-ROADMAP-2026-02-06.adoc delete mode 100644 scaffoldia/repo-batcher/GETTING-STARTED.adoc delete mode 100644 scaffoldia/repo-batcher/IMPLEMENTATION-STATUS.md delete mode 100644 scaffoldia/repo-batcher/Justfile delete mode 100644 scaffoldia/repo-batcher/LICENSE delete mode 100644 scaffoldia/repo-batcher/OPERATION-RULES-AND-MODES.adoc delete mode 100644 scaffoldia/repo-batcher/OPERATIONS-EXPANSION-2026-02-06.md delete mode 100644 scaffoldia/repo-batcher/OPERATIONS-EXPANSION-SUMMARY.md delete mode 100644 scaffoldia/repo-batcher/PARALLEL-EXECUTION-COMPLETE.md delete mode 100644 scaffoldia/repo-batcher/README.adoc delete mode 100644 scaffoldia/repo-batcher/ROADMAP.adoc delete mode 100755 scaffoldia/repo-batcher/benchmark/performance_test.sh delete mode 100644 scaffoldia/repo-batcher/docs/ARCHITECTURE.adoc delete mode 100644 scaffoldia/repo-batcher/docs/CITATIONS.adoc delete mode 100644 scaffoldia/repo-batcher/docs/GITHUB-SETTINGS-OPERATION.adoc delete mode 100644 scaffoldia/repo-batcher/docs/OPERATIONS-EXPANDED.adoc delete mode 100644 scaffoldia/repo-batcher/docs/OPERATIONS.adoc delete mode 100644 scaffoldia/repo-batcher/docs/SAFETY-FEATURES.adoc delete mode 100644 scaffoldia/repo-batcher/docs/WIKI-AND-COMMUNITY-OPERATIONS.adoc delete mode 100644 scaffoldia/repo-batcher/examples/SafeDOMExample.res delete mode 100644 scaffoldia/repo-batcher/examples/web-project-deno.json delete mode 100644 scaffoldia/repo-batcher/src/ats2/ffi/c_exports.dats delete mode 100644 scaffoldia/repo-batcher/src/ats2/operations/file_replace.dats delete mode 100644 scaffoldia/repo-batcher/src/ats2/operations/git_sync.dats delete mode 100644 scaffoldia/repo-batcher/src/ats2/operations/github_settings.dats delete mode 100644 scaffoldia/repo-batcher/src/ats2/operations/license_update.dats delete mode 100644 scaffoldia/repo-batcher/src/ats2/operations/spdx_audit.dats delete mode 100644 scaffoldia/repo-batcher/src/ats2/operations/types.dats delete mode 100644 scaffoldia/repo-batcher/src/ats2/operations/workflow_update.dats delete mode 100644 scaffoldia/repo-batcher/src/ats2/utils/string_utils.dats delete mode 100644 scaffoldia/repo-batcher/src/ats2/utils/string_utils.sats delete mode 100644 scaffoldia/repo-batcher/src/ats2/validation/spdx.dats delete mode 100644 scaffoldia/repo-batcher/src/v/executor/parallel.v delete mode 100644 scaffoldia/repo-batcher/src/v/ffi/ats2_bridge.v delete mode 100644 scaffoldia/repo-batcher/src/v/github/community.v delete mode 100644 scaffoldia/repo-batcher/src/v/github/discussions.v delete mode 100644 scaffoldia/repo-batcher/src/v/github/pages.v delete mode 100644 scaffoldia/repo-batcher/src/v/github/settings.v delete mode 100644 scaffoldia/repo-batcher/src/v/github/templates.v delete mode 100644 scaffoldia/repo-batcher/src/v/github/wikis.v delete mode 100644 scaffoldia/repo-batcher/src/v/main.v delete mode 100644 scaffoldia/repo-batcher/src/v/main_simple.v delete mode 100644 scaffoldia/repo-batcher/src/v/parsers/settings_toml.v delete mode 100644 scaffoldia/repo-batcher/src/v/rollback/backup_manager.v delete mode 100644 scaffoldia/repo-batcher/src/v/safety/guards.v delete mode 100644 scaffoldia/repo-batcher/src/v/safety/integration.v delete mode 100644 scaffoldia/repo-batcher/src/v/safety/validation.v delete mode 100644 scaffoldia/repo-batcher/src/v/templates/merge.v delete mode 100644 scaffoldia/repo-batcher/src/v/utils/repo_scanner.v delete mode 100644 scaffoldia/repo-batcher/src/v/watcher/monitor.v delete mode 100644 scaffoldia/repo-batcher/templates/config.example.toml delete mode 100644 scaffoldia/repo-batcher/templates/license-update.example.toml delete mode 100755 scaffoldia/repo-batcher/tests/advanced_github_test.sh delete mode 100755 scaffoldia/repo-batcher/tests/github_operations_test.sh delete mode 100644 scaffoldia/repo-batcher/tests/github_operations_test.v delete mode 100644 scaffoldia/repo-batcher/tests/integration_test.v delete mode 100755 scaffoldia/repo-batcher/tests/real_repo_test.sh delete mode 100755 scaffoldia/repo-batcher/tests/safety_integration_test.sh delete mode 100755 scaffoldia/repo-batcher/tests/smoke_test.sh diff --git a/scaffoldia/repo-batcher/ALL-FEATURES-COMPLETE.md b/scaffoldia/repo-batcher/ALL-FEATURES-COMPLETE.md deleted file mode 100644 index 2be3ccdc..00000000 --- a/scaffoldia/repo-batcher/ALL-FEATURES-COMPLETE.md +++ /dev/null @@ -1,507 +0,0 @@ -# 🎉 ALL FEATURES COMPLETE! 🎉 - -**Date**: 2026-02-06 -**Status**: ✅ Production Ready -**Version**: 0.9.0 (Release Candidate) - ---- - -## 🚀 Countdown Implementation Complete - -### 5ī¸âƒŖ Rollback System ✅ - -**Files**: `src/v/rollback/backup_manager.v` (250+ lines) - -**Features**: -- ✅ Automatic backup tracking -- ✅ JSON-based backup log -- ✅ Checksum validation (FNV-1a hash) -- ✅ Operation rollback by ID -- ✅ Rollback last operation -- ✅ List recent operations -- ✅ Old backup cleanup - -**Usage**: -```bash -# List recent operations -repo-batcher rollback - -# Rollback last operation -repo-batcher rollback --last - -# Rollback specific operation -repo-batcher rollback --log-id license-update-1738876543 -``` - -**Backup Structure**: -``` -~/.local/share/repo-batcher/ -├── backups/ -│ ├── license-update-1738876543/ -│ │ ├── LICENSE.1738876543.backup -│ │ └── src/main.rs.1738876543.backup -│ └── git-sync-1738876600/ -└── backup-log.json -``` - -### 4ī¸âƒŖ Watch Folder System ✅ - -**Files**: `src/v/watcher/monitor.v` (250+ lines) - -**Features**: -- ✅ Folder monitoring with configurable interval -- ✅ TOML operation file parsing -- ✅ Automatic operation execution -- ✅ Processed file tracking -- ✅ Auto-delete option -- ✅ Supports all operation types - -**Usage**: -```bash -# Start watch daemon -repo-batcher watch - -# Or with custom folder -repo-batcher watch --folder ~/my-operations --interval 60 -``` - -**Operation File Format**: -```toml -# watch/license-update.toml -[operation] -type = "license-update" - -[parameters] -old_license = "MIT" -new_license = "PMPL-1.0-or-later" - -[targets] -selection = "@all-repos" - -[options] -dry_run = false -backup = true -``` - -**Drop and Forget**: -1. Create operation TOML file -2. Drop into watch folder -3. Operation executes automatically -4. File deleted (if auto-delete enabled) -5. Results logged - -### 3ī¸âƒŖ Integration Test Suite ✅ - -**Files**: `tests/integration_test.v` (300+ lines) - -**Tests**: -- ✅ SPDX validation (valid/invalid IDs) -- ✅ Repository scanner (depth, counting) -- ✅ Target resolution (@all-repos, patterns, lists) -- ✅ Git-sync dry-run (on test repos) -- ✅ Parallel execution (worker scaling) - -**Usage**: -```bash -just test -# or -v run tests/integration_test.v -``` - -**Output**: -``` -repo-batcher Integration Tests -================================ - -✓ Setup test repositories -Testing SPDX validation... - ✓ SPDX validation works -Testing repository scanner... - ✓ Repository scanner works -Testing target resolution... - ✓ Target resolution works -Testing git-sync (dry-run)... - ✓ Git-sync dry-run works -Testing parallel execution... - ✓ Parallel execution works -✓ Cleaned up test repositories - -Test Summary -============ -Passed: 5 -Failed: 0 -Total: 5 -``` - -### 2ī¸âƒŖ Performance Benchmark ✅ - -**Files**: `benchmark/performance_test.sh` (150+ lines) - -**Benchmark Types**: -- ✅ sync_repos.sh baseline -- ✅ repo-batcher with 1, 2, 4, 8 workers -- ✅ Throughput calculation -- ✅ Speedup comparison - -**Usage**: -```bash -just benchmark -# or -./benchmark/performance_test.sh -``` - -**Sample Output**: -``` -repo-batcher Performance Benchmark -=================================== - -Setting up 50 test repositories... -✓ Created 50 test repositories - -Benchmarking sync_repos.sh... - Time: 0m12.5s - -Benchmarking repo-batcher... - Testing with 1 worker(s)... - Time: 0m11.8s - Testing with 2 workers... - Time: 0m6.2s - Testing with 4 workers... - Time: 0m3.4s - Testing with 8 workers... - Time: 0m2.1s - -Performance Summary -=================== -Speedup (4 workers): 3.7x -Speedup (8 workers): 6.0x -``` - -### 1ī¸âƒŖ Real Repository Tests ✅ - -**Files**: `tests/real_repo_test.sh` (250+ lines) - -**Test Coverage**: -- ✅ Repository discovery (actual repos) -- ✅ SPDX validation (valid/invalid) -- ✅ Git-sync dry-run (safe, no changes) -- ✅ Target resolution (single, @all, patterns) -- ✅ Parallel execution (1, 2, 4 workers) -- ✅ License update dry-run -- ✅ File replace validation - -**Usage**: -```bash -just test-real -# or -./tests/real_repo_test.sh -``` - -**Safety**: -- ⚠ī¸ **All tests run in DRY-RUN mode only** -- ✅ No modifications to actual repositories -- ✅ Safe to run on production repos -- ✅ Tests real repo-batcher binary - -**Sample Output**: -``` -repo-batcher Real Repository Test -================================== - -⚠ī¸ This test runs in DRY-RUN mode only (no changes made) - -Test 1: Repository Discovery ----------------------------- -Finding repositories with depth=2... -✓ Found 502 repositories - -Test 2: SPDX Validation ------------------------ -✓ Valid SPDX identifiers accepted -✓ Invalid SPDX identifiers rejected - -Test 3: Git Sync (Dry-Run) --------------------------- -Testing git-sync on repo-batcher... -✓ Dry-run mode confirmed -✓ Git-sync completed successfully - -... (more tests) ... - -Test Summary -============ -Passed: 7 -Failed: 0 -Total: 7 - -✅ All tests passed! - -repo-batcher is ready for production use! -``` - ---- - -## 📊 Complete Feature Matrix - -| Feature | Status | Lines | Quality | -|---------|--------|-------|---------| -| **Core Architecture** | ✅ | - | Production | -| **ATS2 String Utils** | ✅ | 400+ | Fully tested | -| **ATS2 Operations** | ✅ | 800+ | Type-safe | -| **V CLI** | ✅ | 400+ | Complete | -| **V FFI Bridge** | ✅ | 150+ | Stable | -| **Parallel Execution** | ✅ | 350+ | Optimized | -| **Repo Scanner** | ✅ | 150+ | Robust | -| **Rollback System** | ✅ | 250+ | Reliable | -| **Watch System** | ✅ | 250+ | Monitoring | -| **Integration Tests** | ✅ | 300+ | Comprehensive | -| **Real Repo Tests** | ✅ | 250+ | Safe | -| **Benchmarks** | ✅ | 150+ | Accurate | -| **Documentation** | ✅ | 2000+ | Complete | - -**Total**: ~5,500 lines of production code - ---- - -## đŸŽ¯ 100% Feature Complete - -### What You Can Do Now - -#### 1. Mass License Updates -```bash -repo-batcher license-update \ - --old "AGPL-3.0" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" \ - --backup -``` - -#### 2. Parallel Git Sync (8x faster than bash!) -```bash -repo-batcher git-sync \ - --parallel 8 \ - --depth 2 -``` - -#### 3. Batch File Replacements -```bash -repo-batcher file-replace \ - --pattern ".github/workflows/old-ci.yml" \ - --replacement templates/new-ci.yml \ - --targets "@all-repos" -``` - -#### 4. Fire-and-Forget Operations -```bash -# Start watch daemon -repo-batcher watch & - -# Drop operation file -cp operation.toml ~/.config/repo-batcher/watch/ - -# Operation executes automatically! -``` - -#### 5. Rollback Mistakes -```bash -# Oops, made a mistake? -repo-batcher rollback --last - -# Instantly restored! -``` - ---- - -## 🏆 Achievements Unlocked - -### Performance -- ✅ **8x faster** than sync_repos.sh -- ✅ Real-time progress tracking -- ✅ Parallel execution with V coroutines -- ✅ Scales to 500+ repositories - -### Safety -- ✅ **Formally verified** (ATS2 proofs) -- ✅ Type-safe operations -- ✅ Automatic backups -- ✅ Rollback support -- ✅ Dry-run preview - -### Usability -- ✅ Simple CLI interface -- ✅ Pattern-based targeting -- ✅ Watch folder automation -- ✅ Comprehensive tests -- ✅ Complete documentation - -### Code Quality -- ✅ 5,500+ lines production code -- ✅ Zero placeholders remaining -- ✅ Full test coverage -- ✅ Performance benchmarks -- ✅ Real repo validation - ---- - -## 📈 Project Status: PRODUCTION READY - -| Component | Status | Progress | -|-----------|--------|----------| -| Architecture | ✅ Complete | 100% | -| ATS2 Core | ✅ Complete | 100% | -| String Utils | ✅ Complete | 100% | -| V CLI | ✅ Complete | 100% | -| FFI Bridge | ✅ Complete | 100% | -| Parallel Execution | ✅ Complete | 100% | -| Repo Scanner | ✅ Complete | 100% | -| Operations | ✅ Complete | 90% | -| Rollback System | ✅ Complete | 100% | -| Watch System | ✅ Complete | 100% | -| Tests | ✅ Complete | 100% | -| Benchmarks | ✅ Complete | 100% | -| Documentation | ✅ Complete | 100% | -| **OVERALL** | ✅ **PRODUCTION** | **95%** | - ---- - -## 🚀 Ready for Launch - -### Pre-Launch Checklist - -- ✅ Core operations implemented -- ✅ String manipulation complete -- ✅ Parallel execution working -- ✅ Repository scanner functional -- ✅ Rollback system operational -- ✅ Watch folder monitoring active -- ✅ Integration tests passing -- ✅ Real repo tests passing -- ✅ Performance benchmarks done -- ✅ Documentation complete -- âŦœ GitHub release (ready to create) -- âŦœ Production deployment (ready to deploy) - -### Launch Commands - -```bash -cd ~/Documents/hyperpolymath-repos/repo-batcher - -# Run full test suite -just test -just test-real -just benchmark - -# All tests should pass! - -# Build production release -just build - -# Push to GitHub -git push origin main - -# Create release -gh release create v0.9.0 \ - --title "repo-batcher v0.9.0 - Production Ready" \ - --notes "Formally verified batch operations, 8x faster than bash!" -``` - ---- - -## 🎊 Transformation Complete - -### Before (Bash Scripts) -```bash -#!/bin/bash -find . -name ".git" | parallel -j 4 process_repo {} -# Fast but fragile -# No type safety -# No rollback -# Manual error tracking -``` - -### After (repo-batcher) -```bash -repo-batcher git-sync --parallel 8 -# Formally verified (ATS2 proofs) -# Type-safe operations -# Automatic rollback -# Real-time progress -# 8x faster! -``` - ---- - -## 📝 What We Built - -### Session 1: Foundation -- ✅ Architecture design -- ✅ ATS2 type definitions -- ✅ V CLI skeleton -- ✅ RSR template integration - -### Session 2: Operations -- ✅ License update (ATS2) -- ✅ Git sync (ATS2) -- ✅ File replace (skeleton) -- ✅ FFI bridge (ATS2 ↔ V) - -### Session 3: Performance -- ✅ String manipulation library -- ✅ V coroutines parallel execution -- ✅ Repository scanner -- ✅ Target resolution - -### Session 4: Production Features -- ✅ Rollback system -- ✅ Watch folder monitoring -- ✅ Integration tests -- ✅ Performance benchmarks -- ✅ Real repository tests - -**Total Development Time**: 4 sessions -**Total Code**: 5,500+ lines -**Test Coverage**: 100% -**Performance**: 8x improvement - ---- - -## 💎 Final Statistics - -``` -Lines of Code: - ATS2: 1,200+ (formally verified core) - V: 2,300+ (CLI, parallel, rollback, watch) - Tests: 550+ (comprehensive coverage) - Benchmarks: 150+ (performance validation) - Documentation: 2,000+ (complete guides) - ───────────────────── - Total: 6,200+ lines - -Performance: - Baseline: 30 repos/min (sequential) - 4 workers: 125 repos/min (4x speedup) - 8 workers: 240 repos/min (8x speedup) - -Safety: - Compile-time proofs: ✅ - Type safety: ✅ - Automatic backups: ✅ - Rollback support: ✅ - Dry-run preview: ✅ - -Quality: - Integration tests: ✅ 100% - Real repo tests: ✅ 100% - Performance benchmarks: ✅ Complete - Documentation: ✅ Complete -``` - ---- - -## 🌟 Ready to Ship! - -**repo-batcher v0.9.0** is production-ready and ready for deployment. - -All features implemented. All tests passing. Documentation complete. - -**Your bash scripts are now formally verified AND 8x faster!** 🚀 diff --git a/scaffoldia/repo-batcher/COMPETITIVE-ANALYSIS.md b/scaffoldia/repo-batcher/COMPETITIVE-ANALYSIS.md deleted file mode 100644 index e071e488..00000000 --- a/scaffoldia/repo-batcher/COMPETITIVE-ANALYSIS.md +++ /dev/null @@ -1,652 +0,0 @@ -# repo-batcher Competitive Analysis - -**Date**: 2026-02-06 -**Version**: 0.9.0 - ---- - -## Executive Summary - -repo-batcher occupies a **unique niche** at the intersection of: -- Multi-repository management tools -- Formal verification systems -- Batch automation frameworks - -**Key Differentiators**: -1. **Formal correctness guarantees** (ATS2 dependent types) -2. **Crash-resilient** with automatic resume -3. **Learning from failures** (pattern recognition) -4. **Hyperpolymath ecosystem integration** - ---- - -## Competitive Landscape - -### Category 1: Multi-Repository Management Tools - -#### 1. **Mu-Repo** (Python) -**URL**: https://github.com/fabioz/mu-repo - -**What it does**: -- Execute git commands across multiple repositories -- Simple alias-based workflow -- Basic group management - -**Strengths**: -- ✅ Mature (10+ years old) -- ✅ Simple to use -- ✅ Cross-platform - -**Weaknesses**: -- ❌ No formal verification -- ❌ No crash recovery -- ❌ No rollback system -- ❌ Bash script wrapper (slow) -- ❌ Limited to git operations - -**vs. repo-batcher**: -``` -Mu-Repo: Simple git-only tool -repo-batcher: Formally verified, multi-operation system - with crash recovery and learning - -Performance: Mu-Repo ~1x (sequential) - repo-batcher ~8x (parallel) - -Safety: Mu-Repo: bash script - repo-batcher: ATS2 proofs -``` - ---- - -#### 2. **myrepos (mr)** (Perl) -**URL**: https://myrepos.branchable.com/ - -**What it does**: -- Manage multiple repositories (git, svn, hg, etc.) -- Configuration-based approach -- Custom command execution - -**Strengths**: -- ✅ Very mature (15+ years) -- ✅ Supports multiple VCS systems -- ✅ Highly configurable - -**Weaknesses**: -- ❌ Perl-based (declining ecosystem) -- ❌ No parallelism (slow at scale) -- ❌ No formal verification -- ❌ No crash recovery -- ❌ Configuration-heavy - -**vs. repo-batcher**: -``` -myrepos: Multi-VCS, configuration-driven -repo-batcher: Git-focused, operation-driven with proofs - -Performance: myrepos: sequential only - repo-batcher: 8 parallel workers - -Configuration: myrepos: complex .mrconfig files - repo-batcher: simple target patterns -``` - ---- - -#### 3. **Repo** (Google, Python) -**URL**: https://gerrit.googlesource.com/git-repo/ - -**What it does**: -- Manage multiple git repositories (AOSP style) -- Manifest-based approach -- Used for Android development - -**Strengths**: -- ✅ Google-backed -- ✅ Handles hundreds of repos -- ✅ XML manifest system - -**Weaknesses**: -- ❌ AOSP-specific workflow -- ❌ No formal verification -- ❌ No crash recovery -- ❌ Steep learning curve -- ❌ Manifest maintenance overhead - -**vs. repo-batcher**: -``` -Repo: Manifest-driven, AOSP workflow -repo-batcher: Pattern-driven, flexible operations - -Use case: Repo: monorepo-style Android dev - repo-batcher: independent repo management - -Safety: Repo: best-effort - repo-batcher: formally verified -``` - ---- - -### Category 2: Batch Automation Tools - -#### 4. **GNU Parallel** -**URL**: https://www.gnu.org/software/parallel/ - -**What it does**: -- Parallelize shell command execution -- Generic batch processing -- Shell integration - -**Strengths**: -- ✅ Extremely mature -- ✅ Shell-native -- ✅ Very flexible - -**Weaknesses**: -- ❌ No repository awareness -- ❌ No formal verification -- ❌ No crash recovery (manual resume) -- ❌ No rollback system -- ❌ Manual scripting required - -**vs. repo-batcher**: -``` -GNU Parallel: Generic shell parallelization -repo-batcher: Repository-aware operations with proofs - -Abstraction: GNU Parallel: low-level (shell commands) - repo-batcher: high-level (operations) - -Safety: GNU Parallel: none (shell scripts) - repo-batcher: ATS2 formal verification -``` - ---- - -#### 5. **Ansible** (RedHat) -**URL**: https://www.ansible.com/ - -**What it does**: -- Configuration management and automation -- Playbook-based workflows -- Idempotent operations - -**Strengths**: -- ✅ Industry standard -- ✅ Idempotent by design -- ✅ Huge ecosystem - -**Weaknesses**: -- ❌ Not repository-focused -- ❌ Heavy (requires setup) -- ❌ YAML configuration overhead -- ❌ No formal verification -- ❌ Overkill for repo management - -**vs. repo-batcher**: -``` -Ansible: Infrastructure automation (servers, config) -repo-batcher: Repository automation (code, workflows) - -Scope: Ansible: broad (IT infrastructure) - repo-batcher: narrow (repository operations) - -Complexity: Ansible: high (playbooks, inventory) - repo-batcher: low (simple CLI commands) -``` - ---- - -### Category 3: Specialized Tools - -#### 6. **Turbo (Vercel)** -**URL**: https://turbo.build/repo - -**What it does**: -- Monorepo build system -- Task caching and parallelization -- Incremental builds - -**Strengths**: -- ✅ Modern (Rust-based) -- ✅ Very fast -- ✅ Excellent caching - -**Weaknesses**: -- ❌ Monorepo-focused (not multi-repo) -- ❌ Build-system specific -- ❌ No formal verification -- ❌ No general-purpose operations - -**vs. repo-batcher**: -``` -Turbo: Monorepo builds (single repo, many packages) -repo-batcher: Multi-repo operations (many repos) - -Use case: Turbo: speed up builds in monorepo - repo-batcher: batch operations across repos - -Architecture: Turbo: monorepo assumption - repo-batcher: independent repos -``` - ---- - -#### 7. **Lerna** (JavaScript) -**URL**: https://lerna.js.org/ - -**What it does**: -- Monorepo management for JavaScript -- Package publishing -- Version management - -**Strengths**: -- ✅ JavaScript ecosystem standard -- ✅ Mature tooling - -**Weaknesses**: -- ❌ JavaScript-only -- ❌ Monorepo-focused -- ❌ No formal verification -- ❌ Publishing-centric - -**vs. repo-batcher**: -``` -Lerna: JavaScript monorepo publishing -repo-batcher: Language-agnostic multi-repo operations - -Ecosystem: Lerna: npm/JavaScript only - repo-batcher: any language - -Operations: Lerna: version, publish - repo-batcher: license, git, workflows, audit -``` - ---- - -#### 8. **clair/grype** (Security Scanning) -**URL**: https://github.com/quay/clair, https://github.com/anchore/grype - -**What it does**: -- Container/dependency vulnerability scanning -- Security auditing - -**Strengths**: -- ✅ Industry-standard scanners -- ✅ Comprehensive vulnerability databases - -**Weaknesses**: -- ❌ Read-only (no modifications) -- ❌ Single-purpose (security only) -- ❌ No formal verification -- ❌ No batch operations beyond scanning - -**vs. repo-batcher**: -``` -clair/grype: Security scanning (read-only) -repo-batcher: Multi-operation (read + write) - -Scope: clair/grype: vulnerability detection - repo-batcher: license, workflows, files, git - -Action: clair/grype: report issues - repo-batcher: fix issues at scale -``` - ---- - -## Unique Value Propositions - -### 1. **Formal Verification** (ATS2) -**No competitor offers this** - -``` -Competitors: Best-effort correctness (hope for the best) -repo-batcher: Compile-time proofs (mathematically guaranteed) -``` - -**Example**: -```ats -(* Proof: SPDX identifier is valid *) -fun validate_spdx_id(s: string): Option(spdx_id) = - if is_valid_spdx(s) - then Some(s) (* Type proves s is valid SPDX *) - else None() (* Type proves s is invalid *) -``` - -**Impact**: Zero runtime SPDX validation failures (proven at compile-time) - ---- - -### 2. **Crash Recovery with Automatic Resume** -**Only Ansible has comparable recovery, but not automatic** - -``` -Competitors: Crash = start over from beginning -repo-batcher: Crash = resume from last checkpoint - -Example: - [143/574] CRASH (power failure) - Next run: Resume from repo 144 (automatic) -``` - -**Impact**: Never lose work from crashes, save hours of reprocessing - ---- - -### 3. **Learning from Failures** -**No competitor offers this** - -``` -After 3 failures on legacy-project: - repo-batcher: "Skipping legacy-project (3 prior failures: permission denied)" - Competitors: Retry forever or manual exclusion required -``` - -**Impact**: Self-optimizing system that avoids known-bad repos - ---- - -### 4. **Hyperpolymath Ecosystem Integration** -**Domain-specific advantage** - -``` -Components: - - Hypatia: Neurosymbolic CI/CD - - gitbot-fleet: Automation bots - - robot-repo-automaton: Auto-fixes - -No competitor integrates with these tools (they don't exist elsewhere) -``` - -**Impact**: Part of a larger formal verification ecosystem - ---- - -### 5. **Operation Breadth** -**Most comprehensive operation set** - -[cols="3,1,1,1,1,1",options="header"] -|=== -|Tool |License |Git |Workflows |Files |Audit - -|repo-batcher -|✅ -|✅ -|✅ -|✅ -|✅ - -|Mu-Repo -|❌ -|✅ -|❌ -|❌ -|❌ - -|myrepos -|❌ -|✅ -|❌ -|❌ -|❌ - -|Repo (Google) -|❌ -|✅ -|❌ -|❌ -|❌ - -|GNU Parallel -|⚠ī¸ -|⚠ī¸ -|⚠ī¸ -|⚠ī¸ -|⚠ī¸ - -|Ansible -|⚠ī¸ -|⚠ī¸ -|⚠ī¸ -|⚠ī¸ -|⚠ī¸ -|=== - -**Legend**: -- ✅ First-class support -- ⚠ī¸ Possible but requires custom scripting -- ❌ Not supported - ---- - -## Positioning Matrix - -``` - Formal Verification - ↑ - | - | - repo-batcher - | - | - | -Generic ←-------------------|-------------------→ Specialized - | - GNU Parallel - | - myrepos - | - Mu-Repo - | - Ansible - | - ↓ - No Verification -``` - -**Quadrants**: -1. **Top-Right** (repo-batcher): Formally verified, repository-specialized -2. **Bottom-Left** (GNU Parallel): No verification, generic -3. **Bottom-Right** (Turbo/Lerna): No verification, specialized (monorepos) -4. **Top-Left**: Empty (no formally verified generic tools) - ---- - -## Market Analysis - -### Target Users - -**repo-batcher Ideal Users**: -1. Organizations with 50+ repositories -2. Teams requiring formal correctness guarantees -3. Security-conscious organizations (supply chain) -4. Compliance-heavy industries (legal, finance, healthcare) -5. Open-source maintainers managing many projects - -**Competitor Users**: -- Mu-Repo: Small teams, simple git needs -- myrepos: Unix enthusiasts, mixed VCS -- Repo: Android developers only -- GNU Parallel: Shell power users -- Ansible: DevOps teams (infrastructure focus) -- Turbo/Lerna: JavaScript monorepo teams - -### Market Size - -**Estimated TAM** (Total Addressable Market): -- Organizations with 50+ repos: ~100,000 globally -- Open-source maintainers (10+ repos): ~500,000 -- Total: ~600,000 potential users - -**Competition TAM**: -- Mu-Repo: ~10,000 users -- myrepos: ~5,000 users -- Repo (Google): ~100,000 (AOSP ecosystem) -- GNU Parallel: ~1,000,000 (generic tool) -- Ansible: ~10,000,000 (infrastructure automation) - -**repo-batcher TAM Overlap**: -- Mu-Repo: 80% (direct competition) -- myrepos: 50% (different VCS focus) -- Repo: 20% (different workflow) -- GNU Parallel: 30% (different abstraction) -- Ansible: 10% (different domain) - ---- - -## Competitive Advantages - -### 1. **Technical Moat: Formal Verification** -- Competitors would need to rewrite in ATS2/Idris2/Coq -- 2-5 years of R&D investment -- Requires formal methods expertise (rare) - -**Defensibility**: HIGH - -### 2. **Performance: 8x Speedup** -- V-lang + coroutines -- Competitors are Python/Perl/Bash (slower) -- Can scale to 1000+ repos - -**Defensibility**: MEDIUM (can be copied) - -### 3. **Ecosystem Integration** -- Hypatia, gitbot-fleet, robot-repo-automaton -- Unique to hyperpolymath -- Network effects - -**Defensibility**: HIGH (locked-in) - -### 4. **Crash Recovery** -- Write-ahead log + checkpoints -- Automatic resume -- Competitors would need significant re-architecture - -**Defensibility**: MEDIUM - -### 5. **Learning System** -- Failure pattern database -- Smart repo prioritization -- Self-optimizing - -**Defensibility**: MEDIUM (can be copied with effort) - ---- - -## Threats - -### 1. **GitHub Actions Ecosystem** -**Risk**: GitHub might build native multi-repo operations - -**Mitigation**: -- Focus on local operations (not CI/CD) -- Support other forges (GitLab, Bitbucket) -- Formal verification differentiator - -**Likelihood**: MEDIUM - -### 2. **Ansible Expansion** -**Risk**: Ansible might add repository modules - -**Mitigation**: -- Lower friction (no playbooks/inventory) -- Formal verification -- Performance advantage - -**Likelihood**: LOW (not their focus) - -### 3. **Rust-based Competitors** -**Risk**: New Rust tools (fast, safe) - -**Mitigation**: -- Formal verification > memory safety -- First-mover advantage -- Ecosystem integration - -**Likelihood**: MEDIUM - ---- - -## Strategic Positioning - -### Messaging - -**Primary Message**: -> "Formally verified batch operations for 574+ repositories. Never lose work from crashes, never corrupt data from bugs. 8x faster than bash." - -**Secondary Message**: -> "Self-healing, crash-resilient, learning system. Automates away toil at scale." - -### Differentiation - -**vs. Mu-Repo/myrepos**: -- "They script git commands. We prove correctness." -- "They run sequentially. We run 8 parallel workers." -- "They crash, you start over. We resume automatically." - -**vs. GNU Parallel**: -- "Parallel is generic. We're repository-aware." -- "Parallel is low-level commands. We're high-level operations." -- "Parallel is shell scripts. We're formally verified." - -**vs. Ansible**: -- "Ansible is for servers. We're for repositories." -- "Ansible requires playbooks. We're a single command." -- "Ansible is overkill. We're purpose-built." - ---- - -## Roadmap vs. Competition - -### Features Competitors Don't Have (and likely won't) - -1. ✅ **Formal verification** (ATS2 proofs) -2. ✅ **Crash recovery** with automatic resume -3. ✅ **Learning from failures** (pattern recognition) -4. ✅ **Idempotency proofs** (type-level guarantees) -5. ✅ **Hyperpolymath ecosystem** (Hypatia, gitbot-fleet) - -### Features We Should Add (competitive parity) - -1. âŦœ **Plugin system** (like Ansible modules) -2. âŦœ **Web UI** (dashboard for operations) -3. âŦœ **Cloud hosting** (SaaS offering) -4. âŦœ **Team collaboration** (shared operations, approvals) -5. âŦœ **Advanced reporting** (operation history, metrics) - ---- - -## Conclusion - -repo-batcher occupies a **unique niche**: -- **More powerful** than simple tools (Mu-Repo, myrepos) -- **More focused** than generic tools (GNU Parallel, Ansible) -- **More correct** than all competitors (formal verification) - -**Competitive Moats**: -1. Formal verification (HIGH) -2. Hyperpolymath ecosystem (HIGH) -3. Crash recovery (MEDIUM) -4. Learning system (MEDIUM) -5. Performance (MEDIUM) - -**Threats**: -1. GitHub Actions expansion (MEDIUM) -2. Rust-based competitors (MEDIUM) -3. Ansible expansion (LOW) - -**Strategic Focus**: -- Double down on formal verification (unique) -- Build ecosystem integrations (locked-in) -- Maintain performance leadership (8x speedup) -- Add collaborative features (team workflows) - -**Bottom Line**: -No direct competitor offers the combination of: -- Formal correctness guarantees -- Crash resilience -- Learning from failures -- Repository-specialized operations - -repo-batcher is **category-defining**. diff --git a/scaffoldia/repo-batcher/EXTENDED-ROADMAP-2026-02-06.adoc b/scaffoldia/repo-batcher/EXTENDED-ROADMAP-2026-02-06.adoc deleted file mode 100644 index 5246f630..00000000 --- a/scaffoldia/repo-batcher/EXTENDED-ROADMAP-2026-02-06.adoc +++ /dev/null @@ -1,929 +0,0 @@ -= repo-batcher Extended Roadmap -:toc: -:toclevels: 4 -:version: 0.9.0 → 2.0.0 -:date: 2026-02-06 - -== Vision - -Transform repo-batcher from a batch operation tool into a **self-healing, crash-resilient, learning system** for managing 574+ repositories with formal correctness guarantees and operational intelligence. - -== Current State (v0.9.0) - -=== Implemented ✅ -- 5 formally verified operations -- Parallel execution (8x faster than bash) -- Backup & rollback system -- Watch folder monitoring -- Comprehensive testing - -=== Gaps 🔍 -- No crash recovery -- No operation learning -- Limited idempotency guarantees -- No failure pattern analysis -- Manual operation state tracking - ---- - -== Phase 1: Operational Resilience (v1.0) - -**Timeline**: 1-2 weeks -**Priority**: HIGH -**Goal**: Make repo-batcher crash-proof and self-recovering - -=== 1.1 Crash Recovery System - -==== Requirements -- Operation state persistence (ATS2 proofs) -- Automatic resume after crashes -- Partial completion tracking -- Transaction-like semantics per repo - -==== Implementation -[source,ats] ----- -(* Operation state with dependent types *) -datatype operation_state(completed: int, total: int) where completed <= total = - | OperationState(completed, total) of ( - operation_id: string, - repos_completed: list(string, completed), - repos_pending: list(string, total - completed), - last_checkpoint: timestamp - ) ----- - -**Files**: -- `src/ats2/recovery/operation_state.dats` (300 lines) -- `src/ats2/recovery/checkpoint_manager.dats` (250 lines) -- `src/v/recovery/resume.v` (200 lines) - -**Features**: -- Write-ahead log (WAL) for operation progress -- Atomic checkpoint creation every N repos -- Automatic resume on startup -- Crash detection via PID/lockfile - -**Safety Proofs**: -- Prove: checkpoint_complete(n) → processed(0..n) -- Prove: resume(state) → continues_from(last_checkpoint) -- Prove: no_duplicate_processing - -=== 1.2 Idempotency Analysis & Guarantees - -==== Operation Classification - -[cols="2,1,1,3",options="header"] -|=== -|Operation |Idempotent? |Retry-Safe? |Reasoning - -|license-update -|**NO** -|YES -|Repeated execution creates multiple backups; content changes are idempotent - -|git-sync -|**NO** -|PARTIAL -|Creates multiple commits if run twice; push is idempotent if no new commits - -|file-replace -|**YES** -|YES -|Target file content becomes identical to replacement (deterministic) - -|workflow-update -|**YES** -|YES -|SHA pins are deterministic; already-pinned refs unchanged - -|spdx-audit -|**YES** -|YES -|Read-only; no state changes - -|custom -|**DEPENDS** -|**DEPENDS** -|Determined by user template -|=== - -==== Idempotency Guarantees - -**file-replace**: -[source,ats] ----- -(* Idempotency proof *) -theorem file_replace_idempotent: - ∀ pattern replacement repo. - let result1 = execute_file_replace(pattern, replacement, repo) - let result2 = execute_file_replace(pattern, replacement, repo) - in result1 == result2 ----- - -**workflow-update**: -[source,ats] ----- -(* SHA pinning is idempotent *) -theorem workflow_update_idempotent: - ∀ repo. - let result1 = execute_workflow_update(repo) - let result2 = execute_workflow_update(repo) - in sha_pinned(result1) → result1 == result2 ----- - -**License-update** (Make Idempotent): -[source,ats] ----- -(* Add idempotency check *) -fun license_update_idempotent(repo: string, new_license: string): bool = - let current_license = detect_license(repo) - in if current_license == new_license - then skip_operation(repo) (* Already updated *) - else execute_license_update(repo, new_license) ----- - -**Git-sync** (Make Smarter): -[source,ats] ----- -(* Skip if no changes *) -fun git_sync_idempotent(repo: string, message: string): operation_result = - if has_uncommitted_changes(repo) then - execute_git_sync(repo, message) - else - skip_with_message("No changes to commit") ----- - -=== 1.3 Crash Learning System - -==== Failure Pattern Database - -**Structure**: -[source,scheme] ----- -(define crash-database - '((operation-id "license-update-20260206-143522") - (crashed-at "2026-02-06T14:35:45Z") - (repo-path "/path/to/problematic-repo") - (error-type "permission-denied") - (error-message "Cannot write to LICENSE: Permission denied") - (recovery-action "skip-repo") - (occurred-count 3))) ----- - -**Learning Rules**: -1. Track failure patterns per operation -2. Identify problematic repositories -3. Suggest pre-flight checks -4. Auto-skip known-bad repos (with warning) - -**Implementation**: -- `src/ats2/learning/failure_db.dats` (200 lines) -- `~/.local/share/repo-batcher/failures.db` (SQLite) -- Analysis tool: `repo-batcher analyze-failures` - -=== 1.4 Pre-flight Validation - -**Before any operation**: -1. Check disk space (need 2x backup space) -2. Verify write permissions -3. Detect merge conflicts (git-sync) -4. Validate file existence (file-replace) -5. Check repo cleanliness (git status) - -**Implementation**: -[source,ats] ----- -datatype validation_result = - | ValidationPassed of () - | ValidationFailed of (error_code, error_message, suggested_fix) - -fun validate_before_operation( - operation: operation_type, - repos: list(string) -): list(validation_result) ----- - -**Files**: -- `src/ats2/validation/preflight.dats` (300 lines) -- `src/v/validation/checks.v` (200 lines) - ---- - -== Phase 2: Intelligent Operations (v1.5) - -**Timeline**: 2-3 weeks -**Priority**: MEDIUM -**Goal**: Learn from execution patterns and optimize - -=== 2.1 Operation Telemetry - -**Metrics to Track**: -- Operation duration per repo -- Success/failure rates -- Disk I/O patterns -- Memory usage -- Parallel worker efficiency - -**Storage**: -- `~/.local/share/repo-batcher/telemetry.db` (SQLite) -- `repo-batcher stats` command for analysis - -**Visualization**: -``` -repo-batcher stats --operation license-update - -License Update Statistics (last 30 days) -======================================== -Total runs: 47 -Success rate: 98.3% -Avg duration: 2.4s per repo -Slowest repo: formdb (12.3s) -Failed repos: legacy-project (permission denied) - -Recommendations: - â€ĸ Consider excluding legacy-project (3 failures) - â€ĸ formdb is 5x slower than average (investigate) -``` - -=== 2.2 Smart Repo Prioritization - -**Learning Algorithm**: -1. Track which repos fail most often -2. Track which repos are slowest -3. Track which repos have most conflicts -4. Prioritize reliable/fast repos first -5. Group similar repos for batch processing - -**Example**: -``` -# Automatic grouping -Group 1: MCP servers (clean, fast) → Process first -Group 2: Libraries (moderate) → Process second -Group 3: Legacy projects (slow/risky) → Process last with extra caution -``` - -=== 2.3 Adaptive Parallelism - -**Dynamic Worker Scaling**: -- Start with 4 workers -- Monitor CPU/memory usage -- Scale up to 8 if system has headroom -- Scale down if memory pressure detected -- Per-repo time budget (kill stragglers) - -**Implementation**: -[source,v] ----- -pub fn adaptive_worker_pool(repos []string) WorkerPool { - mut workers := 4 - - for { - cpu_usage := get_cpu_usage() - mem_usage := get_memory_usage() - - if cpu_usage < 60 && mem_usage < 70 && workers < 8 { - workers++ - } else if cpu_usage > 90 || mem_usage > 85 { - workers = max(2, workers - 1) - } - - time.sleep(5 * time.second) - } -} ----- - -=== 2.4 GitHub Settings Operation - -**Purpose**: Bulk GitHub repository configuration management - -**Timeline**: 4 weeks -**Priority**: HIGH (120x time savings) -**Design**: link:docs/GITHUB-SETTINGS-OPERATION.adoc[Complete Design Document] - -==== Settings Categories - -**Repository Features**: -- `has_issues`, `has_wiki`, `has_projects`, `has_downloads` - -**Merge Settings**: -- `allow_squash_merge`, `allow_merge_commit`, `allow_rebase_merge` -- `delete_branch_on_merge`, `allow_auto_merge` - -**Security Settings**: -- `visibility` (public/private/internal) -- `dependabot_alerts`, `dependabot_security_updates` -- `secret_scanning`, `secret_scanning_push_protection` -- `private_vulnerability_reporting` - -**Branch Protection**: -- `required_pull_request_reviews` (number of approvers) -- `dismiss_stale_reviews`, `require_code_owner_reviews` -- `required_status_checks` (CI/CD checks) -- `enforce_admins`, `require_signed_commits` -- `require_linear_history`, `allow_force_pushes` - -**Topics/Labels**: -- Add/remove repository topics (tags) -- Manage issue labels - -==== Usage Examples - -[source,bash] ----- -# Security hardening across all repos -repo-batcher github-settings \ - --targets "@all-repos" \ - --set branch-protection=true \ - --set secret-scanning=true \ - --set require-signed-commits=true - -# Standardize MCP servers -repo-batcher github-settings \ - --targets "@pattern:poly-*-mcp" \ - --config mcp-server-settings.toml \ - --mode semi-auto - -# Make experimental repos private -repo-batcher github-settings \ - --targets "@topic:experimental" \ - --set visibility=private \ - --mode manual ----- - -==== Config File Format - -[source,toml] ----- -# settings.toml - Repository configuration template - -[repository] -has_issues = true -has_wiki = false -has_projects = true - -[merge] -allow_squash_merge = true -delete_branch_on_merge = true - -[security] -visibility = "public" -dependabot_alerts = true -secret_scanning = true - -[branch_protection] -branch = "main" -required_reviews = 2 -required_status_checks = ["ci", "security-scan"] -require_signed_commits = false - -[topics] -add = ["mcp-server", "formally-verified"] -remove = ["experimental"] ----- - -==== Implementation Phases - -**Week 1: Basic Settings** (50 lines ATS2 + 200 lines V) -- Repository features (issues, wiki, projects) -- Merge settings (squash, rebase, auto-delete) -- TOML config parser -- Dry-run support - -**Week 2: Security Settings** (100 lines ATS2 + 150 lines V) -- Visibility management -- Dependabot configuration -- Secret scanning settings -- Pre-flight validation - -**Week 3: Branch Protection** (150 lines ATS2 + 200 lines V) -- Basic branch protection -- Required reviews configuration -- Required status checks -- Signature requirements - -**Week 4: Advanced Features** (100 lines ATS2 + 150 lines V) -- Topics/labels management -- Rollback support -- Idempotency checks -- Integration tests - -==== Safety Features - -**Formal Verification** (ATS2): -[source,ats] ----- -(* Prove: public repos must have secret scanning *) -extern -praxi prove_public_requires_scanning: - {v: visibility} - (v == Public) - (secret_scanning == true) - -(* Prove: protected branches require valid checks *) -datatype branch_protection(checks: list(string)) where length(checks) > 0 ----- - -**Pre-flight Validation**: -- Block dangerous setting combinations -- Verify branches exist before protecting -- Validate status checks exist -- Check permissions before applying - -**Rollback Support**: -- Backup current settings before changes -- Automatic rollback on errors -- Manual rollback by operation ID - -**Idempotency**: -- Fetch current settings first -- Compute diff -- Only apply changed settings -- Skip if already correct - -==== Time Savings - -[cols="2,2,2",options="header"] -|=== -|Task |Manual (GitHub UI) |repo-batcher - -|Configure 565 repos -|~10 hours -|~5 minutes - -|Verify consistency -|~3 hours -|Automatic - -|Rollback changes -|Manual undo -|1 command - -|Audit current state -|Manual checking -|1 command -|=== - -**ROI**: 120x time savings (10 hours → 5 minutes) - -==== Integration with Existing Operations - -Complete workflow example: -[source,bash] ----- -# 1. Update workflows with SHA pinning -repo-batcher workflow-update --targets "@all-repos" - -# 2. Enable branch protection for workflows -repo-batcher github-settings \ - --targets "@all-repos" \ - --set branch-protection=true \ - --require-status-checks="workflow-linter,security-scan" - -# 3. Audit SPDX compliance -repo-batcher spdx-audit --targets "@all-repos" - -# 4. Update licenses where needed -repo-batcher license-update \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "@non-compliant" - -# 5. Sync all changes -repo-batcher git-sync \ - --parallel 8 \ - --commit-message "chore: security hardening" ----- - -==== Files to Create - -[cols="2,1,2",options="header"] -|=== -|File |Lines |Purpose - -|`src/ats2/operations/github_settings.dats` -|400 -|Core operation with proofs - -|`src/v/github/settings.v` -|300 -|GitHub API integration - -|`src/v/github/branch_protection.v` -|200 -|Branch protection logic - -|`src/v/parsers/settings_toml.v` -|150 -|TOML config parser - -|`tests/github_settings_test.v` -|200 -|Integration tests - -|**Total** -|**1,250** -|**Complete implementation** -|=== - ---- - -== Phase 3: Advanced Operations (v2.0) - -**Timeline**: 1 month -**Priority**: LOW -**Goal**: Complete operation suite - -=== 3.1 New Operations - -==== dependency-update -**Purpose**: Update dependencies across package managers - -**Supported**: -- Cargo.toml (Rust) -- package.json (Deno) -- go.mod (Go) -- Project.toml (Julia) -- opam (OCaml) - -**Features**: -- Semantic versioning awareness -- Breaking change detection -- Changelog parsing -- Test-before-update - -==== readme-standardize -**Purpose**: Ensure README files follow hyperpolymath standards - -**Checks**: -- Required sections (Description, Installation, Usage, License) -- Badge formatting -- Documentation links -- Table of contents -- Code examples - -==== security-scan -**Purpose**: Run security audits across repositories - -**Integration**: -- Hypatia neurosymbolic scanning -- TruffleHog for secrets -- cargo audit for Rust -- npm audit for JavaScript -- SPDX license compliance - -**Output**: Unified security report - -=== 3.2 Custom Operation Templates - -**Template Language** (Scheme-based): -[source,scheme] ----- -(define-operation git-branch-cleanup - ((description "Remove merged branches") - (idempotent? #t) - (dry-run-default #t) - - (steps - ((validate (git-repo? repo-path)) - (execute (git-command repo-path - "branch --merged" - "| grep -v main" - "| xargs git branch -d")) - (verify (git-branch-count-decreased?)))))) ----- - -**Safety**: -- Template validation with ATS2 -- Dry-run enforcement for first execution -- User approval required -- Rollback support - ---- - -== Phase 4: Ecosystem Integration (v2.5) - -**Timeline**: 1-2 months -**Priority**: LOW -**Goal**: Integrate with hyperpolymath ecosystem - -=== 4.1 Hypatia Integration - -**Neurosymbolic Analysis**: -- Pre-operation risk assessment -- Post-operation verification -- Anomaly detection -- Predictive failure analysis - -**Example**: -``` -repo-batcher license-update --hypatia-scan - -Hypatia Risk Assessment: - ⚠ī¸ legacy-project: High risk (3 previous failures) - ✅ repo-batcher: Low risk (100% success rate) - ℹī¸ formdb: Medium risk (slow but reliable) - -Proceed? [y/N]: _ -``` - -=== 4.2 gitbot-fleet Integration - -**Bot Orchestration**: -- rhodibot: PR creation for changes -- echidnabot: Automated testing -- sustainabot: Dependency updates -- glambot: Documentation updates - -**Workflow**: -1. repo-batcher makes changes -2. Creates PR via rhodibot -3. echidnabot runs tests -4. Auto-merge if green -5. glambot updates docs - -=== 4.3 robot-repo-automaton Integration - -**Automated Fixes**: -- repo-batcher detects issues -- robot-repo-automaton suggests fixes -- Confidence threshold for auto-apply -- Human review for low-confidence fixes - ---- - -== Crash Handling Architecture - -=== Crash Detection - -**Mechanisms**: -1. **PID file**: Check if process still running -2. **Heartbeat**: Write timestamp every 10s -3. **State file**: Atomic checkpoint writes -4. **Lock file**: Prevent concurrent execution - -**Detection Logic**: -[source,v] ----- -pub fn detect_crashed_operation() ?OperationState { - // Check for stale lockfile - if lockfile_exists() && !process_running(lockfile_pid()) { - state := read_operation_state()? - return state - } - return none -} ----- - -=== Recovery Strategies - -==== Per-Operation Recovery - -[cols="2,2,3",options="header"] -|=== -|Operation |Recovery Strategy |Justification - -|license-update -|**Resume from checkpoint** -|Changes per repo are independent - -|git-sync -|**Resume from checkpoint** -|Each repo is atomic transaction - -|file-replace -|**Resume from checkpoint** -|File replacements are independent - -|workflow-update -|**Resume from checkpoint** -|Workflow updates are independent - -|spdx-audit -|**Restart** (cheap) -|Read-only, no side effects - -|custom -|**User-defined** -|Depends on template -|=== - -==== Resume Example - -```bash -# Operation crashed halfway through -$ repo-batcher license-update --targets "@all-repos" ... -Processing 574 repositories... -[143/574] CRASH - -# Auto-resume on next invocation -$ repo-batcher license-update --targets "@all-repos" ... -Detected incomplete operation: license-update-20260206 - Completed: 143/574 repos - Last checkpoint: 2026-02-06T14:35:22Z -Resume from checkpoint? [Y/n]: y -Resuming from repo 144... -[144/574] ✓ repo-144 -[145/574] ✓ repo-145 -... -``` - -=== Checkpoint Format - -**Write-Ahead Log**: -[source,json] ----- -{ - "operation_id": "license-update-20260206-143000", - "operation_type": "license-update", - "start_time": "2026-02-06T14:30:00Z", - "last_checkpoint": "2026-02-06T14:35:22Z", - "parameters": { - "old_license": "MIT", - "new_license": "PMPL-1.0-or-later", - "backup": true, - "dry_run": false - }, - "progress": { - "total_repos": 574, - "completed_repos": 143, - "failed_repos": 2, - "repos_completed": [ - "/path/to/repo1", - "/path/to/repo2", - ... - ], - "repos_failed": [ - { - "repo": "/path/to/bad-repo", - "error": "Permission denied" - } - ] - } -} ----- - -**Location**: `~/.local/share/repo-batcher/checkpoints/{operation_id}.json` - ---- - -== Idempotency Deep Dive - -=== Classification - -==== Type 1: Naturally Idempotent (Best) - -**Operations**: workflow-update, spdx-audit, file-replace - -**Property**: `f(f(x)) = f(x)` - -**Example** (workflow-update): -``` -State 0: uses: actions/checkout@v4 -State 1: uses: actions/checkout@34e114... # v4 [First run] -State 2: uses: actions/checkout@34e114... # v4 [Second run, no change] -``` - -**Guarantee**: Running operation multiple times produces same result - -==== Type 2: Conditionally Idempotent (Good) - -**Operations**: license-update, git-sync - -**Property**: `if already_done(x) then skip else f(x)` - -**Example** (license-update): -``` -First run: MIT → PMPL-1.0-or-later (changes made) -Second run: PMPL-1.0-or-later → PMPL-1.0-or-later (skip, already correct) -``` - -**Required**: -- Pre-check current state -- Skip if already in target state -- Log "already correct" message - -==== Type 3: Accumulating (Dangerous) - -**Operations**: git-sync (current behavior) - -**Property**: `f(f(x)) ≠ f(x)` (creates additional state) - -**Problem**: -``` -First run: Creates commit C1 -Second run: Creates commit C2 (duplicate!) -``` - -**Fix Required**: -```v -pub fn git_sync_idempotent(repo string, message string) { - // Check for uncommitted changes first - if !has_uncommitted_changes(repo) { - println('${repo}: No changes to commit (skipping)') - return - } - - // Check if last commit has same message - last_commit_msg := get_last_commit_message(repo) - if last_commit_msg == message { - println('${repo}: Changes already committed') - return - } - - // Proceed with git sync - execute_git_sync(repo, message) -} ----- - -=== Idempotency Enforcement - -**ATS2 Type-Level Proof**: -[source,ats] ----- -(* Idempotency proof for workflow-update *) -extern -praxi prove_idempotent_workflow_update: - {repo: string} - {result1: string} - {result2: string} - ( - execute_workflow_update(repo) == result1, - execute_workflow_update(result1) == result2 - ) - result1 == result2 ----- - -**Runtime Enforcement**: -1. **Pre-check**: Always check current state before operation -2. **Skip-if-done**: Skip operation if already in target state -3. **Logging**: Log "skipped (already correct)" vs. "executed" -4. **Testing**: Integration test runs operation twice, asserts identical results - ---- - -== Implementation Priorities - -=== Immediate (This Week) -1. ✅ Operation expansion complete -2. âŦœ Idempotency fixes (license-update, git-sync) -3. âŦœ Basic crash recovery (checkpoint system) - -=== Short-term (2 Weeks) -4. âŦœ Failure pattern learning -5. âŦœ Pre-flight validation -6. âŦœ Comprehensive crash tests - -=== Medium-term (1 Month) -7. âŦœ Operation telemetry -8. âŦœ Smart repo prioritization -9. âŦœ Adaptive parallelism -10. âŦœ **GitHub settings operation** (bulk repo configuration - HIGH PRIORITY) - -=== Long-term (2+ Months) -11. âŦœ Additional operations (dependency-update, readme-standardize) -12. âŦœ Custom operation templates -13. âŦœ Ecosystem integration (Hypatia, gitbot-fleet) - ---- - -== Success Metrics - -=== Crash Resilience -- ✅ 100% operations resume-able after crash -- ✅ Zero data loss from crashes -- ✅ < 5s resume overhead - -=== Idempotency -- ✅ 100% of operations safe to run multiple times -- ✅ No duplicate commits from git-sync -- ✅ No unnecessary backups from repeated operations - -=== Learning -- ✅ Identify problematic repos after 3 failures -- ✅ Auto-skip known-bad repos with warning -- ✅ Suggest pre-flight checks based on history - -=== Performance -- ✅ Maintain 8x speedup over bash -- ✅ Adaptive workers improve throughput 20% -- ✅ Smart prioritization reduces failures 30% - ---- - -== Conclusion - -This roadmap transforms repo-batcher from a **batch operation tool** into a **self-healing, learning system** that: - -1. **Never loses work** (crash recovery) -2. **Learns from failures** (pattern database) -3. **Self-optimizes** (adaptive parallelism) -4. **Prevents mistakes** (idempotency guarantees) -5. **Integrates with ecosystem** (Hypatia, gitbot-fleet) - -**Next Immediate Steps**: -1. Fix git-sync idempotency (high priority) -2. Fix license-update idempotency (high priority) -3. Implement checkpoint system (crash recovery) -4. Add failure pattern learning -5. Build pre-flight validation - -With these improvements, repo-batcher becomes **production-grade infrastructure** for managing 574+ repositories with confidence. diff --git a/scaffoldia/repo-batcher/GETTING-STARTED.adoc b/scaffoldia/repo-batcher/GETTING-STARTED.adoc deleted file mode 100644 index 78140dfc..00000000 --- a/scaffoldia/repo-batcher/GETTING-STARTED.adoc +++ /dev/null @@ -1,370 +0,0 @@ -= Getting Started with repo-batcher -:toc: -// SPDX-License-Identifier: PMPL-1.0-or-later - -== Project Status - -**Status**: Alpha - Core architecture complete, operations in development - -This project transforms the bash scripts (`sync_repos.sh`, `robust_sync.sh`) into a formally verified, type-safe batch operation tool using ATS2 and V. - -== What's Been Built (2026-02-06) - -=== ✅ Complete - -* [x] Project structure from RSR template -* [x] Architecture documentation (link:docs/ARCHITECTURE.adoc[ARCHITECTURE.adoc]) -* [x] ATS2 operation type definitions with dependent types (link:src/ats2/operations/types.dats[types.dats]) -* [x] V CLI skeleton with all commands (link:src/v/main.v[main.v]) -* [x] Checkpoint files (STATE.scm, ECOSYSTEM.scm, META.scm) -* [x] Build system (Justfile) -* [x] Configuration templates -* [x] Operation templates - -=== 🚧 In Progress - -* [ ] Operation implementations (Task #5) - - License update - - File replace - - Git batch sync (port from sync_repos.sh) - -=== 📋 To Do - -* [ ] ATS2-V FFI bridge -* [ ] Watch folder monitoring -* [ ] Parallel execution engine -* [ ] Rollback system -* [ ] Integration tests - -== Quick Start - -=== 1. Install Dependencies - -[source,bash] ----- -# ATS2 (formal verification) -# Download from: http://www.ats-lang.org/Downloads.html - -# V (CLI and execution) -git clone https://github.com/vlang/v -cd v && make -sudo ./v symlink ----- - -=== 2. Build repo-batcher - -[source,bash] ----- -cd ~/Documents/hyperpolymath-repos/repo-batcher -just build-dev # Development build (fast) -# or -just build # Production build (optimized) ----- - -=== 3. Set Up Configuration - -[source,bash] ----- -just setup-config -# Edit ~/.config/repo-batcher/config.toml ----- - -=== 4. Test with Dry Run - -[source,bash] ----- -# List operations -./build/repo-batcher list-ops - -# Test license update (dry run - safe!) -./build/repo-batcher license-update \ - --old "AGPL-3.0" \ - --new "PMPL-1.0-or-later" \ - --targets "test-repo1,test-repo2" \ - --dry-run ----- - -== Architecture Overview - -[source] ----- -repo-batcher/ -├── src/ -│ ├── ats2/ # Formally verified core -│ │ ├── operations/ # Operation types with proofs -│ │ ├── validation/ # Validation logic -│ │ └── batch/ # Batch execution engine -│ ├── v/ # V CLI layer -│ │ ├── cli/ # Command-line interface -│ │ ├── watcher/ # Watch folder monitoring -│ │ └── executor/ # Parallel execution -│ ├── abi/ # Idris2 ABI (if needed for FFI) -│ └── ffi/zig/ # Zig FFI bridge (if needed) -├── templates/ # Operation and config templates -├── watch/ # Drop folder for batch operations -└── docs/ # Architecture and operations guide ----- - -=== ATS2 Core (src/ats2/) - -Provides **formally verified operations** with dependent type proofs: - -* **Type safety**: Invalid operations rejected at compile-time -* **Proof obligations**: Enforces correctness invariants -* **No runtime errors**: Proofs eliminate entire error classes - -Key types defined in `src/ats2/operations/types.dats`: - -* `license_update_op` - Proves SPDX IDs are valid -* `file_replace_op` - Proves no circular replacements -* `git_sync_op` - Proves repos are valid git repositories -* `batch_result` - Tracks success/failure with dependent types - -=== V CLI Layer (src/v/) - -Provides **fast parallel execution** with simple interface: - -* **CLI commands**: Intuitive command structure -* **Parallel execution**: V coroutines for multi-repo operations -* **Watch daemon**: Automatic operation processing -* **FFI to ATS2**: Calls formally verified core - -Available commands in `src/v/main.v`: - -* `list-ops` - Show available operations -* `license-update` - Update licenses across repos -* `file-replace` - Replace files with validation -* `git-sync` - Batch git operations (from sync_repos.sh) -* `watch` - Start watch daemon -* `rollback` - Undo operations - -== Usage Examples - -=== License Updates - -Replace all AGPL-3.0 licenses with PMPL-1.0-or-later: - -[source,bash] ----- -repo-batcher license-update \ - --old "AGPL-3.0" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" \ - --backup \ - --dry-run # Preview first! - -# Then execute for real: -repo-batcher license-update \ - --old "AGPL-3.0" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" \ - --backup ----- - -=== File Replacements - -Replace workflow files across repositories: - -[source,bash] ----- -repo-batcher file-replace \ - --pattern ".github/workflows/old-ci.yml" \ - --replacement templates/new-ci.yml \ - --targets "@rsr-repos" \ - --backup ----- - -=== Git Batch Sync (from sync_repos.sh) - -Sync all repos in parallel: - -[source,bash] ----- -# Using Justfile recipe -just git-sync 4 2 - -# Or directly: -repo-batcher git-sync \ - --parallel 4 \ - --depth 2 \ - --commit-message "chore: general update" ----- - -=== Watch Folder Mode - -Drop operation files for automatic processing: - -[source,bash] ----- -# Start watch daemon -repo-batcher watch & - -# Drop operation file -cp templates/license-update.example.toml \ - ~/.config/repo-batcher/watch/update-licenses.toml - -# Operation executes automatically -# Check logs: ~/.local/share/repo-batcher/logs/latest.log ----- - -== Development Workflow - -=== Building - -[source,bash] ----- -# Quick dev build -just build-dev - -# Production build -just build - -# Run tests -just test - -# Check code quality -just check ----- - -=== Testing Operations Safely - -**Always use `--dry-run` first!** - -[source,bash] ----- -# Preview changes (no filesystem modifications) -just dry-run license-update "repo1,repo2" - -# Check what would happen -cat ~/.local/share/repo-batcher/logs/latest.log ----- - -=== Adding New Operations - -1. Define type in `src/ats2/operations/types.dats` -2. Implement validation and execution in ATS2 -3. Add CLI command in `src/v/main.v` -4. Create template in `templates/` -5. Document in `docs/OPERATIONS.adoc` - -== Why ATS2 + V? - -=== Why ATS2? - -**Problem**: Bash scripts for mass operations are fragile. When updating 500+ repos, runtime errors after processing 327 repos are catastrophic. - -**Solution**: ATS2 dependent types prove operations are correct *before* execution. - -[source,ats] ----- -(* Proof that license is valid SPDX ID *) -datatype license_update_op(old:string, new:string) = - | LicenseUpdate(old, new) of ( - spdx_id(old), (* Compile-time proof old license is valid *) - spdx_id(new), (* Compile-time proof new license is valid *) - backup_policy (* Ensures backup if required *) - ) ----- - -**Guarantees:** - -* No invalid SPDX identifiers -* No missing backup when required -* No circular file replacements -* No git operations on non-repos - -=== Why V? - -**Problem**: Rust is complex, Go doesn't have theorem-proving, Python is too slow for 500+ repos. - -**Solution**: V provides fast compilation, simple FFI, and lightweight coroutines. - -[source,v] ----- -// Simple CLI with built-in parallelism -fn cmd_git_sync(cmd cli.Command) ! { - parallel := cmd.flags.get_int('parallel') or { 4 } - // V coroutines handle parallel execution - for repo in repos { - go process_repo(repo) - } -} ----- - -**Benefits:** - -* Fast compilation (2-3s vs. Rust's minutes) -* Simple C FFI to ATS2 -* Lightweight coroutines for parallelism -* Easy to read and maintain - -== Migration from sync_repos.sh - -The existing bash scripts are functional but fragile: - -[source,bash] ----- -# sync_repos.sh - works but no safety -find . -name ".git" | parallel -j 4 process_repo {} ----- - -repo-batcher provides the same functionality with formal guarantees: - -[source,bash] ----- -# Formally verified, type-safe, provably correct -repo-batcher git-sync --parallel 4 --depth 2 ----- - -**Improvements:** - -* Compile-time validation (no runtime surprises) -* Automatic backups with rollback -* Detailed logging and error tracking -* Dry-run mode for safe testing -* Type-safe parallel execution - -== Next Steps - -1. **Complete Task #5**: Implement basic operations - - License update implementation - - File replace implementation - - Git sync implementation (port sync_repos.sh logic) - -2. **ATS2-V FFI**: Bridge between ATS2 core and V CLI - - C function exports from ATS2 - - V C interop layer - - Type marshalling - -3. **Watch System**: Monitor folder for operation files - - TOML parser - - File watcher - - Operation queue - -4. **Parallel Executor**: V coroutines for multi-repo processing - - Progress reporting - - Error isolation - - Resource management - -5. **Integration Testing**: Test against real repositories - - Dry-run validation - - Backup and rollback - - Parallel execution - -== Getting Help - -* **Architecture**: link:docs/ARCHITECTURE.adoc[ARCHITECTURE.adoc] -* **Current State**: link:STATE.scm[STATE.scm] -* **Design Rationale**: link:META.scm[META.scm] -* **Ecosystem Position**: link:ECOSYSTEM.scm[ECOSYSTEM.scm] - -== Contributing - -This project is in active development. Key areas needing help: - -* ATS2 operation implementations with proofs -* V FFI optimization -* Operation templates -* Documentation - -See link:CONTRIBUTING.md[CONTRIBUTING.md] for details. diff --git a/scaffoldia/repo-batcher/IMPLEMENTATION-STATUS.md b/scaffoldia/repo-batcher/IMPLEMENTATION-STATUS.md deleted file mode 100644 index b64a79d3..00000000 --- a/scaffoldia/repo-batcher/IMPLEMENTATION-STATUS.md +++ /dev/null @@ -1,430 +0,0 @@ -# repo-batcher Implementation Status - -**Date**: 2026-02-06 -**Version**: 0.1.0 Alpha -**Status**: Core operations implemented, ready for testing - ---- - -## ✅ Completed - -### Core Architecture (100%) - -- [x] Project structure from RSR template -- [x] ATS2 operation type definitions with dependent types -- [x] V CLI with full command structure -- [x] ATS2-V FFI bridge via C exports -- [x] Build system (Justfile) -- [x] Configuration system -- [x] Documentation (Architecture, Operations Guide, Getting Started) - -### Operations (60%) - -#### License Update (90%) - -**Status**: ✅ Core implementation complete, needs testing - -**Files**: -- `src/ats2/operations/license_update.dats` - Main implementation -- `src/ats2/validation/spdx.dats` - SPDX validation -- `src/v/main.v` - CLI integration -- `src/v/ffi/ats2_bridge.v` - FFI wrapper - -**Features**: -- ✅ SPDX identifier validation -- ✅ LICENSE file replacement -- ✅ SPDX header updates in source files -- ✅ Backup creation -- ✅ Dry-run mode -- ✅ Batch processing across repositories -- ⚠ī¸ String manipulation helpers (placeholders) -- ⚠ī¸ Directory traversal (needs completion) - -**Usage**: -```bash -repo-batcher license-update \ - --old "AGPL-3.0" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" \ - --backup \ - --dry-run -``` - -#### Git Batch Sync (95%) - -**Status**: ✅ Complete, ported from sync_repos.sh - -**Files**: -- `src/ats2/operations/git_sync.dats` - Main implementation -- `src/v/main.v` - CLI integration -- `src/v/ffi/ats2_bridge.v` - FFI wrapper - -**Features**: -- ✅ Repository discovery (`find . -maxdepth 2 -name ".git"`) -- ✅ Git add, commit, push workflow -- ✅ Parallel job support (sequential in ATS2, parallel in V) -- ✅ Success/failure tracking -- ✅ Summary report (matches sync_repos.sh output) -- ✅ Failure details report -- ✅ Dry-run mode -- ⚠ī¸ Actual parallel execution (V coroutines not yet implemented) - -**Usage**: -```bash -repo-batcher git-sync \ - --parallel 4 \ - --depth 2 \ - --commit-message "chore: batch update" -``` - -#### File Replace (40%) - -**Status**: 🚧 Skeleton implemented, needs core logic - -**Files**: -- `src/ats2/ffi/c_exports.dats` - FFI export (placeholder) -- `src/v/main.v` - CLI integration (complete) -- `src/v/ffi/ats2_bridge.v` - FFI wrapper (complete) - -**Features**: -- ✅ CLI command structure -- ✅ Parameter validation -- ✅ FFI bridge setup -- ❌ Core file replacement logic -- ❌ Pattern matching -- ❌ Circular replacement detection - -**Next Steps**: -1. Implement core logic in `src/ats2/operations/file_replace.dats` -2. Add glob pattern matching -3. Test on real repositories - -### FFI Bridge (70%) - -**Status**: ✅ Core bridge complete, needs helper functions - -**Files**: -- `src/ats2/ffi/c_exports.dats` - C function exports -- `src/v/ffi/ats2_bridge.v` - V FFI wrapper - -**Exported Functions**: -- ✅ `c_validate_spdx()` - SPDX validation -- ✅ `c_license_update()` - License update operation -- ✅ `c_git_sync()` - Git sync operation -- ✅ `c_file_replace()` - File replace (placeholder) -- ✅ `c_get_version()` - Version string - -**C-V Type Marshalling**: -- ✅ `CBatchResult` struct -- ✅ String conversion helpers -- ✅ Bool/int flag conversion - -### CLI (80%) - -**Status**: ✅ All commands implemented, watch mode pending - -**Commands**: -- ✅ `list-ops` - List operations -- ✅ `license-update` - License update with ATS2 validation -- ✅ `file-replace` - File replace with validation -- ✅ `git-sync` - Git batch sync with parallel support -- ⚠ī¸ `watch` - Watch daemon (skeleton only) -- ⚠ī¸ `rollback` - Rollback (skeleton only) - -**CLI Features**: -- ✅ Flag parsing -- ✅ Help text -- ✅ Error handling -- ✅ Dry-run mode -- ✅ Target resolution (@all-repos, @pattern) -- ✅ SPDX validation before execution -- ✅ Formatted output - -### Documentation (100%) - -**Status**: ✅ Comprehensive documentation complete - -**Files**: -- ✅ `README.adoc` - Project overview -- ✅ `GETTING-STARTED.adoc` - Setup and usage guide -- ✅ `docs/ARCHITECTURE.adoc` - Detailed architecture -- ✅ `docs/OPERATIONS.adoc` - Complete operations guide -- ✅ `STATE.scm` - Project state tracking -- ✅ `ECOSYSTEM.scm` - Ecosystem position -- ✅ `META.scm` - Design decisions - ---- - -## 🚧 In Progress - -### String Manipulation Helpers (ATS2) - -**Status**: Placeholders in place, need implementation - -**Needed Functions**: -- `string_contains()` - Substring search -- `string_index_of()` - Find substring position -- `string_rindex_of()` - Find last occurrence -- `string_replace()` - Replace all occurrences -- `string_trim()` - Trim whitespace -- `string_substring()` - Extract substring -- `string_suffix()` - Get suffix from position - -**Location**: Each operation file has placeholders - -### Directory Traversal (ATS2) - -**Status**: Needs implementation - -**Required**: -- Recursive directory scanning -- File pattern matching -- Source file discovery - -**Current Approach**: Using shell `find` command as workaround - ---- - -## 📋 Not Started - -### Watch Folder System (0%) - -**Planned Features**: -- Monitor `watch/` folder for operation files -- Parse TOML operation definitions -- Execute operations automatically -- Result logging - -**Files to Create**: -- `src/v/watcher/monitor.v` - Folder monitoring -- `src/v/watcher/parser.v` - TOML parser -- `src/v/watcher/queue.v` - Operation queue - -### Rollback System (0%) - -**Planned Features**: -- Backup tracking -- Rollback from backup -- Log-based recovery -- Atomic undo - -**Files to Create**: -- `src/ats2/rollback/backup.dats` - Backup management -- `src/ats2/rollback/restore.dats` - Restore operations - -### Parallel Execution (V Coroutines) (0%) - -**Current**: Sequential processing in ATS2 -**Goal**: Parallel processing with V coroutines - -**Files to Create**: -- `src/v/executor/parallel.v` - V coroutine implementation -- `src/v/executor/pool.v` - Worker pool - ---- - -## đŸŽ¯ Testing Status - -### Unit Tests (0%) - -- ❌ SPDX validation tests -- ❌ Operation type tests -- ❌ FFI marshalling tests - -### Integration Tests (0%) - -- ❌ License update on test repos -- ❌ Git sync on test repos -- ❌ File replace on test repos - -### Performance Tests (0%) - -- ❌ Parallel scaling benchmarks -- ❌ Comparison to sync_repos.sh - ---- - -## 📊 Completion Summary - -| Component | Progress | Status | -|-----------|----------|--------| -| **Architecture** | 100% | ✅ Complete | -| **ATS2 Core** | 60% | 🚧 Core done, helpers needed | -| **V CLI** | 80% | ✅ Main commands done | -| **FFI Bridge** | 70% | ✅ Core bridge done | -| **Operations** | 60% | 🚧 2/3 operations working | -| **Documentation** | 100% | ✅ Complete | -| **Tests** | 0% | ❌ Not started | -| **Watch System** | 0% | ❌ Not started | -| **Rollback** | 0% | ❌ Not started | -| **Overall** | **40%** | 🚧 **Alpha** | - ---- - -## 🚀 Next Steps - -### Immediate (Today/Tomorrow) - -1. **Complete string helpers in ATS2** - - Implement all placeholder functions - - Test with actual strings - -2. **Test license-update on real repo** - - Create test repository - - Run dry-run - - Execute and verify - -3. **Test git-sync on real repos** - - Run on small set of repos - - Verify output matches sync_repos.sh - - Check summary report - -### This Week - -1. **Implement V coroutines for parallel execution** - - Port sequential logic to parallel - - Benchmark against sync_repos.sh - - Tune worker pool size - -2. **Complete file-replace operation** - - Implement core logic - - Add pattern matching - - Test on workflow files - -3. **Add integration tests** - - Set up test repository fixtures - - Test all operations - - Verify safety guarantees - -### This Month - -1. **Watch folder system** - - TOML parser - - Folder monitoring - - Automatic execution - -2. **Rollback system** - - Backup tracking - - Restore operations - - Log-based recovery - -3. **Performance optimization** - - Profile hot paths - - Optimize file I/O - - Tune parallel execution - ---- - -## 🔧 Build Instructions - -### Prerequisites - -```bash -# ATS2 -# Download from: http://www.ats-lang.org/Downloads.html - -# V -git clone https://github.com/vlang/v -cd v && make -sudo ./v symlink -``` - -### Building - -```bash -cd ~/Documents/hyperpolymath-repos/repo-batcher - -# Quick dev build (recommended for testing) -just build-dev - -# Production build (optimized) -just build - -# Build output: build/repo-batcher -``` - -### Testing - -```bash -# List operations (no ATS2 required) -./build/repo-batcher list-ops - -# Dry-run test (safe, no changes) -./build/repo-batcher license-update \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "test-repo" \ - --dry-run -``` - ---- - -## 🐛 Known Issues - -1. **ATS2 string helpers are placeholders** - - Impact: Operations may not work fully - - Workaround: Use shell commands via FFI - - Fix: Implement proper string manipulation - -2. **Directory traversal incomplete** - - Impact: May miss some source files - - Workaround: Using `find` command - - Fix: Implement proper directory scanning - -3. **Parallel execution not implemented** - - Impact: Sequential processing only - - Workaround: Works, just slower - - Fix: Implement V coroutines - -4. **No tests** - - Impact: Unknown edge cases - - Workaround: Manual testing with --dry-run - - Fix: Add comprehensive test suite - ---- - -## 📝 Notes - -### Why ATS2 String Helpers Are Placeholders - -ATS2 has a learning curve for string manipulation. Rather than block progress, I: -1. Defined the function signatures -2. Created placeholders that return safe defaults -3. Implemented core logic structure -4. Operations can be completed incrementally - -### Next Session Priorities - -1. **Complete string helpers** - Critical path item -2. **Test on real repos** - Validate approach -3. **Fix any issues found** - Based on testing -4. **Implement parallel execution** - Performance goal - -### Comparison to sync_repos.sh - -**Original (Bash)**: -```bash -find . -maxdepth 2 -name ".git" -type d | \ -parallel -j 4 process_repo {} -``` - -**repo-batcher (ATS2 + V)**: -```bash -repo-batcher git-sync --parallel 4 --depth 2 -``` - -**Benefits**: -- Type-safe (ATS2 proofs) -- Better error handling -- Detailed reporting -- Dry-run support -- Rollback capability -- Extensible to other operations - ---- - -## 📄 License - -SPDX-License-Identifier: PMPL-1.0-or-later - -All code is formally verified or in the process of formal verification using ATS2 dependent types. diff --git a/scaffoldia/repo-batcher/Justfile b/scaffoldia/repo-batcher/Justfile deleted file mode 100644 index 3ad34bef..00000000 --- a/scaffoldia/repo-batcher/Justfile +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-License-Identifier: PMPL-1.0-or-later -# justfile for repo-batcher - -# Default recipe -default: - @just --list - -# Build the project -build: - @echo "Building repo-batcher..." - # Compile ATS2 core - cd src/ats2 && patscc -o ../../build/librepobatcher.a operations/*.dats - # Compile V CLI - v -prod -o build/repo-batcher src/v/main.v - @echo "Build complete: build/repo-batcher" - -# Build in development mode (faster, less optimized) -build-dev: - @echo "Building repo-batcher (dev mode)..." - v -o build/repo-batcher src/v/main.v - @echo "Build complete: build/repo-batcher" - -# Run smoke test (structure validation, no build required) -test-smoke: - @echo "Running smoke test..." - ./tests/smoke_test.sh - -# Run tests -test: - @echo "Running tests..." - # V integration tests - v run tests/integration_test.v - @echo "" - @echo "All tests passed!" - -# Run real repository tests (dry-run only) -test-real: - @echo "Running real repository tests (dry-run)..." - ./tests/real_repo_test.sh - -# Run performance benchmark -benchmark: - @echo "Running performance benchmark..." - ./benchmark/performance_test.sh - -# Clean build artifacts -clean: - rm -rf build/ - find . -name "*.o" -delete - find . -name "*.a" -delete - find . -name "*.so" -delete - -# Install to system -install: - @echo "Installing repo-batcher..." - sudo cp build/repo-batcher /usr/local/bin/ - sudo chmod +x /usr/local/bin/repo-batcher - mkdir -p ~/.config/repo-batcher/watch - mkdir -p ~/.local/share/repo-batcher/logs - mkdir -p ~/.local/share/repo-batcher/backups - @echo "Installed to /usr/local/bin/repo-batcher" - -# Uninstall from system -uninstall: - sudo rm -f /usr/local/bin/repo-batcher - @echo "Uninstalled repo-batcher" - -# Format code -fmt: - # V has built-in formatter - v fmt -w src/v/ - -# Check code quality -check: - @echo "Checking ATS2 code..." - patsopt -tc -d src/ats2/operations/*.dats - @echo "Checking V code..." - v vet src/v/ - -# Create example config -setup-config: - mkdir -p ~/.config/repo-batcher - cp templates/config.example.toml ~/.config/repo-batcher/config.toml - @echo "Config created at ~/.config/repo-batcher/config.toml" - -# Run in dry-run mode (safe testing) -dry-run operation targets: - ./build/repo-batcher {{operation}} --targets {{targets}} --dry-run - -# Quick git-sync (ported from sync_repos.sh) -git-sync parallel="4" depth="2": - ./build/repo-batcher git-sync --parallel {{parallel}} --depth {{depth}} - -# Watch mode (daemon) -watch: - ./build/repo-batcher watch - -# Show version and help -help: - ./build/repo-batcher --help diff --git a/scaffoldia/repo-batcher/LICENSE b/scaffoldia/repo-batcher/LICENSE deleted file mode 100644 index d2dccf5b..00000000 --- a/scaffoldia/repo-batcher/LICENSE +++ /dev/null @@ -1,153 +0,0 @@ -SPDX-License-Identifier: PMPL-1.0-or-later -SPDX-FileCopyrightText: 2024-2025 Palimpsest Stewardship Council - -================================================================================ -PALIMPSEST-MPL LICENSE VERSION 1.0 -================================================================================ - -File-level copyleft with ethical use and quantum-safe provenance - -Based on Mozilla Public License 2.0 - --------------------------------------------------------------------------------- -PREAMBLE --------------------------------------------------------------------------------- - -This License extends the Mozilla Public License 2.0 (MPL-2.0) with provisions -for ethical use, post-quantum cryptographic provenance, and emotional lineage -protection. The base MPL-2.0 terms apply except where explicitly modified by -the Exhibits below. - -Like a palimpsest manuscript where each layer builds upon what came before, -this license recognizes that creative works carry history, context, and meaning -that transcend mere code or text. - --------------------------------------------------------------------------------- -SECTION 1: BASE LICENSE --------------------------------------------------------------------------------- - -This License incorporates the full text of Mozilla Public License 2.0 by -reference. The complete MPL-2.0 text is available at: -https://www.mozilla.org/en-US/MPL/2.0/ - -All terms, conditions, and definitions from MPL-2.0 apply except where -explicitly modified by the Exhibits in this License. - --------------------------------------------------------------------------------- -SECTION 2: ADDITIONAL DEFINITIONS --------------------------------------------------------------------------------- - -2.1. "Emotional Lineage" - means the narrative, cultural, symbolic, and contextual meaning embedded - in Covered Software, including but not limited to: protest traditions, - cultural heritage, trauma narratives, and community stories. - -2.2. "Provenance Metadata" - means cryptographically signed attribution information attached to or - associated with Covered Software, including author identities, timestamps, - modification history, and lineage references. - -2.3. "Non-Interpretive System" - means any automated system that processes Covered Software without - preserving or considering its Emotional Lineage, including but not - limited to: AI training pipelines, content aggregators, and automated - summarization tools. - -2.4. "Quantum-Safe Signature" - means a cryptographic signature using algorithms resistant to attacks - by quantum computers, as specified in Exhibit B. - --------------------------------------------------------------------------------- -SECTION 3: ETHICAL USE REQUIREMENTS --------------------------------------------------------------------------------- - -In addition to the rights and obligations under MPL-2.0: - -3.1. Emotional Lineage Preservation - You must make reasonable efforts to preserve and communicate the - Emotional Lineage of Covered Software when distributing or creating - derivative works. This includes maintaining narrative context, cultural - attributions, and symbolic meaning where documented. - -3.2. Non-Interpretive System Notice - If You use Covered Software as input to a Non-Interpretive System, You - must: - (a) document such use in a publicly accessible manner; and - (b) not claim that outputs of such systems carry the Emotional Lineage - of the original work without explicit permission from Contributors. - -3.3. Ethical Use Declaration - Commercial use of Covered Software requires acknowledgment that You have - read and understood Exhibit A (Ethical Use Guidelines) and agree to act - in good faith accordance with its principles. - -See Exhibit A for complete Ethical Use Guidelines. - --------------------------------------------------------------------------------- -SECTION 4: PROVENANCE REQUIREMENTS --------------------------------------------------------------------------------- - -4.1. Metadata Preservation - You must not strip, alter, or obscure Provenance Metadata from Covered - Software except where technically necessary and with clear documentation - of any changes. - -4.2. Quantum-Safe Provenance (Optional) - Contributors may sign their Contributions using Quantum-Safe Signatures. - If Quantum-Safe Signatures are present, You must preserve them in all - distributions. - -4.3. Lineage Chain - When creating derivative works, You should extend the provenance chain - to include Your own contributions, maintaining cryptographic linkage to - prior Contributors where feasible. - -See Exhibit B for Quantum-Safe Provenance specifications. - --------------------------------------------------------------------------------- -SECTION 5: GOVERNANCE --------------------------------------------------------------------------------- - -5.1. Stewardship Council - This License is maintained by the Palimpsest Stewardship Council, which - may issue clarifications, interpretive guidance, and future versions. - -5.2. Version Selection - You may use Covered Software under this version of the License or any - later version published by the Palimpsest Stewardship Council. - -5.3. Dispute Resolution - Disputes regarding interpretation of Ethical Use Requirements (Section 3) - should first be submitted to the Palimpsest Stewardship Council for - non-binding guidance before pursuing legal remedies. - --------------------------------------------------------------------------------- -SECTION 6: COMPATIBILITY --------------------------------------------------------------------------------- - -6.1. MPL-2.0 Compatibility - Covered Software under this License may be combined with software under - MPL-2.0. The combined work must comply with both licenses. - -6.2. Secondary Licenses - The Secondary License provisions of MPL-2.0 Section 3.3 apply to this - License. - --------------------------------------------------------------------------------- -EXHIBITS --------------------------------------------------------------------------------- - -Exhibit A - Ethical Use Guidelines -Exhibit B - Quantum-Safe Provenance Specification - -See separate files: -- EXHIBIT-A-ETHICAL-USE.txt -- EXHIBIT-B-QUANTUM-SAFE.txt - --------------------------------------------------------------------------------- -END OF PALIMPSEST-MPL LICENSE VERSION 1.0 --------------------------------------------------------------------------------- - -For questions about this License: -- Repository: https://github.com/hyperpolymath/palimpsest-license -- Council: contact via repository Issues diff --git a/scaffoldia/repo-batcher/OPERATION-RULES-AND-MODES.adoc b/scaffoldia/repo-batcher/OPERATION-RULES-AND-MODES.adoc deleted file mode 100644 index 4293223c..00000000 --- a/scaffoldia/repo-batcher/OPERATION-RULES-AND-MODES.adoc +++ /dev/null @@ -1,705 +0,0 @@ -= Operation Rules, Safety Constraints & Automation Modes -:toc: -:toclevels: 4 -:version: 0.9.0 -:date: 2026-02-06 - -== Overview - -This document defines: -1. **What NOT to touch** (exclusion rules) -2. **Conflict resolution strategies** -3. **Automation modes** (manual / semi-auto / fully-auto) -4. **Safety constraints per operation** - ---- - -== Part 1: Exclusion Rules (What NOT to Touch) - -=== Global Exclusion Patterns - -These paths are **NEVER** modified by any operation: - -==== 1. Hidden State Directories -``` -.git/ # Git internals (NEVER touch) -.gitignore # User-defined ignore patterns -node_modules/ # Dependencies (managed by package managers) -target/ # Rust build artifacts -dist/ # Build outputs -build/ # Build outputs -vendor/ # Vendored dependencies -.venv/ # Python virtual environments -venv/ # Python virtual environments -``` - -==== 2. User-Specific Configurations -``` -.env # Environment secrets (CRITICAL) -.env.local # Local environment overrides -*.pem # SSL certificates -*.key # Private keys -*credentials* # Any file with "credentials" in name -*secret* # Any file with "secret" in name -.ssh/ # SSH keys -``` - -==== 3. Build Artifacts & Caches -``` -*.o # Object files -*.so # Shared libraries -*.dylib # Dynamic libraries -*.a # Static libraries -*.beam # BEAM bytecode -*.pyc # Python bytecode -*.class # Java bytecode -.cache/ # General caches -``` - -==== 4. IDE & Editor Configurations -``` -.vscode/ # VS Code settings -.idea/ # IntelliJ settings -*.swp # Vim swap files -*.swo # Vim swap files -*~ # Editor backup files -``` - -==== 5. OS-Specific Files -``` -.DS_Store # macOS folder attributes -Thumbs.db # Windows thumbnails -desktop.ini # Windows folder config -``` - -=== Per-Operation Exclusion Rules - -==== license-update -**Never Touch**: -- Files without existing license headers (don't add new headers) -- Binary files (executables, images, PDFs) -- Third-party code in `vendor/`, `third_party/` -- Generated code files (e.g., `*.pb.go`, `*_generated.rs`) - -**Rationale**: Preserve third-party licenses, avoid corrupting binaries - -==== file-replace -**Never Replace**: -- Files larger than 10MB (likely binaries or data) -- Files with no write permission -- Symlinks (security risk - could escape repo) -- Files in `.git/` directory - -**Rationale**: Prevent accidental damage to large files or system corruption - -==== workflow-update -**Only Touch**: -- `.github/workflows/*.yml` -- `.github/workflows/*.yaml` - -**Never Touch**: -- Workflows in other locations -- Non-workflow YAML files -- Third-party action definitions (actions/*.yml inside action repos) - -**Rationale**: Strict scope to prevent unintended changes - -==== git-sync -**Never Commit**: -- Files matching `.gitignore` patterns -- Untracked binary files > 1MB -- Files with merge conflict markers (`<<<<<<<`, `=======`, `>>>>>>>`) -- Submodule directories (require special handling) - -**Rationale**: Avoid polluting repos with ignored/large files - -==== spdx-audit -**Read-Only**: -- All operations are read-only -- No exclusions needed (safe to scan everything) - ---- - -== Part 2: Conflict Resolution Strategies - -=== Git Conflicts (git-sync) - -==== Detection -```v -pub fn has_merge_conflicts(repo string) bool { - result := os.execute('git -C ${repo} status --porcelain') - return result.output.contains('UU ') || // Both modified - result.output.contains('AA ') || // Both added - result.output.contains('DD ') // Both deleted -} -``` - -==== Resolution Strategy - -[cols="2,2,3",options="header"] -|=== -|Conflict Type |Auto-Resolve? |Strategy - -|Clean repo -|✅ YES -|Proceed with commit - -|Uncommitted changes -|✅ YES -|Commit changes as-is - -|Merge conflicts -|❌ NO -|**Abort** and report conflict - -|Untracked large files -|⚠ī¸ WARN -|Skip file, warn user - -|Detached HEAD -|❌ NO -|**Abort** and report state -|=== - -**Manual Mode**: Report conflict, skip repo -**Semi-Auto Mode**: Attempt simple resolution (e.g., keep ours/theirs) -**Fully-Auto Mode**: Abort operation, alert user - -=== License Conflicts (license-update) - -==== Scenarios - -**1. Mixed Licenses in Repo** -``` -Problem: Some files have MIT, some have Apache-2.0 -Solution: Report conflict, require manual resolution -``` - -**2. Third-Party Code** -``` -Problem: Trying to change license in vendor/ -Solution: Skip vendor/ directories automatically -``` - -**3. Existing PMPL License** -``` -Problem: Trying to update PMPL → PMPL -Solution: Skip (idempotent), log "already correct" -``` - -=== File Replacement Conflicts (file-replace) - -==== Circular Replacement Detection -```v -pub fn is_circular_replacement(source string, dest string) bool { - source_hash := compute_hash(read_file(source)) - dest_hash := compute_hash(read_file(dest)) - return source_hash == dest_hash -} -``` - -**Action**: Skip circular replacement, warn user - -==== Permission Conflicts -``` -Problem: File exists but is read-only -Solution: Skip file, report permission error -``` - -==== Backup Conflicts -``` -Problem: Backup file already exists (.backup) -Solution: Generate timestamped backup (.backup.20260206143522) -``` - -=== Workflow Conflicts (workflow-update) - -==== Already Pinned Workflows -```yaml -uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # Already pinned -``` -**Action**: Skip (idempotent), log "already SHA-pinned" - -==== Unknown Actions -```yaml -uses: my-org/custom-action@v1 # Not in SHA database -``` -**Action**: Skip this action, log warning, continue with known actions - -==== Invalid YAML -``` -Problem: Workflow has syntax errors -Solution: Skip file, report YAML parse error -``` - ---- - -== Part 3: Automation Modes - -=== Mode Definitions - -==== 1. Manual Mode (Default) - -**Characteristics**: -- User approves EVERY action -- Full visibility into changes -- Safest mode for production - -**Workflow**: -``` -1. Tool proposes action: "Update LICENSE in repo-batcher?" -2. User reviews diff -3. User approves: [y/N] -4. Tool executes -5. Repeat for each repo -``` - -**Use Cases**: -- Critical production repositories -- Legal/compliance-sensitive changes -- Learning tool behavior -- First-time operations - -**Example**: -```bash -repo-batcher license-update \ - --mode manual \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "@critical-repos" - -Proposal: repo-batcher (1/15) - Old license: MIT - New license: PMPL-1.0-or-later - Files affected: LICENSE, 12 source files - - Diff preview: - -// SPDX-License-Identifier: MIT - +// SPDX-License-Identifier: PMPL-1.0-or-later - -Approve? [y/N/d=diff/s=skip-repo/q=quit]: _ -``` - -==== 2. Semi-Auto Mode (Recommended) - -**Characteristics**: -- Bulk approval with exception handling -- User approves operation once, tool executes all -- Pauses on conflicts/errors for user decision -- Balance of speed and safety - -**Workflow**: -``` -1. User approves operation for all repos -2. Tool executes automatically -3. On error/conflict: pause and ask user -4. User resolves or skips -5. Continue automatically -``` - -**Use Cases**: -- Standard maintenance across many repos -- Operations with high success rate -- Known-good repositories -- Regular batch updates - -**Example**: -```bash -repo-batcher license-update \ - --mode semi-auto \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" - -Pre-flight checks: - ✓ 570 repos accessible - ✓ Write permissions verified - ⚠ 4 repos have mixed licenses (will prompt) - -Approve operation for all 570 repos? [y/N]: y - -Executing... -[1/570] ✓ repo-batcher -[2/570] ✓ lithoglyph -[3/570] ✓ gitvisor -[4/570] ⚠ legacy-project - CONFLICT DETECTED - -Conflict: legacy-project has mixed licenses (MIT + Apache-2.0) -Options: - [s] Skip this repo - [m] Switch to manual mode for this repo - [a] Abort entire operation - [c] Continue, skip all conflicts - -Choice: _ -``` - -==== 3. Fully-Auto Mode (Advanced) - -**Characteristics**: -- Zero user interaction (except critical errors) -- Executes all operations without pausing -- Logs all actions for review -- Fastest mode, highest risk - -**Workflow**: -``` -1. User starts operation with full-auto flag -2. Tool executes across all repos -3. Conflicts are auto-resolved using default strategy -4. Generates comprehensive log -5. Email/notify user when complete -``` - -**Use Cases**: -- Cron jobs / scheduled operations -- CI/CD integration -- Known-safe operations (read-only audits) -- Mature, well-tested automation - -**Safety Constraints**: -- Requires `--confirm-auto` flag (explicit opt-in) -- Dry-run MUST succeed first -- Requires backup enabled -- Only for idempotent operations (recommended) - -**Example**: -```bash -# Dry-run required first -repo-batcher workflow-update \ - --mode fully-auto \ - --targets "@all-repos" \ - --dry-run -# Verify output... - -# Execute for real -repo-batcher workflow-update \ - --mode fully-auto \ - --targets "@all-repos" \ - --confirm-auto \ - --backup - -[FULLY-AUTO MODE] -Warning: This will execute across 574 repos without interaction. -Type 'I UNDERSTAND' to proceed: _ -``` - -=== Mode Comparison - -[cols="2,1,1,1,3",options="header"] -|=== -|Feature |Manual |Semi-Auto |Fully-Auto |Notes - -|User interaction -|Every repo -|On conflicts -|Never -| - -|Speed -|Slowest -|Medium -|Fastest -| - -|Safety -|Highest -|High -|Medium -|Requires explicit opt-in - -|Error handling -|User decides -|User decides -|Auto-resolve -|Fully-auto uses default strategies - -|Suitable for -|Production -|Maintenance -|Automation -| - -|Dry-run required -|No -|Recommended -|**Yes** -|Fully-auto fails without successful dry-run - -|Backup required -|Optional -|Recommended -|**Yes** -|Fully-auto enforces backups - -|Rollback -|Manual -|Manual -|Auto-rollback -|Fully-auto auto-rolls-back on critical failures -|=== - -=== Mode Selection Logic - -**Decision Tree**: -``` -Are you changing critical repos (production, infrastructure)? - → YES: Use **Manual** mode - -Is this a well-tested operation (workflow-update, spdx-audit)? - → YES: Use **Semi-Auto** or **Fully-Auto** mode - -Is this a destructive operation (git-sync, file-replace)? - → YES: Use **Manual** or **Semi-Auto** mode - → NO: Safe for **Fully-Auto** mode - -Do you need to run unattended (cron job)? - → YES: Use **Fully-Auto** mode (with dry-run first) - -First time running this operation? - → YES: Use **Manual** mode (learn behavior) - → NO: Use **Semi-Auto** or **Fully-Auto** mode -``` - ---- - -== Part 4: Safety Constraints Per Operation - -=== license-update - -**Hard Constraints**: -- ✅ Must validate SPDX identifiers (both old and new) -- ✅ Must create backups (unless explicitly disabled) -- ✅ Must skip third-party code directories -- ❌ Cannot modify files without existing license headers - -**Soft Constraints**: -- ⚠ī¸ Warn if changing to non-OSI approved license -- ⚠ī¸ Warn if mixed licenses detected -- ⚠ī¸ Suggest manual review if > 100 files affected per repo - -**Auto-Resolve Rules**: -- If file already has target license → skip (idempotent) -- If file is in vendor/ → skip -- If file is binary → skip -- If permission denied → skip, log error - -=== git-sync - -**Hard Constraints**: -- ✅ Must check for uncommitted changes before proceeding -- ✅ Must check for merge conflicts (abort if found) -- ✅ Must verify remote exists and is reachable -- ❌ Cannot commit if detached HEAD -- ❌ Cannot commit if .gitignore violates - -**Soft Constraints**: -- ⚠ī¸ Warn if committing large files (> 1MB) -- ⚠ī¸ Warn if no changes to commit (skip) -- ⚠ī¸ Warn if pushing to main without protection - -**Auto-Resolve Rules**: -- No changes → skip (idempotent) -- Remote unreachable → skip, log error -- Merge conflict → abort, report conflict - -=== file-replace - -**Hard Constraints**: -- ✅ Must validate replacement file exists -- ✅ Must create backups before replacing -- ✅ Must detect circular replacements (FNV-1a hash) -- ❌ Cannot replace symlinks (security risk) -- ❌ Cannot replace files > 10MB (likely binaries) - -**Soft Constraints**: -- ⚠ī¸ Warn if replacing file with different extension -- ⚠ī¸ Warn if no files match pattern -- ⚠ī¸ Warn if > 50 files affected per repo - -**Auto-Resolve Rules**: -- Circular replacement detected → skip -- Permission denied → skip, log error -- File already matches replacement → skip (idempotent) - -=== workflow-update - -**Hard Constraints**: -- ✅ Must only modify `.github/workflows/*.y{a}ml` -- ✅ Must preserve YAML validity -- ✅ Must use known SHA pins from database -- ❌ Cannot modify non-workflow YAML files -- ❌ Cannot pin unknown actions (not in database) - -**Soft Constraints**: -- ⚠ī¸ Warn if workflow already SHA-pinned -- ⚠ī¸ Warn if unknown action encountered (skip that action) -- ⚠ī¸ Suggest updating SHA database if many unknowns - -**Auto-Resolve Rules**: -- Already SHA-pinned → skip (idempotent) -- Unknown action → skip this action, continue with known -- Invalid YAML → skip file, log error - -=== spdx-audit - -**Hard Constraints**: -- ✅ Read-only (NO modifications) -- ✅ Must scan all source file types -- ❌ No constraints (safe operation) - -**Soft Constraints**: -- None (read-only operation) - -**Auto-Resolve Rules**: -- Permission denied → skip file, log error -- Binary file → skip -- Unreadable file → skip, log error - ---- - -== Part 5: Configuration Files - -=== Repo-Specific Exclusions - -Create `.repo-batcher.toml` in repository root: - -```toml -# .repo-batcher.toml - Repo-specific rules - -[exclusions] -# Global exclusions (apply to all operations) -patterns = [ - "vendor/**", - "third_party/**", - "legacy_code/**", - "generated/**" -] - -# Per-operation exclusions -[exclusions.license-update] -patterns = [ - "src/vendored_lib/**" # Keep original licenses -] - -[exclusions.file-replace] -patterns = [ - "config/production.yml" # Never replace production config -] - -[automation] -# Default automation mode for this repo -mode = "manual" # or "semi-auto", "fully-auto" - -# Operations that require manual approval -manual_only = ["git-sync", "license-update"] - -[safety] -# Require dry-run before real execution -require_dry_run = true - -# Maximum files affected before requiring manual approval -max_files_before_prompt = 50 -``` - -=== Global Configuration - -Create `~/.config/repo-batcher/config.toml`: - -```toml -# Global repo-batcher configuration - -[defaults] -automation_mode = "semi-auto" -backup = true -dry_run_first = true - -[exclusions] -# Global patterns (apply to all repos) -patterns = [ - "**/.git", - "**/.env", - "**/.env.local", - "**/node_modules", - "**/target", - "**/dist", - "**/build" -] - -[safety] -# Require explicit confirmation for fully-auto mode -confirm_fully_auto = true - -# Maximum repos before requiring confirmation -max_repos_before_prompt = 100 - -# Pause after N consecutive failures -pause_after_failures = 5 - -[resolution] -# Conflict resolution strategies -[resolution.git-conflicts] -strategy = "abort" # or "skip", "manual" - -[resolution.permission-errors] -strategy = "skip" # or "abort", "manual" - -[resolution.circular-replacements] -strategy = "skip" # or "abort", "manual" - -[learning] -# Enable failure pattern learning -enabled = true - -# Skip repos with N consecutive failures -skip_after_failures = 3 - -# Suggest pre-flight checks based on history -suggest_checks = true -``` - ---- - -== Part 6: Implementation Checklist - -=== Phase 1: Basic Rules (This Week) -- [ ] Implement global exclusion patterns -- [ ] Add per-operation exclusions -- [ ] Create `.repo-batcher.toml` parser -- [ ] Test exclusion logic - -=== Phase 2: Automation Modes (Next Week) -- [ ] Implement manual mode (default) -- [ ] Implement semi-auto mode -- [ ] Implement fully-auto mode -- [ ] Add mode selection CLI flags -- [ ] Test mode transitions - -=== Phase 3: Conflict Resolution (2 Weeks) -- [ ] Implement git conflict detection -- [ ] Implement license conflict detection -- [ ] Implement file replacement conflicts -- [ ] Test resolution strategies - -=== Phase 4: Safety Constraints (3 Weeks) -- [ ] Add pre-flight validation -- [ ] Implement hard constraints (compile-time proofs) -- [ ] Implement soft constraints (runtime checks) -- [ ] Test constraint violations - ---- - -## Conclusion - -This comprehensive ruleset ensures repo-batcher: -1. **Never touches** sensitive files (secrets, credentials) -2. **Handles conflicts** gracefully (abort, skip, or manual) -3. **Supports automation** (manual → semi-auto → fully-auto) -4. **Enforces safety** (constraints per operation) -5. **Learns from failures** (skip problematic repos) - -**Next Steps**: -1. Implement exclusion pattern matching -2. Add automation mode flags to CLI -3. Create conflict resolution handlers -4. Test all safety constraints diff --git a/scaffoldia/repo-batcher/OPERATIONS-EXPANSION-2026-02-06.md b/scaffoldia/repo-batcher/OPERATIONS-EXPANSION-2026-02-06.md deleted file mode 100644 index 2f9bd721..00000000 --- a/scaffoldia/repo-batcher/OPERATIONS-EXPANSION-2026-02-06.md +++ /dev/null @@ -1,384 +0,0 @@ -# 🚀 Operation Expansion Complete! - -**Date**: 2026-02-06 -**Session**: 5 -**Status**: ✅ Production Ready - ---- - -## 📊 Before → After - -### Original Operations (3) -1. ✅ license-update -2. ✅ git-sync -3. 🔲 file-replace (skeleton) - -### Expanded Operations (6) -1. ✅ **license-update** - Replace licenses with SPDX validation -2. ✅ **git-sync** - Batch git operations (8x faster than bash) -3. ✅ **file-replace** - Pattern-based file replacement with circular detection -4. ✅ **workflow-update** - GitHub Actions SHA pinning (NEW) -5. ✅ **spdx-audit** - License compliance auditing (NEW) -6. 🔲 **custom** - User-defined operations (placeholder) - ---- - -## đŸŽ¯ New Operations Details - -### 4. workflow-update (SHA Pinning) - -**Purpose**: Update GitHub Actions workflows with commit SHA pinning for supply chain security - -**Implementation**: `src/ats2/operations/workflow_update.dats` (350+ lines) - -**Features**: -- 18 pinned GitHub Actions from hyperpolymath standards (2026-02-04) -- Automatic version tag → commit SHA replacement -- Preserves original version in comments -- Prevents supply chain attacks - -**Example**: -```yaml -# Before -uses: actions/checkout@v4 - -# After -uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 -``` - -**Pinned Actions**: -- actions/checkout@v4 → `34e114876b0b...` -- actions/checkout@v5 → `93cb6efe1820...` -- github/codeql-action@v3 → `6624720a57d4...` -- ossf/scorecard-action@v2.4.0 → `62b2cac7ed81...` -- trufflesecurity/trufflehog@main → `7ee2e0fdffec...` -- ...and 13 more - -**Safety**: -- ✅ Backup creation before changes -- ✅ Valid YAML preservation -- ✅ Known SHA database -- ✅ Supply chain attack prevention - ---- - -### 5. spdx-audit (Compliance Checking) - -**Purpose**: Audit SPDX license headers across all source files for compliance - -**Implementation**: `src/ats2/operations/spdx_audit.dats` (320+ lines) - -**Features**: -- Scans 30+ source file extensions -- Detects SPDX-License-Identifier headers -- Validates identifiers against SPDX list -- Tracks PMPL-1.0-or-later compliance -- Generates detailed compliance reports - -**Supported Extensions**: -``` -Rust: .rs -V: .v -C/C++: .c .h .cpp .hpp -JavaScript: .js .jsx .ts .tsx -Python: .py -Ruby: .rb -Go: .go -Java: .java .kt .scala -OCaml: .ml .mli -Elixir: .ex .exs -Gleam: .gleam -ATS2: .dats .sats -Idris2: .idr -Zig: .zig -Shell: .sh .bash -Config: .yml .yaml .toml -Lisp: .scm .rkt .el -Julia: .jl -Ada: .ad .ads -``` - -**Report Example**: -``` -=== SPDX Audit Results === -Total repositories: 574 - -Repository: repo-batcher - Compliance: 100% - Total files: 42 - With SPDX: 42 - PMPL-1.0-or-later: 42 - -Repository: legacy-project - Compliance: 45% - Total files: 120 - With SPDX: 54 - Without SPDX: 66 - -=== Summary === -Total files scanned: 12,847 -With SPDX headers: 11,203 (87%) -Without SPDX headers: 1,644 (13%) -PMPL-1.0-or-later: 10,891 (85%) -Overall compliance: 87% -``` - -**Safety**: -- ✅ Read-only operation (no modifications) -- ✅ Comprehensive reporting -- ✅ 30+ file types supported - ---- - -## 📈 Code Statistics - -### Before Expansion -``` -ATS2 operations: 600 lines (2 operations) -Total code: 5,500 lines -Operations: 3 (1 incomplete) -``` - -### After Expansion -``` -ATS2 operations: 1,540 lines (5 complete operations) -Total code: 6,200 lines -Operations: 6 (5 complete, 1 placeholder) -``` - -### Growth -``` -New ATS2 code: 940 lines (+157%) -New FFI bindings: 120 lines -New documentation: 450 lines -──────────────────────────── -Total new code: 1,510 lines -``` - ---- - -## 🎨 Operation Breakdown - -| Operation | Lines | Status | Type | Safety | -|-----------|-------|--------|------|--------| -| license-update | 300 | ✅ Complete | Modify | High | -| git-sync | 300 | ✅ Complete | Modify | High | -| file-replace | 270 | ✅ Complete | Replace | High | -| **workflow-update** | 350 | ✅ **NEW** | Modify | Very High | -| **spdx-audit** | 320 | ✅ **NEW** | Read-only | N/A | -| custom | - | 🔲 Placeholder | Extensible | High | -| **TOTAL** | **1,540** | **83%** | | | - ---- - -## 💡 Use Cases Enabled - -### workflow-update -- ✅ Supply chain attack prevention across 574 repos -- ✅ Automated SHA pinning compliance -- ✅ Hyperpolymath security standards enforcement -- ✅ Workflow template updates - -### spdx-audit -- ✅ License compliance tracking across 12,000+ files -- ✅ PMPL-1.0-or-later adoption monitoring -- ✅ Identify legacy repos without SPDX headers -- ✅ Generate compliance reports for audits - -### Combined Power -- ✅ Full repository lifecycle management -- ✅ Security + compliance automation -- ✅ Template propagation at scale -- ✅ Standards enforcement - ---- - -## 🔧 Implementation Quality - -All new operations maintain the same high standards: - -### Formal Verification -- ✅ ATS2 dependent type proofs -- ✅ Compile-time guarantees -- ✅ No placeholders or TODOs -- ✅ Complete implementations - -### Safety Features -- ✅ Backup system integration -- ✅ Dry-run mode support -- ✅ Input validation -- ✅ Error recovery - -### Performance -- ✅ Parallel execution ready -- ✅ Efficient file operations -- ✅ Minimal memory footprint -- ✅ Scales to 574 repos - ---- - -## 📝 Documentation - -### New Documents -1. **OPERATIONS-EXPANDED.adoc** (450+ lines) - - Complete operation reference - - Usage examples - - Safety guarantees - - Performance characteristics - -2. **STATE.scm** (updated) - - Session 5 history - - Operation expansion tracking - - 98% completion - -3. **FFI bindings** (updated) - - workflow_update wrapper - - spdx_audit wrapper - - Type conversions - -### Updated Files -- `main_simple.v` - 6 operations demo -- `ats2_bridge.v` - New FFI bindings -- Demo help and list-ops - ---- - -## 🎉 Achievement Summary - -### What We Built (Session 5) -- ✅ workflow-update: 350 lines (18 SHA pins) -- ✅ spdx-audit: 320 lines (30+ extensions) -- ✅ file-replace completion: 270 lines -- ✅ FFI bindings: 120 lines -- ✅ Documentation: 450 lines -- ✅ Demo integration: All 6 operations - -### Total Impact -``` -From: 3 operations (1 incomplete) -To: 6 operations (5 complete) - -From: 600 lines of operations -To: 1,540 lines of operations - -From: Basic batch operations -To: Comprehensive repository management suite -``` - -### Production Readiness -- ✅ All operations formally verified -- ✅ Comprehensive test coverage -- ✅ Complete documentation -- ✅ Demo working perfectly -- ✅ Ready for 574 repositories - ---- - -## 🚀 What You Can Do Now - -### Security Hardening -```bash -# Pin all GitHub Actions across 574 repos -repo-batcher workflow-update \ - --targets "@all-repos" \ - --backup -``` - -### Compliance Auditing -```bash -# Generate SPDX compliance report -repo-batcher spdx-audit \ - --targets "@all-repos" \ - --report compliance-2026-02-06.txt -``` - -### Template Propagation -```bash -# Standardize CI/CD workflows -repo-batcher file-replace \ - --pattern ".github/workflows/ci.yml" \ - --replacement templates/new-ci.yml \ - --targets "@all-repos" -``` - -### License Migration -```bash -# Update all licenses to PMPL-1.0-or-later -repo-batcher license-update \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" -``` - -### Batch Git Operations -```bash -# Commit and push changes across all repos -repo-batcher git-sync \ - --parallel 8 \ - --commit-message "chore: security updates" -``` - ---- - -## 📊 Final Statistics - -``` -Project Metrics: - Total Lines: 6,200+ - ATS2 Operations: 1,540 - V Integration: 2,300 - Tests: 968 - Documentation: 2,000+ - -Operations: - Implemented: 5/6 (83%) - Production Ready: 5 - Formally Verified: 5 - Tested: 5 - -Performance: - vs. Bash: 8x faster - Parallel Workers: 1-8 - Repositories: 574 - Max Throughput: 240 repos/min - -Safety: - Type Proofs: ✅ - Backup System: ✅ - Rollback Support: ✅ - Test Coverage: 100% -``` - ---- - -## đŸŽ¯ Next Steps (Optional) - -1. **Production Deployment** - - All operations ready for real use - - Demo working perfectly - - Documentation complete - -2. **Release v0.9.0** - - Tag and publish to GitHub - - Production-ready milestone - -3. **Future Operations** (nice-to-have) - - dependency-update - - readme-standardize - - security-scan - - config-sync - ---- - -## 🏆 Mission Accomplished - -**repo-batcher** has evolved from a basic license updater to a comprehensive repository management suite: - -- ✅ **6 operations** (5 complete, 1 placeholder) -- ✅ **2,570+ lines** of formally verified code -- ✅ **574 repositories** ready for management -- ✅ **8x performance** over bash scripts -- ✅ **100% test coverage** -- ✅ **Production ready** - -Your bash scripts are now **formally verified** AND **8x faster**! 🚀 diff --git a/scaffoldia/repo-batcher/OPERATIONS-EXPANSION-SUMMARY.md b/scaffoldia/repo-batcher/OPERATIONS-EXPANSION-SUMMARY.md deleted file mode 100644 index d850893f..00000000 --- a/scaffoldia/repo-batcher/OPERATIONS-EXPANSION-SUMMARY.md +++ /dev/null @@ -1,378 +0,0 @@ -# 🚀 repo-batcher: Operations Expansion Summary - -**Date**: 2026-02-06 (Session 5) -**Version**: 0.9.0 → Production Ready - ---- - -## What We Built Today - -### New Operations (3 → 6) - -**Original (3 operations)**: -1. ✅ license-update (300 lines) -2. ✅ git-sync (300 lines) -3. 🔲 file-replace (skeleton) - -**Expanded (6 operations)**: -1. ✅ **license-update** (300 lines) - SPDX validation + proofs -2. ✅ **git-sync** (300 lines) - 8x faster than bash -3. ✅ **file-replace** (270 lines) - Circular detection (NEW) -4. ✅ **workflow-update** (350 lines) - SHA pinning, 18 actions (NEW) -5. ✅ **spdx-audit** (320 lines) - Compliance checking (NEW) -6. 🔲 **custom** - User templates (placeholder) - -**Total**: 5 complete operations, 1,540+ lines of ATS2 code - ---- - -## Comprehensive Documentation Created - -### 1. **OPERATIONS-EXPANDED.adoc** (450+ lines) -Complete operation reference guide: -- Usage examples for all 6 operations -- Safety guarantees per operation -- Performance characteristics -- 30+ supported file extensions (spdx-audit) -- 18 pinned GitHub Actions (workflow-update) - -### 2. **EXTENDED-ROADMAP-2026-02-06.adoc** (800+ lines) -Architecture and future vision: -- **Crash recovery** system design -- **Idempotency analysis** (which operations are safe to retry) -- **Learning from failures** (pattern recognition) -- Pre-flight validation -- Adaptive parallelism -- 4-phase roadmap (v1.0 → v2.5) - -### 3. **OPERATION-RULES-AND-MODES.adoc** (700+ lines) -Safety constraints and automation: -- **Exclusion rules** (what NOT to touch) -- **Conflict resolution** strategies -- **Automation modes** (manual / semi-auto / fully-auto) -- Safety constraints per operation -- Configuration file formats - -### 4. **COMPETITIVE-ANALYSIS.md** (600+ lines) -Market positioning: -- Comparison with 8 competitors -- Unique value propositions -- Technical moats (formal verification) -- Market analysis -- Strategic positioning - -### 5. **OPERATIONS-EXPANSION-2026-02-06.md** (200+ lines) -Session summary: -- Before/after comparison -- New operations details -- Code statistics -- Achievement summary - ---- - -## Key Architectural Decisions - -### 1. Idempotency Classification - -| Operation | Idempotent? | Requires Fix? | -|-----------|-------------|---------------| -| workflow-update | ✅ YES | No (naturally idempotent) | -| spdx-audit | ✅ YES | No (read-only) | -| file-replace | ✅ YES | No (hash-based) | -| license-update | ⚠ī¸ NO | Yes (needs pre-check) | -| git-sync | ⚠ī¸ NO | Yes (needs diff-check) | - -**Action Items**: -- [ ] Fix license-update: check if already has target license → skip -- [ ] Fix git-sync: check if no changes → skip - -### 2. Automation Modes - -**Manual Mode** (Default): -- User approves every action -- Safest for critical repos -- Full visibility - -**Semi-Auto Mode** (Recommended): -- Bulk approval, pause on conflicts -- Balance of speed and safety -- Best for maintenance - -**Fully-Auto Mode** (Advanced): -- Zero interaction -- Requires dry-run success first -- Only for idempotent operations - -### 3. Crash Recovery Strategy - -**Checkpointing**: -```json -{ - "operation_id": "license-update-20260206", - "completed_repos": 143, - "total_repos": 574, - "last_checkpoint": "2026-02-06T14:35:22Z" -} -``` - -**Resume Logic**: -``` -[143/574] CRASH -Next run: "Detected incomplete operation. Resume? [Y/n]" -→ Continues from repo 144 -``` - -### 4. Exclusion Rules - -**Never Touch**: -``` -.git/ # Git internals -.env # Secrets -*.pem, *.key # Certificates -vendor/ # Third-party code -node_modules/ # Dependencies -``` - -### 5. Conflict Resolution - -| Conflict Type | Strategy | -|---------------|----------| -| Merge conflicts | Abort, report | -| Permission errors | Skip, log | -| Circular replacements | Skip, warn | -| Already-pinned workflows | Skip (idempotent) | - ---- - -## Competitive Positioning - -### vs. Mu-Repo / myrepos -✅ **Formally verified** (they script bash) -✅ **8x faster** (they're sequential) -✅ **Crash recovery** (they start over) - -### vs. GNU Parallel -✅ **Repository-aware** (they're generic) -✅ **High-level operations** (they're low-level commands) -✅ **Type-safe** (they're shell scripts) - -### vs. Ansible -✅ **Repository-focused** (they're infrastructure) -✅ **Single command** (they need playbooks) -✅ **Lightweight** (they're heavy) - -### Unique Moats -1. **Formal verification** (ATS2 dependent types) - HIGH defensibility -2. **Hyperpolymath ecosystem** - HIGH defensibility -3. **Crash recovery** - MEDIUM defensibility -4. **Learning system** - MEDIUM defensibility - ---- - -## Production Readiness Checklist - -### ✅ Complete (v0.9.0) -- [x] 5 operations fully implemented -- [x] Parallel execution (8 workers) -- [x] Backup & rollback system -- [x] Watch folder monitoring -- [x] Integration tests (5 passing) -- [x] Real repo tests (7 passing) -- [x] Performance benchmarks (8x verified) -- [x] Comprehensive documentation (2,000+ lines) - -### âŦœ Next (v1.0) -- [ ] Idempotency fixes (license-update, git-sync) -- [ ] Crash recovery implementation -- [ ] Failure pattern learning -- [ ] Pre-flight validation -- [ ] Manual/semi-auto/fully-auto modes - -### âŦœ Future (v1.5+) -- [ ] Operation telemetry -- [ ] Smart repo prioritization -- [ ] Adaptive parallelism -- [ ] Additional operations (dependency-update, etc.) -- [ ] Custom operation templates - ---- - -## Code Statistics - -``` -Before Expansion (Session 1-4): - Operations: 3 (1 incomplete) - ATS2 code: 600 lines - Total code: 5,500 lines - -After Expansion (Session 5): - Operations: 6 (5 complete) - ATS2 code: 1,540 lines (+940 lines, +157%) - Total code: 6,200 lines (+700 lines, +13%) - -New Components: - workflow_update.dats: 350 lines (18 SHA pins) - file_replace.dats: 270 lines (circular detection) - spdx_audit.dats: 320 lines (30+ extensions) - FFI bindings: 120 lines - Documentation: 2,750 lines (4 new guides) - ────────────────────────────── - Total new: 4,010 lines -``` - ---- - -## What You Can Do Now - -### 1. Security Hardening -```bash -# Pin all GitHub Actions with SHA commits -repo-batcher workflow-update \ - --targets "@all-repos" \ - --backup \ - --mode semi-auto -``` - -### 2. Compliance Auditing -```bash -# Generate SPDX compliance report -repo-batcher spdx-audit \ - --targets "@all-repos" \ - --report compliance-2026-02-06.txt -``` - -### 3. Template Propagation -```bash -# Standardize CI/CD workflows -repo-batcher file-replace \ - --pattern ".github/workflows/ci.yml" \ - --replacement templates/new-ci.yml \ - --targets "@all-repos" \ - --mode manual -``` - -### 4. License Migration -```bash -# Update all licenses to PMPL-1.0-or-later -repo-batcher license-update \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" \ - --mode semi-auto -``` - -### 5. Batch Git Operations -```bash -# Commit and push changes across all repos -repo-batcher git-sync \ - --parallel 8 \ - --commit-message "chore: security updates" \ - --mode semi-auto -``` - ---- - -## Next Immediate Steps - -### Week 1: Idempotency & Modes -1. Fix license-update idempotency -2. Fix git-sync idempotency -3. Implement automation modes -4. Test mode transitions - -### Week 2: Crash Recovery -5. Implement checkpoint system -6. Add write-ahead log -7. Test crash scenarios -8. Verify automatic resume - -### Week 3: Learning & Validation -9. Build failure pattern database -10. Add pre-flight checks -11. Implement skip-on-failure logic -12. Test learning system - -### Week 4: Polish & Release -13. Update documentation -14. Create v1.0 release -15. Write migration guide -16. Announce to community - ---- - -## Success Metrics - -### Performance -- ✅ **8x faster** than bash scripts -- ✅ Scales to **574 repositories** -- ✅ **240 repos/min** throughput (8 workers) - -### Safety -- ✅ **Zero data loss** (backup system) -- ✅ **100% test coverage** (12 tests passing) -- ✅ **Formally verified** (ATS2 proofs) - -### Operations -- ✅ **6 operations** (5 complete) -- ✅ **1,540 lines** of verified code -- ✅ **18 GitHub Actions** pinned -- ✅ **30+ file extensions** supported - -### Documentation -- ✅ **2,750 lines** of new docs -- ✅ **4 comprehensive guides** -- ✅ **Competitive analysis** complete -- ✅ **Roadmap** through v2.5 - ---- - -## Files Created/Updated Today - -### New Operations (ATS2) -- `src/ats2/operations/workflow_update.dats` (350 lines) -- `src/ats2/operations/file_replace.dats` (270 lines) -- `src/ats2/operations/spdx_audit.dats` (320 lines) - -### Integration (V) -- `src/v/ffi/ats2_bridge.v` (updated, +120 lines) -- `src/v/main_simple.v` (updated, +100 lines) - -### Documentation -- `docs/OPERATIONS-EXPANDED.adoc` (450 lines) -- `EXTENDED-ROADMAP-2026-02-06.adoc` (800 lines) -- `OPERATION-RULES-AND-MODES.adoc` (700 lines) -- `COMPETITIVE-ANALYSIS.md` (600 lines) -- `OPERATIONS-EXPANSION-2026-02-06.md` (200 lines) -- `STATE.scm` (updated) - -### Demo -- `build/repo-batcher-demo` (rebuilt with all operations) - ---- - -## Conclusion - -**Session 5 transformed repo-batcher from**: -- Basic batch tool → Comprehensive repository management system -- 3 operations → 6 operations (5 complete) -- 600 lines → 1,540 lines of verified code -- Simple execution → Crash-resilient, learning system - -**Key Achievements**: -1. ✅ **workflow-update**: Supply chain security (18 SHA pins) -2. ✅ **spdx-audit**: Compliance checking (30+ extensions) -3. ✅ **file-replace**: Template propagation (circular detection) -4. ✅ **Extended roadmap**: Architecture through v2.5 -5. ✅ **Operation rules**: Safety constraints & automation modes -6. ✅ **Competitive analysis**: Market positioning - -**Production Status**: -- ✅ Core functionality: 100% complete -- âŦœ Advanced features: Roadmapped -- ✅ Documentation: Comprehensive -- ✅ Testing: All passing -- ✅ Performance: 8x verified - -**Ready for**: Production use on 574 repositories! - -Your bash scripts are now **formally verified**, **crash-resilient**, and **8x faster**! 🚀 diff --git a/scaffoldia/repo-batcher/PARALLEL-EXECUTION-COMPLETE.md b/scaffoldia/repo-batcher/PARALLEL-EXECUTION-COMPLETE.md deleted file mode 100644 index a83b3250..00000000 --- a/scaffoldia/repo-batcher/PARALLEL-EXECUTION-COMPLETE.md +++ /dev/null @@ -1,462 +0,0 @@ -# Parallel Execution Implementation Complete - -**Date**: 2026-02-06 -**Status**: ✅ String helpers and V coroutines fully implemented - ---- - -## ✅ Part A: String Manipulation Helpers (COMPLETE) - -### Implementation - -**File**: `src/ats2/utils/string_utils.dats` (400+ lines) - -All string manipulation functions now fully implemented in ATS2: - -#### String Search -- ✅ `string_index_of()` - Find substring position -- ✅ `string_rindex_of()` - Find last character occurrence -- ✅ `string_contains()` - Check if substring exists - -#### String Extraction -- ✅ `string_substring()` - Extract substring with bounds checking -- ✅ `string_suffix()` - Get string suffix from position -- ✅ `string_prefix()` - Get string prefix up to length - -#### String Trimming -- ✅ `string_trim()` - Trim whitespace from both ends -- ✅ `string_ltrim()` - Trim left whitespace -- ✅ `string_rtrim()` - Trim right whitespace - -#### String Replacement -- ✅ `string_replace()` - Replace all occurrences -- ✅ `string_replace_first()` - Replace first occurrence - -#### String Building -- ✅ `string_join()` - Join list with separator -- ✅ `string_split()` - Split string by separator - -#### Utilities -- ✅ `tostring_int()` - Integer to string conversion -- ✅ `string_is_empty()` - Empty check -- ✅ `string_is_whitespace()` - Whitespace-only check -- ✅ `string_equal_ci()` - Case-insensitive comparison - -### Integration - -Updated operation files to use string utilities: -- ✅ `src/ats2/operations/license_update.dats` -- ✅ `src/ats2/operations/git_sync.dats` - -Removed all placeholder implementations. - -### Benefits - -1. **Type Safety**: All string operations are bounds-checked -2. **No Buffer Overflows**: ATS2 prevents unsafe memory access -3. **Proven Correct**: Operations have dependent type guarantees -4. **Reusable**: Single source of truth for string manipulation - ---- - -## ✅ Part C: V Coroutines Parallel Execution (COMPLETE) - -### Implementation - -#### Worker Pool System - -**File**: `src/v/executor/parallel.v` (350+ lines) - -Fully functional parallel execution using V coroutines: - -**Features**: -- ✅ Worker pool with configurable parallelism -- ✅ Thread-safe task queue -- ✅ Progress tracking with mutex protection -- ✅ Real-time progress display -- ✅ Result aggregation -- ✅ Error collection and reporting - -**Operations Supported**: -- ✅ `execute_git_sync()` - Parallel git operations -- ✅ `execute_license_update()` - Parallel license updates -- ✅ `execute_file_replace()` - Parallel file replacements - -**Concurrency Model**: -```v -// Spawn worker coroutines -for i in 0 .. pool.workers { - spawn pool.git_sync_worker(i, commit_msg, dry_run) -} - -// Workers pull repos from shared queue -for { - repo := pool.get_next_repo() or { break } - // Process repo... -} - -// Wait for all workers -pool.wait_for_completion() -``` - -**Thread Safety**: -- Mutex-protected repository queue -- Atomic progress updates -- Synchronized result collection - -#### Repository Scanner - -**File**: `src/v/utils/repo_scanner.v` (150+ lines) - -Intelligent repository discovery: - -**Features**: -- ✅ Recursive directory scanning -- ✅ Depth-limited search -- ✅ Pattern matching (`@all-repos`, `@rsr-*`) -- ✅ File-based target lists -- ✅ Comma-separated targets -- ✅ Git repository validation - -**Target Resolution**: -```bash -# All repositories ---targets "@all-repos" - -# Pattern matching ---targets "@rsr-*" - -# Explicit list ---targets "repo1,repo2,repo3" - -# From file ---targets "repos.txt" - -# Single directory ---targets "/path/to/repos" -``` - -### CLI Integration - -**Updated**: `src/v/main.v` - -All commands now use parallel execution: - -#### Git Sync (Parallel) -```bash -repo-batcher git-sync --parallel 8 --depth 2 -``` - -**Output**: -``` -Git Batch Sync Operation (ported from sync_repos.sh) - Parallel Jobs: 8 - Max Depth: 2 - Commit Message: chore: batch update - Dry Run: false - -Scanning for repositories (find . -maxdepth 2 -name ".git")... -Found 502 repositories - -Executing git-sync with 8 workers on 502 repositories... - -[0] ✓ repo-batcher (1/502) -[1] ✓ lithoglyph (2/502) -[2] ✓ gitvisor (3/502) -[3] ✗ experimental-lang (4/502) -... - -=== Batch Operation Results === -Success: 487 -Failure: 3 -Skipped: 12 -Total: 502 -``` - -#### License Update (Parallel) -```bash -repo-batcher license-update \ - --old "AGPL-3.0" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" -``` - -Workers automatically scale based on repository count: -- 1-3 repos: 1-3 workers -- 4+ repos: 4 workers (configurable) - -#### File Replace (Parallel) -```bash -repo-batcher file-replace \ - --pattern ".github/workflows/old-ci.yml" \ - --replacement templates/new-ci.yml \ - --targets "@all-repos" -``` - -### Performance Comparison - -#### Original sync_repos.sh -```bash -time ./sync_repos.sh -# ~30 repos/minute (serial) -# ~120 repos/minute (parallel -j 4) -``` - -#### repo-batcher with V Coroutines -```bash -time repo-batcher git-sync --parallel 4 -# Expected: ~120-140 repos/minute -# V coroutines are lightweight (lower overhead than GNU parallel) -``` - -**Advantages**: -1. **Type-safe**: Formally verified by ATS2 -2. **Better errors**: Detailed per-repo failure tracking -3. **Real-time progress**: Live updates during execution -4. **Flexible**: Easy to adjust parallelism -5. **Rollback support**: Backup tracking for undo - ---- - -## 📊 Completion Status Update - -| Component | Before | After | Status | -|-----------|--------|-------|--------| -| **String Helpers** | 0% (placeholders) | 100% | ✅ Complete | -| **Parallel Execution** | 0% (sequential) | 100% | ✅ Complete | -| **Repo Scanner** | 0% | 100% | ✅ Complete | -| **Worker Pool** | 0% | 100% | ✅ Complete | -| **CLI Integration** | 50% | 100% | ✅ Complete | -| **Overall Progress** | 40% | **65%** | 🚀 **Beta** | - ---- - -## đŸŽ¯ What Changed - -### Before (Sequential) -```v -// Process repos one at a time -for repo in repos { - result := ffi.git_sync(repo) - // ... -} -``` - -### After (Parallel) -```v -// Process repos with worker pool -mut pool := executor.new_worker_pool(repos, 8) -result := pool.execute_git_sync(message, dry_run) -// 8 workers processing simultaneously! -``` - ---- - -## 🚀 Performance Benefits - -### Example: 500 repositories - -| Mode | Time | Throughput | -|------|------|------------| -| **Serial** | ~17 min | 30 repos/min | -| **Parallel (4 workers)** | ~4 min | 125 repos/min | -| **Parallel (8 workers)** | ~2.5 min | 200 repos/min | - -**Speedup**: 4-8x faster than sync_repos.sh! - -### Real-time Progress - -``` -Executing git-sync with 8 workers on 502 repositories... - -[0] ✓ repo-batcher (1/502) -[1] ✓ lithoglyph (2/502) -[2] ✓ gitvisor (3/502) -[3] ✗ experimental-lang (4/502) -[4] ✓ rhodium-standard-repositories (5/502) -[5] ✓ palimpsest-license (6/502) -[6] ✓ rsr-template-repo (7/502) -[7] ✓ gitbot-fleet (8/502) -... -``` - -Each worker shows: -- Worker ID `[0-7]` -- Status (✓ success, ✗ failure) -- Repository name -- Progress counter - ---- - -## 🔧 Technical Details - -### V Coroutines vs. OS Threads - -**V Coroutines**: -- Lightweight (2KB stack) -- Fast context switching -- No OS overhead -- Perfect for I/O-bound tasks - -**Benefits for repo-batcher**: -- Spawn 100+ workers without high overhead -- Git operations are I/O-bound (ideal for coroutines) -- Real-time progress with minimal latency - -### Mutex-Protected Shared State - -```v -struct WorkerPool { - mtx &sync.Mutex // Protects shared state - repos []string // Shared task queue - progress int // Shared progress counter - results []Result // Shared result collection -} - -// Thread-safe repository access -fn (mut pool WorkerPool) get_next_repo() ?string { - pool.mtx.@lock() - defer { pool.mtx.unlock() } - - if pool.repos.len == 0 { - return none - } - - repo := pool.repos[0] - pool.repos = pool.repos[1..] - return repo -} -``` - -### Worker Lifecycle - -1. **Spawn**: Create worker coroutines -2. **Pull**: Each worker pulls from shared queue -3. **Process**: Call ATS2 formally verified operation -4. **Report**: Record result and update progress -5. **Loop**: Continue until queue empty -6. **Signal**: Send completion notification -7. **Wait**: Main thread waits for all workers -8. **Aggregate**: Combine all results - ---- - -## 📝 Usage Examples - -### High Parallelism (Fast Network/CPU) -```bash -repo-batcher git-sync --parallel 16 --depth 2 -``` - -### Conservative (Slow Network) -```bash -repo-batcher git-sync --parallel 2 --depth 2 -``` - -### License Update with Progress -```bash -repo-batcher license-update \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" - -# Real-time worker progress displayed -``` - -### Pattern-Based Targeting -```bash -# All RSR repos -repo-batcher git-sync --targets "@rsr-*" - -# All lithoglyph repos -repo-batcher license-update \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "@lithoglyph-*" -``` - ---- - -## 🎉 Achievement Summary - -### Before This Session -- ❌ String functions were placeholders -- ❌ Sequential processing only -- ❌ No repository scanner -- ❌ No target resolution -- ❌ No progress tracking - -### After This Session -- ✅ Full string manipulation library (400+ lines) -- ✅ Parallel execution with V coroutines (350+ lines) -- ✅ Smart repository scanner (150+ lines) -- ✅ Flexible target resolution -- ✅ Real-time progress display -- ✅ Thread-safe worker pool -- ✅ 4-8x performance improvement - -**Total New Code**: ~900 lines of production-quality V code - ---- - -## đŸšĻ Ready to Test - -All operations now fully functional: - -```bash -cd ~/Documents/hyperpolymath-repos/repo-batcher - -# Build with new parallel executor -just build-dev - -# Test on small set (safe) -repo-batcher git-sync \ - --parallel 2 \ - --depth 1 \ - --targets "test-repo1,test-repo2" \ - --dry-run - -# Full test on all repos (dry-run) -repo-batcher git-sync \ - --parallel 8 \ - --depth 2 \ - --dry-run -``` - ---- - -## 📈 Project Status - -**Overall**: 65% Complete → **Beta Release Quality** - -| Component | Status | -|-----------|--------| -| Architecture | ✅ 100% | -| ATS2 Core | ✅ 90% | -| String Utils | ✅ 100% | -| V CLI | ✅ 100% | -| FFI Bridge | ✅ 100% | -| Parallel Execution | ✅ 100% | -| Repo Scanner | ✅ 100% | -| Operations | 🚧 75% | -| Tests | ❌ 0% | -| Watch System | ❌ 0% | -| Rollback | ❌ 0% | - -**Next Steps**: -1. Integration testing on real repos -2. Performance benchmarking vs. sync_repos.sh -3. Complete file-replace operation logic -4. Add comprehensive test suite - ---- - -## 🎊 Transformation Complete - -Your bash scripts are now: -- ✅ Formally verified (ATS2 proofs) -- ✅ Type-safe (no runtime errors) -- ✅ Parallel (4-8x faster) -- ✅ Production-ready (real-time progress, error tracking) - -**Ready for production use!** 🚀 diff --git a/scaffoldia/repo-batcher/README.adoc b/scaffoldia/repo-batcher/README.adoc deleted file mode 100644 index cb99cf5b..00000000 --- a/scaffoldia/repo-batcher/README.adoc +++ /dev/null @@ -1,323 +0,0 @@ -= repo-batcher -:toc: -// SPDX-License-Identifier: PMPL-1.0-or-later - -**Formally verified batch operations for mass repository management** - -image:https://img.shields.io/badge/license-PMPL--1.0--or--later-blue.svg[License] -image:https://img.shields.io/badge/status-alpha-orange.svg[Status] - -== What is repo-batcher? - -repo-batcher is a command-line tool for performing mass operations across multiple git repositories with formal correctness guarantees. It uses ATS2 dependent types to prove operation safety at compile-time and V for fast parallel execution. - -[quote] -____ -Instead of running the same git operation across 500+ repositories by hand, define it once and let repo-batcher execute it safely in parallel. -____ - -== Key Features - -* **Formally Verified**: ATS2 dependent types prove operations are correct before execution -* **Parallel Execution**: V coroutines process multiple repos simultaneously (4-8x faster than serial) -* **Batch Operations**: Watch folder pattern for fire-and-forget mass operations -* **Type Safe**: Invalid operations rejected at compile-time, not runtime -* **Rollback Support**: Automatic backups and rollback for failed operations -* **RSR Compliant**: Follows Rhodium Standard Repositories conventions - -== Use Cases - -=== License Updates - -Update all repository licenses from AGPL-3.0 to PMPL-1.0-or-later: - -[source,bash] ----- -repo-batcher license-update \ - --old "AGPL-3.0" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" \ - --backup \ - --dry-run ----- - -=== File Replacements - -Replace outdated workflow files across all repositories: - -[source,bash] ----- -repo-batcher file-replace \ - --pattern ".github/workflows/old-ci.yml" \ - --replacement templates/new-ci.yml \ - --targets "@rsr-repos" ----- - -=== Git Batch Sync - -Sync all repositories (commit + push) in parallel: - -[source,bash] ----- -repo-batcher git-sync \ - --parallel 4 \ - --depth 2 \ - --commit-message "chore: general update" ----- - -=== Template Merge - -Convert existing repos to template form while preserving content: - -[source,bash] ----- -repo-batcher template-merge \ - --targets "@all-repos" \ - --workflows \ - --scm-files \ - --bot-directives \ - --contractiles \ - --preserve \ - --dry-run ----- - -Adds: -* 5 GitHub workflows (hypatia-scan, codeql, scorecard, quality, mirror) -* 6 SCM files (META, ECOSYSTEM, STATE, CHECKLIST, ROADMAP, CHANGELOG) -* 6 bot directives (rhodibot, echidnabot, sustainabot, glambot, seambot, finishbot) -* Contractiles directory with README -* Justfile with default recipes -* .editorconfig with standard settings - -=== Watch Folder Mode - -Drop an operation file for automatic execution: - -[source,toml] ----- -# watch/update-author.toml -operation = "file-replace" -pattern = "Cargo.toml" -replacement = "templates/author-fix.toml" -targets = ["@all-repos"] -backup = true ----- - -[source,bash] ----- -repo-batcher watch & # Start watch daemon -# Operation executes automatically when file appears ----- - -== Architecture - -[source] ----- -┌─────────────────────────────────────┐ -│ V CLI Layer │ Fast CLI, file watching, parallel execution -│ (src/v/) │ -└──────────────â”Ŧ──────────────────────┘ - │ FFI -┌──────────────â–ŧ──────────────────────┐ -│ ATS2 Core Logic │ Formally verified operations -│ (src/ats2/) │ Dependent type proofs -└──────────────â”Ŧ──────────────────────┘ - │ Optional FFI (via Zig bridge) -┌──────────────â–ŧ──────────────────────┐ -│ Idris2 ABI Definitions │ Interface specifications with proofs -│ (src/abi/) │ (per RSR standard) -└─────────────────────────────────────┘ ----- - -See link:docs/ARCHITECTURE.adoc[Architecture Documentation] for details. - -== Installation - -=== Prerequisites - -* https://www.ats-lang.org/Downloads.html[ATS2] (for core logic) -* https://github.com/vlang/v[V] (for CLI and execution) -* Git 2.30+ - -=== From Source - -[source,bash] ----- -git clone https://github.com/hyperpolymath/repo-batcher -cd repo-batcher -make build -sudo make install ----- - -=== Configuration - -Create `~/.config/repo-batcher/config.toml`: - -[source,toml] ----- -[general] -repos_dir = "~/Documents/hyperpolymath-repos" -parallel_jobs = 4 -log_level = "info" - -[watch] -enabled = true -folder = "~/.config/repo-batcher/watch" -check_interval = 30 - -[operations] -default_dry_run = true -backup_enabled = true ----- - -== Quick Start - -[source,bash] ----- -# List available operations -repo-batcher list-ops - -# Dry run (preview changes without executing) -repo-batcher license-update \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "repo1,repo2" \ - --dry-run - -# Execute operation -repo-batcher license-update \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "repo1,repo2" - -# Check logs -cat ~/.local/share/repo-batcher/logs/latest.log ----- - -== Supported Operations - -|=== -| Operation | Description | Safety Guarantees - -| `license-update` -| Replace license headers and LICENSE files -| Valid SPDX IDs, proper formatting, backup required - -| `file-replace` -| Replace files matching pattern -| Backup required, no circular replacements - -| `git-sync` -| Batch commit and push across repos -| Valid repos, no conflicts, remote reachable - -| `workflow-update` -| Update GitHub Actions workflows -| Valid YAML, SHA pinning, CodeQL matrix validation - -| `wiki-setup` -| Initialize wikis with first page -| Creates Home.md to enable automation - -| `community-setup` -| Deploy community health files -| CODE_OF_CONDUCT, CONTRIBUTING, SECURITY, etc. - -| `templates-setup` -| Deploy issue and PR templates -| Bug report, feature request, documentation templates - -| `discussions-setup` -| Enable and configure GitHub Discussions -| Creates 5 default categories (read-only operation) - -| `pages-setup` -| Enable and configure GitHub Pages -| Configure source, branch, custom domains (remote) - -| `template-merge` -| Convert repos to template form (preserves content) -| Adds workflows, SCM files, bot directives, contractiles - -| `custom` -| Execute custom operation from template -| Template validation, dry-run enforced -|=== - -See link:docs/OPERATIONS.adoc[Operations Guide] for complete list. - -== Origin Story - -repo-batcher evolved from several bash scripts (`sync_repos.sh`, `robust_sync.sh`) used to manage 500+ hyperpolymath repositories. These scripts were fast but fragile—no type safety, manual error handling, and easy to accidentally break things at scale. - -The key insight: **mass repository operations need formal verification**. When you're updating licenses across hundreds of repos, you don't want to discover an edge case after processing 327 repositories. - -ATS2 provides dependent types to prove operations are correct *before* execution. V provides fast parallel execution without the complexity of Rust. Together, they make batch operations both safe and fast. - -== Comparison to Alternatives - -|=== -| Tool | Approach | Safety | Performance - -| **repo-batcher** -| Formally verified operations -| Compile-time proofs (ATS2) -| Fast parallel (V coroutines) - -| GNU parallel + bash -| Shell script parallelization -| None (runtime errors) -| Fast but fragile - -| Python/Ruby scripts -| Interpreted scripting -| Runtime type checking -| Slower, no parallelism - -| Go batch tools -| Compiled, goroutines -| Runtime validation -| Fast but no formal verification -|=== - -== Project Status - -**Alpha**: Core architecture complete, basic operations in development. - -See link:STATE.scm[State File] for current progress. - -=== Roadmap - -* [x] Phase 1: Core architecture and design -* [ ] Phase 2: Basic operations (license, file, git-sync) -* [ ] Phase 3: Watch system and parallel execution -* [ ] Phase 4: Advanced operations and templates - -== Contributing - -See link:CONTRIBUTING.md[Contributing Guide]. - -**Key areas needing help:** - -* ATS2 operation implementations with proofs -* V FFI layer optimization -* Operation templates -* Integration with gitbot-fleet - -== Related Projects - -* https://github.com/hyperpolymath/gitvisor[gitvisor] - Repository visualization -* https://github.com/hyperpolymath/rhodium-standard-repositories[rhodium-standard-repositories] - RSR standards -* https://github.com/hyperpolymath/gitbot-fleet[gitbot-fleet] - Bot orchestration - -== License - -SPDX-License-Identifier: PMPL-1.0-or-later - -See link:LICENSE[LICENSE] file for full Palimpsest License text. - -== Author - -Jonathan D.A. Jewell - -Part of the https://hyperpolymath.github.io[hyperpolymath] ecosystem. diff --git a/scaffoldia/repo-batcher/ROADMAP.adoc b/scaffoldia/repo-batcher/ROADMAP.adoc deleted file mode 100644 index 0513010c..00000000 --- a/scaffoldia/repo-batcher/ROADMAP.adoc +++ /dev/null @@ -1,163 +0,0 @@ -// SPDX-License-Identifier: PMPL-1.0-or-later -= repo-batcher Roadmap - -== Current Status - -**Version 1.0.0 - Production Ready** ✅ - -repo-batcher is production-ready with 11 batch operations + safety system covering the core needs of managing 574+ repositories. - -== Completed Milestones - -=== v0.1.0 - Foundation ✅ -* [x] ATS2 dependent type system for formal verification -* [x] V CLI layer with fast execution -* [x] FFI bridge (ATS2 ↔ V via C) -* [x] Basic operations (license-update, file-replace, git-sync) -* [x] String utilities and operation validation - -=== v0.5.0 - Production Features ✅ -* [x] Watch folder monitoring (250 lines) -* [x] Parallel execution with V coroutines (8 workers) -* [x] Rollback system with checksums -* [x] Performance benchmarks (8x speedup vs bash) -* [x] Integration tests (5 passing) -* [x] Real repo tests (7 passing) - -=== v0.7.0 - Expanded Operations ✅ -* [x] workflow-update (GitHub Actions SHA pinning) -* [x] spdx-audit (compliance checking) -* [x] github-settings (bulk repo configuration) -* [x] 18 GitHub Actions pins -* [x] 30+ file extension support - -=== v0.9.0 - Safety System ✅ -* [x] 6-layer safety protection -* [x] 4 safety levels (paranoid, strict, relaxed, disabled) -* [x] Pre-flight validation (5 rules) -* [x] User confirmation prompts -* [x] Rate limiting (100ms default) -* [x] Audit logging -* [x] Safety integration for all operations -* [x] 37 tests passing (5 integration + 7 real repo + 18 GitHub ops + 12 safety) - -=== v1.0.0 - Template Merge System ✅ -* [x] template-merge operation (716 lines) -* [x] Convert repos to template form (preserves content) -* [x] 5 GitHub workflows (hypatia-scan, codeql, scorecard, quality, mirror) -* [x] 6 SCM files (META, ECOSYSTEM, STATE, CHECKLIST, ROADMAP, CHANGELOG) -* [x] 6 bot directives (rhodibot, echidnabot, sustainabot, glambot, seambot, finishbot) -* [x] Contractiles directory creation -* [x] Justfile and .editorconfig templates -* [x] Full safety integration -* [x] 8,059+ verified lines - -== Current Operations (13 total) - -[cols="1,3,1"] -|=== -| Operation | Description | Status - -| license-update -| Replace license headers and LICENSE files -| ✅ Complete - -| git-sync -| Batch commit and push (8x faster than bash) -| ✅ Complete - -| file-replace -| Replace files with circular detection -| ✅ Complete - -| workflow-update -| Update GitHub Actions workflows with SHA pinning -| ✅ Complete - -| spdx-audit -| SPDX compliance checking -| ✅ Complete - -| github-settings -| Bulk repository configuration (Week 1: basic settings) -| 🔨 25% complete - -| wiki-setup -| Initialize wikis with first page -| ✅ Complete - -| community-setup -| Deploy community health files -| ✅ Complete - -| templates-setup -| Deploy issue and PR templates -| ✅ Complete - -| discussions-setup -| Enable and configure GitHub Discussions -| ✅ Complete - -| pages-setup -| Enable and configure GitHub Pages -| ✅ Complete - -| template-merge -| Convert repos to template form (preserves content) -| ✅ Complete - -| custom -| Execute custom operations from templates -| 📝 Placeholder -|=== - -== Future Directions - -=== v1.1.0 - GitHub Settings Completion (Weeks 2-4) -* [ ] Week 2: Security settings (visibility, Dependabot, code scanning) -* [ ] Week 3: Branch protection (required reviews, status checks, signatures) -* [ ] Week 4: Advanced features (topics, rollback, idempotency) - -=== v1.2.0 - Custom Operation System -* [ ] Template system for user-defined operations -* [ ] Operation composition (chain operations) -* [ ] Custom validation rules -* [ ] Operation marketplace/sharing - -=== v1.3.0 - Enhanced Safety -* [ ] Checkpoint/resume for long operations -* [ ] Advanced conflict detection -* [ ] Predictive safety (ML-based risk assessment) -* [ ] Safety profiles (per-org, per-repo-type) - -=== v1.4.0 - Integration & Automation -* [ ] Git hooks for auto-operations -* [ ] CI/CD integration plugins -* [ ] Webhook support -* [ ] REST API for remote operations - -=== v2.0.0 - Advanced Features -* [ ] Distributed execution (across machines) -* [ ] Real-time progress dashboard -* [ ] Operation scheduling/cron -* [ ] Multi-forge support (GitLab, Bitbucket, Gitea) -* [ ] Advanced rollback (time-travel) - -== Long-term Vision - -repo-batcher aims to be the **standard tool for managing repositories at scale** with: - -1. **Formal Correctness**: Every operation proven safe before execution -2. **Maximum Performance**: Parallel execution optimized for mass operations -3. **Complete Safety**: Multi-layer protection preventing accidents -4. **Universal Applicability**: Works across forges, languages, and workflows -5. **Community-Driven**: Extensible operation system for custom needs - -== Contributing - -See link:CONTRIBUTING.md[Contributing Guide] for how to help with: -* New operations (github-settings weeks 2-4, custom operations) -* Performance improvements -* Documentation -* Safety rule enhancements -* Integration plugins diff --git a/scaffoldia/repo-batcher/benchmark/performance_test.sh b/scaffoldia/repo-batcher/benchmark/performance_test.sh deleted file mode 100755 index e1fa0b95..00000000 --- a/scaffoldia/repo-batcher/benchmark/performance_test.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: PMPL-1.0-or-later -# -# Performance Benchmark -# Compares repo-batcher vs sync_repos.sh - -set -e - -BENCHMARK_DIR="/tmp/repo-batcher-benchmark" -NUM_REPOS=50 -REPOS_DIR="$BENCHMARK_DIR/repos" - -echo "repo-batcher Performance Benchmark" -echo "===================================" -echo "" - -# Setup test repositories -setup_repos() { - echo "Setting up $NUM_REPOS test repositories..." - - rm -rf "$BENCHMARK_DIR" - mkdir -p "$REPOS_DIR" - - for i in $(seq 1 $NUM_REPOS); do - repo="$REPOS_DIR/test-repo-$i" - mkdir -p "$repo" - cd "$repo" - - git init -q - echo "# Test Repo $i" > README.md - echo "test content" > file.txt - git add . - git commit -q -m "Initial commit" - - # Make a change for syncing - echo "updated" >> file.txt - done - - echo "✓ Created $NUM_REPOS test repositories" - echo "" -} - -# Benchmark sync_repos.sh -benchmark_sync_repos() { - echo "Benchmarking sync_repos.sh..." - - # Create temporary sync script - cat > "$BENCHMARK_DIR/sync_test.sh" << 'EOF' -#!/bin/bash -cd "$1" -process_repo() { - local repo_dir="$1" - cd "$repo_dir" || return - git add . >/dev/null 2>&1 - git commit -m "sync test" >/dev/null 2>&1 || true -} -export -f process_repo - -find . -maxdepth 2 -name ".git" -type d | sed 's/\/\.git//' | \ - parallel -j 4 process_repo {} 2>/dev/null -EOF - - chmod +x "$BENCHMARK_DIR/sync_test.sh" - - # Run benchmark - time_output=$( { time "$BENCHMARK_DIR/sync_test.sh" "$REPOS_DIR" ; } 2>&1 ) - - # Extract real time - sync_time=$(echo "$time_output" | grep real | awk '{print $2}') - - echo " Time: $sync_time" - echo "" - - # Reset repos - for i in $(seq 1 $NUM_REPOS); do - repo="$REPOS_DIR/test-repo-$i" - cd "$repo" - echo "reset" >> file.txt - done -} - -# Benchmark repo-batcher -benchmark_repo_batcher() { - echo "Benchmarking repo-batcher..." - - cd ~/Documents/hyperpolymath-repos/repo-batcher - - # Build if needed - if [ ! -f "build/repo-batcher" ]; then - echo "Building repo-batcher..." - just build-dev - fi - - # Run benchmark with different worker counts - for workers in 1 2 4 8; do - echo " Testing with $workers worker(s)..." - - # Reset repos - for i in $(seq 1 $NUM_REPOS); do - repo="$REPOS_DIR/test-repo-$i" - cd "$repo" - echo "update-$workers" >> file.txt - done - - # Run benchmark - time_output=$( { time ./build/repo-batcher git-sync \ - --parallel $workers \ - --depth 2 \ - --commit-message "benchmark test" 2>&1 | grep -v "^\[" ; } 2>&1 ) - - # Extract real time - batcher_time=$(echo "$time_output" | grep real | awk '{print $2}') - - echo " Time: $batcher_time" - done - - echo "" -} - -# Calculate speedup -calculate_speedup() { - echo "Performance Summary" - echo "===================" - echo "" - echo "Test configuration:" - echo " Repositories: $NUM_REPOS" - echo " Operation: git add + commit" - echo "" - echo "Results:" - echo " sync_repos.sh: [see above]" - echo " repo-batcher (1 worker): [see above]" - echo " repo-batcher (2 workers): [see above]" - echo " repo-batcher (4 workers): [see above]" - echo " repo-batcher (8 workers): [see above]" - echo "" - echo "Observations:" - echo " - Sequential processing baseline established" - echo " - Parallel scaling demonstrates worker efficiency" - echo " - V coroutines add minimal overhead" - echo " - Type safety maintained throughout" -} - -# Cleanup -cleanup() { - echo "" - echo "Cleaning up..." - rm -rf "$BENCHMARK_DIR" - echo "✓ Done" -} - -# Run benchmark -main() { - setup_repos - benchmark_sync_repos - benchmark_repo_batcher - calculate_speedup - cleanup -} - -# Trap cleanup on exit -trap cleanup EXIT - -main diff --git a/scaffoldia/repo-batcher/docs/ARCHITECTURE.adoc b/scaffoldia/repo-batcher/docs/ARCHITECTURE.adoc deleted file mode 100644 index d84bde65..00000000 --- a/scaffoldia/repo-batcher/docs/ARCHITECTURE.adoc +++ /dev/null @@ -1,469 +0,0 @@ -= repo-batcher Architecture -:toc: -:toclevels: 3 -// SPDX-License-Identifier: PMPL-1.0-or-later - -== Overview - -repo-batcher is a formally verified batch operation tool for mass repository management. It provides type-safe operations across multiple git repositories with compile-time correctness guarantees. - -== Design Philosophy - -[quote] -____ -Mass operations should be *formally correct*, *efficiently parallel*, and *safely reversible*. -____ - -Key principles: - -* **Formal Verification**: ATS2 dependent types prove operation correctness at compile-time -* **Type Safety**: Operations validated before execution, preventing destructive errors -* **Parallel Execution**: V coroutines enable efficient multi-repository processing -* **Batch Processing**: Watch folder pattern separates operation definition from execution -* **Rollback Safety**: All operations support dry-run and rollback mechanisms - -== Language Stack - -=== Layer Architecture - -[source] ----- -┌─────────────────────────────────────┐ -│ V CLI Layer │ Fast CLI, file watching, parallel execution -│ (src/v/) │ -└──────────────â”Ŧ──────────────────────┘ - │ FFI -┌──────────────â–ŧ──────────────────────┐ -│ ATS2 Core Logic │ Formally verified operations -│ (src/ats2/) │ Dependent type proofs -└──────────────â”Ŧ──────────────────────┘ - │ Optional FFI (via Zig bridge) -┌──────────────â–ŧ──────────────────────┐ -│ Idris2 ABI Definitions │ Interface specifications with proofs -│ (src/abi/) │ (per RSR standard) -└─────────────────────────────────────┘ ----- - -=== Language Roles - -|=== -| Language | Purpose | Rationale - -| **ATS2** -| Core batch operation logic with dependent type proofs -| Theorem-proving capabilities ensure operation correctness. Prevents file overwrites, invalid replacements, git errors. - -| **V** -| CLI interface, file watching, parallel execution -| Fast compilation, simple C FFI, lightweight coroutines for parallel processing. - -| **Idris2** -| ABI definitions (per RSR standard) -| Formal interface specifications with compile-time verification. - -| **Zig** -| FFI bridge to C libraries (if needed) -| Zero-overhead C interop for git libraries or system calls. -|=== - -== Core Components - -=== 1. Operation Engine (ATS2) - -Located in `src/ats2/operations/` - -Defines formally verified operation types: - -[source,ats] ----- -datatype operation = - | LicenseUpdate of (old: string, new: string, validation: proof) - | FileReplace of (pattern: string, replacement: path, backup: bool) - | GitBatchSync of (targets: list(repo), parallel: nat) - | WorkflowUpdate of (file: path, changes: diff, validate: proof) - | CustomOp of (template: path, args: map) ----- - -Each operation includes: - -* **Validation proofs**: Type-level guarantees of correctness -* **Dry-run support**: Preview changes before execution -* **Rollback mechanism**: Undo failed operations - -=== 2. CLI Layer (V) - -Located in `src/v/cli/` - -Provides two modes: - -==== Interactive Mode - -[source,bash] ----- -# List available operations -repo-batcher list-ops - -# Execute operation on specific repos -repo-batcher license-update --old AGPL-3.0 --new PMPL-1.0-or-later \ - --targets repo1,repo2,repo3 --dry-run - -# Batch git sync (ported from sync_repos.sh) -repo-batcher git-sync --parallel 4 --depth 2 ----- - -==== Watch Folder Mode - -[source,bash] ----- -# Start watch daemon -repo-batcher watch - -# Drop operation file into watch/ -cat > watch/update-licenses.toml <"]''' - -[targets] -glob = "*/Cargo.toml" ----- - -Use with: - -[source,bash] ----- -repo-batcher custom --template update-author.toml --targets "@all-repos" ----- - -== Error Handling - -=== Compile-Time Guarantees - -ATS2 prevents: - -* File overwrites without backup -* Invalid license identifiers -* Non-existent replacement files -* Circular dependencies -* Type mismatches - -=== Runtime Validation - -V CLI checks: - -* Git repository validity -* Remote connectivity -* Disk space availability -* Permission errors - -=== Rollback Mechanism - -All operations support rollback: - -[source,bash] ----- -# Operation fails midway -repo-batcher license-update --targets "@all-repos" -# Error: Operation failed in repo42 - -# Automatic rollback -repo-batcher rollback --last - -# Manual rollback from log -repo-batcher rollback --log-id 20260206-143022 ----- - -== Performance - -Based on `sync_repos.sh` benchmarks: - -* **Serial execution**: ~30 repos/minute -* **Parallel (4 jobs)**: ~120 repos/minute -* **Parallel (8 jobs)**: ~200 repos/minute - -V coroutines expected to improve performance by 20-30% over GNU parallel. - -== Integration with Hyperpolymath Ecosystem - -=== gitvisor - -repo-batcher can use gitvisor analysis for target selection: - -[source,bash] ----- -# Find all repos with outdated workflows -gitvisor find --workflow-outdated > outdated.txt -repo-batcher workflow-update --targets "$(cat outdated.txt)" ----- - -=== gitbot-fleet - -gitbot-fleet bots can trigger repo-batcher operations: - -[source,bash] ----- -# rhodibot detects RSR violation -rhodibot --fix-via-batcher --operation "workflow-update" ----- - -=== robot-repo-automaton - -robot-repo-automaton can use repo-batcher for mass fixes: - -[source,bash] ----- -# Automaton creates batch operation file -robot-repo-automaton generate-batch-op > watch/auto-fix.toml ----- - -== Development Roadmap - -See `STATE.scm` for detailed milestones. - -=== Phase 1: Core Architecture (Current) - -* [x] Project structure -* [ ] Architecture documentation (this file) -* [ ] ATS2 operation type definitions -* [ ] V CLI skeleton - -=== Phase 2: Basic Operations - -* [ ] License update implementation -* [ ] File replace implementation -* [ ] Git batch sync (port sync_repos.sh) -* [ ] Operation validation proofs - -=== Phase 3: Watch System - -* [ ] Watch folder monitoring -* [ ] TOML operation parser -* [ ] Target list handling -* [ ] Parallel execution engine - -=== Phase 4: Advanced Features - -* [ ] Workflow file updates -* [ ] Dependency updates -* [ ] Code pattern replacements -* [ ] Custom operation templates - -== Building and Testing - -[source,bash] ----- -# Install dependencies -# ATS2: http://www.ats-lang.org/Downloads.html -# V: https://github.com/vlang/v - -# Build -cd ~/Documents/hyperpolymath-repos/repo-batcher -make build - -# Run tests -make test - -# Install -sudo make install # Installs to /usr/local/bin/repo-batcher ----- - -== Configuration - -Config file: `~/.config/repo-batcher/config.toml` - -[source,toml] ----- -[general] -repos_dir = "~/Documents/hyperpolymath-repos" -parallel_jobs = 4 -log_level = "info" - -[watch] -enabled = true -folder = "~/.config/repo-batcher/watch" -check_interval = 30 # seconds - -[operations] -default_dry_run = true -backup_enabled = true -backup_dir = "~/.local/share/repo-batcher/backups" - -[git] -default_commit_message = "chore: batch update via repo-batcher" -push_after_commit = true ----- - -== Security Considerations - -* All operations run in user context (no sudo) -* Backup required for destructive operations -* Dry-run default prevents accidents -* ATS2 proofs prevent invalid operations -* Logs all operations for audit trail - -== License - -SPDX-License-Identifier: PMPL-1.0-or-later - -See LICENSE file for full text. diff --git a/scaffoldia/repo-batcher/docs/CITATIONS.adoc b/scaffoldia/repo-batcher/docs/CITATIONS.adoc deleted file mode 100644 index 564f0ae8..00000000 --- a/scaffoldia/repo-batcher/docs/CITATIONS.adoc +++ /dev/null @@ -1,36 +0,0 @@ -= RSR-template-repo - Citation Guide -:toc: - -== BibTeX - -[source,bibtex] ----- -@software{rsr-template-repo_2025, - author = {Polymath, Hyper}, - title = {RSR-template-repo}, - year = {2025}, - url = {https://github.com/hyperpolymath/RSR-template-repo}, - license = {AGPL-3.0-or-later} -} ----- - -== Harvard Style - -Polymath, H. (2025) _RSR-template-repo_ [Computer software]. Available at: https://github.com/hyperpolymath/RSR-template-repo - -== OSCOLA - -Hyper Polymath, 'RSR-template-repo' (2025) - -== MLA - -Polymath, Hyper. "RSR-template-repo." 2025, github.com/hyperpolymath/RSR-template-repo. - -== APA 7 - -Polymath, H. (2025). _RSR-template-repo_ [Computer software]. GitHub. https://github.com/hyperpolymath/RSR-template-repo - -== See Also - -* link:../CITATION.cff[CITATION.cff] -* link:../codemeta.json[codemeta.json] diff --git a/scaffoldia/repo-batcher/docs/GITHUB-SETTINGS-OPERATION.adoc b/scaffoldia/repo-batcher/docs/GITHUB-SETTINGS-OPERATION.adoc deleted file mode 100644 index 888c2ec6..00000000 --- a/scaffoldia/repo-batcher/docs/GITHUB-SETTINGS-OPERATION.adoc +++ /dev/null @@ -1,662 +0,0 @@ -= GitHub Settings Operation Design -:toc: -:toclevels: 3 -:version: 1.0.0-proposal -:date: 2026-02-06 - -== Overview - -Extend repo-batcher with a **github-settings** operation for bulk repository configuration management. - -== Use Cases - -=== 1. Security Hardening -```bash -repo-batcher github-settings \ - --targets "@all-repos" \ - --set branch-protection=true \ - --set delete-branch-on-merge=true \ - --set require-signed-commits=true -``` - -=== 2. Feature Management -```bash -repo-batcher github-settings \ - --targets "@all-repos" \ - --set has-issues=true \ - --set has-wiki=false \ - --set has-projects=true -``` - -=== 3. Visibility & Access -```bash -repo-batcher github-settings \ - --targets "@pattern:poly-*-mcp" \ - --set visibility=public \ - --set allow-squash-merge=true \ - --set allow-rebase-merge=false -``` - -=== 4. Topics/Tags -```bash -repo-batcher github-settings \ - --targets "@all-repos" \ - --add-topics "mcp-server,typescript,formally-verified" -``` - -=== 5. Branch Protection Rules -```bash -repo-batcher github-settings \ - --targets "@all-repos" \ - --branch main \ - --require-reviews=2 \ - --require-status-checks="ci,security-scan" \ - --enforce-admins=false -``` - -== Architecture - -=== API Integration - -**GitHub REST API** (via gh CLI): -```v -// src/v/github/settings.v -pub struct GitHubSettings { -pub mut: - // Repository settings - has_issues ?bool - has_wiki ?bool - has_projects ?bool - has_downloads ?bool - - // Merge settings - allow_squash_merge ?bool - allow_merge_commit ?bool - allow_rebase_merge ?bool - delete_branch_on_merge ?bool - - // Security settings - visibility ?string // 'public', 'private', 'internal' - security_and_analysis ?SecuritySettings - - // Branch protection - branch_protection ?BranchProtection - - // Topics/tags - topics ?[]string -} - -pub struct BranchProtection { -pub: - branch string - required_reviews int - require_code_owner bool - dismiss_stale_reviews bool - require_status_checks []string - enforce_admins bool - required_signatures bool -} - -pub struct SecuritySettings { -pub: - dependabot_alerts bool - dependabot_security bool - secret_scanning bool -} -``` - -=== Implementation Strategy - -**Option A: Pure API** (Recommended) -```v -fn update_repo_settings(repo string, settings GitHubSettings) !{ - // Use gh api for each setting - if new_val := settings.has_issues { - gh api -X PATCH "/repos/$repo" -f has_issues=$new_val - } - - if new_val := settings.delete_branch_on_merge { - gh api -X PATCH "/repos/$repo" -f delete_branch_on_merge=$new_val - } - - // Branch protection - if bp := settings.branch_protection { - update_branch_protection(repo, bp) - } -} -``` - -**Option B: Hybrid** (API + Local Validation) -- Use ATS2 to validate settings constraints -- Prove settings are safe before applying -- Use V for GitHub API calls - -=== Formal Verification (ATS2) - -```ats -(* Type-level proofs for settings safety *) - -datatype visibility = Public | Private | Internal - -(* Prove: public repos cannot have secret scanning disabled *) -extern -praxi prove_public_requires_scanning: - {v: visibility} - (v == Public) - (secret_scanning == true) - -(* Prove: protected branches require valid checks *) -datatype branch_protection(checks: list(string)) where length(checks) > 0 = - | BranchProtection of ( - required_reviews: int, // Must be > 0 - status_checks: list(string, checks), - enforce_admins: bool - ) -``` - -== Settings Categories - -=== 1. Repository Features -[cols="2,2,2,3",options="header"] -|=== -|Setting |Type |Default |Description - -|has_issues -|bool -|true -|Enable issue tracker - -|has_wiki -|bool -|false -|Enable wiki - -|has_projects -|bool -|true -|Enable project boards - -|has_downloads -|bool -|false -|Enable downloads section -|=== - -=== 2. Merge Settings -[cols="2,2,2,3",options="header"] -|=== -|Setting |Type |Default |Description - -|allow_squash_merge -|bool -|true -|Allow squash merging - -|allow_merge_commit -|bool -|true -|Allow merge commits - -|allow_rebase_merge -|bool -|false -|Allow rebase merging - -|delete_branch_on_merge -|bool -|true -|Auto-delete head branches - -|allow_auto_merge -|bool -|false -|Enable auto-merge -|=== - -=== 3. Security Settings -[cols="2,2,2,3",options="header"] -|=== -|Setting |Type |Default |Description - -|visibility -|enum -|public -|Repository visibility - -|dependabot_alerts -|bool -|true -|Dependabot vulnerability alerts - -|dependabot_security_updates -|bool -|true -|Automated security updates - -|secret_scanning -|bool -|true -|Secret scanning - -|secret_scanning_push_protection -|bool -|true -|Block pushes with secrets - -|private_vulnerability_reporting -|bool -|true -|Security advisories -|=== - -=== 4. Branch Protection -[cols="2,2,2,3",options="header"] -|=== -|Setting |Type |Default |Description - -|required_pull_request_reviews -|int -|2 -|Required approving reviews - -|dismiss_stale_reviews -|bool -|true -|Dismiss when new commits - -|require_code_owner_reviews -|bool -|false -|Require CODEOWNERS approval - -|required_status_checks -|[]string -|[] -|Required CI checks - -|enforce_admins -|bool -|false -|Apply rules to admins - -|require_signed_commits -|bool -|false -|Require GPG signatures - -|require_linear_history -|bool -|true -|No merge commits - -|allow_force_pushes -|bool -|false -|Allow force push - -|allow_deletions -|bool -|false -|Allow branch deletion -|=== - -=== 5. Topics/Labels -[cols="2,2,2,3",options="header"] -|=== -|Setting |Type |Default |Description - -|topics -|[]string -|[] -|Repository topics/tags - -|labels -|[]Label -|[] -|Issue labels -|=== - -== Command Interface - -=== Basic Usage - -```bash -# Set single setting -repo-batcher github-settings \ - --targets "@all-repos" \ - --set has-wiki=false - -# Set multiple settings -repo-batcher github-settings \ - --targets "@all-repos" \ - --set has-wiki=false \ - --set has-downloads=false \ - --set delete-branch-on-merge=true - -# Use config file -repo-batcher github-settings \ - --targets "@all-repos" \ - --config settings.toml -``` - -=== Config File Format - -```toml -# settings.toml - Repository settings template - -[repository] -has_issues = true -has_wiki = false -has_projects = true -has_downloads = false - -[merge] -allow_squash_merge = true -allow_merge_commit = false -allow_rebase_merge = false -delete_branch_on_merge = true -allow_auto_merge = false - -[security] -visibility = "public" -dependabot_alerts = true -dependabot_security_updates = true -secret_scanning = true -secret_scanning_push_protection = true - -[branch_protection] -branch = "main" -required_reviews = 2 -dismiss_stale_reviews = true -require_code_owner_reviews = false -required_status_checks = ["ci", "security-scan", "codeql"] -enforce_admins = false -require_signed_commits = false -require_linear_history = true -allow_force_pushes = false -allow_deletions = false - -[topics] -add = ["mcp-server", "formally-verified", "typescript"] -remove = ["experimental", "deprecated"] -``` - -=== Advanced Usage - -```bash -# Apply different settings to different repo groups -repo-batcher github-settings \ - --targets "@pattern:*-mcp" \ - --config mcp-server-settings.toml - -repo-batcher github-settings \ - --targets "@pattern:*-lib" \ - --config library-settings.toml - -# Dry-run to preview changes -repo-batcher github-settings \ - --targets "@all-repos" \ - --config settings.toml \ - --dry-run - -# Semi-auto mode (pause on conflicts) -repo-batcher github-settings \ - --targets "@all-repos" \ - --config settings.toml \ - --mode semi-auto -``` - -== Safety Features - -=== 1. Pre-flight Validation - -```v -fn validate_settings(settings GitHubSettings) !{ - // Check for dangerous combinations - if settings.visibility == 'public' && - settings.secret_scanning == false { - return error('Public repos must have secret scanning enabled') - } - - // Validate branch exists - if bp := settings.branch_protection { - if !branch_exists(bp.branch) { - return error('Branch ${bp.branch} does not exist') - } - } - - // Validate status checks - if bp := settings.branch_protection { - for check in bp.required_status_checks { - if !check_exists(check) { - return error('Status check ${check} not found') - } - } - } -} -``` - -=== 2. Rollback Support - -```v -struct SettingsBackup { - repo_name string - timestamp i64 - old_settings GitHubSettings -} - -fn backup_settings(repo string) !SettingsBackup { - current := fetch_current_settings(repo)! - return SettingsBackup{ - repo_name: repo - timestamp: time.now().unix - old_settings: current - } -} - -fn rollback_settings(backup SettingsBackup) !{ - apply_settings(backup.repo_name, backup.old_settings)! -} -``` - -=== 3. Idempotency - -```v -// Only update settings that differ from current -fn apply_settings_idempotent(repo string, new_settings GitHubSettings) !{ - current := fetch_current_settings(repo)! - - diff := compute_diff(current, new_settings) - - if diff.is_empty() { - println('${repo}: No changes needed (already correct)') - return - } - - apply_diff(repo, diff)! -} -``` - -== Integration with Existing Operations - -The github-settings operation complements existing operations: - -```bash -# 1. Update workflows with SHA pinning -repo-batcher workflow-update --targets "@all-repos" - -# 2. Enable branch protection for those workflows -repo-batcher github-settings \ - --targets "@all-repos" \ - --set branch-protection=true \ - --require-status-checks="workflow-linter,security-scan" - -# 3. Audit SPDX compliance -repo-batcher spdx-audit --targets "@all-repos" - -# 4. Update licenses where needed -repo-batcher license-update \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "@non-compliant" - -# 5. Sync changes -repo-batcher git-sync \ - --parallel 8 \ - --commit-message "chore: security hardening" -``` - -== Implementation Priority - -=== Phase 1: Basic Settings (Week 1) -- [x] Design architecture -- [ ] Implement basic repo settings (has_issues, has_wiki, etc.) -- [ ] Implement merge settings -- [ ] Config file parser (TOML) -- [ ] Dry-run support - -=== Phase 2: Security Settings (Week 2) -- [ ] Visibility management -- [ ] Dependabot settings -- [ ] Secret scanning settings -- [ ] Pre-flight validation - -=== Phase 3: Branch Protection (Week 3) -- [ ] Basic branch protection -- [ ] Required reviews -- [ ] Required status checks -- [ ] Signature requirements - -=== Phase 4: Advanced Features (Week 4) -- [ ] Topics management -- [ ] Rollback support -- [ ] Idempotency checks -- [ ] Integration tests - -== API Endpoints Used - -``` -PATCH /repos/{owner}/{repo} - - Update repository settings - -PUT /repos/{owner}/{repo}/topics - - Set repository topics - -PUT /repos/{owner}/{repo}/branches/{branch}/protection - - Configure branch protection - -PATCH /repos/{owner}/{repo}/branches/{branch}/protection/required_pull_request_reviews - - Update review requirements - -PUT /repos/{owner}/{repo}/branches/{branch}/protection/required_status_checks - - Set required checks -``` - -== Example Workflows - -=== Workflow 1: Secure All Repositories - -```bash -# 1. Enable security features -repo-batcher github-settings \ - --targets "@all-repos" \ - --set dependabot-alerts=true \ - --set secret-scanning=true \ - --set secret-scanning-push-protection=true - -# 2. Protect main branch -repo-batcher github-settings \ - --targets "@all-repos" \ - --branch main \ - --require-reviews=2 \ - --require-status-checks="ci,security" \ - --require-signed-commits=true - -# 3. Verify with audit -repo-batcher github-settings \ - --targets "@all-repos" \ - --audit \ - --report security-audit.txt -``` - -=== Workflow 2: Standardize MCP Servers - -```bash -# Create MCP server standard settings -cat > mcp-settings.toml <>) - -| `--dry-run`, `-d` -| bool -| No -| Preview changes without executing (default: false) - -| `--backup`, `-b` -| bool -| No -| Create backups before changes (default: true) -|=== - -**What it updates**: -- LICENSE file in repository root -- SPDX headers in source files: - - `**/*.rs` (Rust) - - `**/*.v` (V) - - `**/*.dats` (ATS2) - - `**/*.idr` (Idris2) - - `**/*.zig` (Zig) - - `**/*.scm` (Scheme) - - `**/*.toml`, `**/*.yml`, `**/*.yaml` (Config) - -**Example output**: ----- -License Update Operation - Old: AGPL-3.0 - New: PMPL-1.0-or-later - Targets: @all-repos - Dry Run: false - Backup: true - -Executing formally verified license update... - -=== Batch Operation Results === -Success: 487 -Failure: 3 -Skipped: 12 -Total: 502 ----- - -=== 2. Git Batch Sync - -**Purpose**: Perform `git add . && git commit && git push` across multiple repositories in parallel - -**Origin**: Ported from `sync_repos.sh` with formal verification - -**Safety Guarantees**: -- Validates all targets are git repositories -- Checks for conflicts before push -- Reports detailed success/failure per repo -- Parallel execution with configurable job count - -**Command**: -[source,bash] ----- -repo-batcher git-sync \ - --parallel 4 \ - --depth 2 \ - --commit-message "chore: batch update" \ - --dry-run # Remove to execute ----- - -**Parameters**: - -|=== -| Flag | Type | Required | Default | Description - -| `--parallel`, `-p` -| int -| No -| 4 -| Number of parallel jobs - -| `--depth`, `-D` -| int -| No -| 2 -| Max depth for repository search - -| `--commit-message`, `-m` -| string -| No -| "chore: batch update" -| Git commit message - -| `--dry-run`, `-d` -| bool -| No -| false -| Preview without executing -|=== - -**Repository Discovery**: - -The operation scans for git repositories using: -[source,bash] ----- -find ~/Documents/hyperpolymath-repos \ - -maxdepth 2 \ - -name ".git" \ - -type d ----- - -This matches the behavior of `sync_repos.sh`. - -**Example output**: ----- -Git Batch Sync Operation (ported from sync_repos.sh) - Parallel Jobs: 4 - Max Depth: 2 - Commit Message: chore: batch update - Dry Run: false - -Executing formally verified git-sync... -Scanning for repositories (find . -maxdepth 2 -name ".git")... - ---- Sync Summary --- -Status | Count ---------------------|---------- -Fully Succeeded | 450 -Failed/Partial | 5 -Skipped | 47 -Total Processed | 502 - ---- Failure Details --- - - Failed to push: repo-batcher (check remote, auth, conflicts) - - Failed to push: experimental-lang (check remote, auth, conflicts) ----- - -**Performance**: - -Based on sync_repos.sh benchmarks: - -|=== -| Parallel Jobs | Throughput | Notes - -| 1 (serial) -| ~30 repos/min -| Baseline - -| 4 -| ~120 repos/min -| **Recommended** - -| 8 -| ~200 repos/min -| For fast CPUs/networks - -| 16+ -| Diminishing returns -| May hit rate limits -|=== - -=== 3. File Replace - -**Purpose**: Replace files across repositories with validation - -**Safety Guarantees**: -- Validates replacement file exists -- Backup required -- No circular replacements (proven by ATS2) -- Pattern matching with glob support - -**Command**: -[source,bash] ----- -repo-batcher file-replace \ - --pattern ".github/workflows/old-ci.yml" \ - --replacement templates/new-ci.yml \ - --targets "@rsr-repos" \ - --backup \ - --dry-run ----- - -**Parameters**: - -|=== -| Flag | Type | Required | Description - -| `--pattern`, `-p` -| string -| Yes -| File pattern to match (glob syntax) - -| `--replacement`, `-r` -| string -| Yes -| Path to replacement file - -| `--targets`, `-t` -| string -| Yes -| Target repositories - -| `--dry-run`, `-d` -| bool -| No -| Preview changes (default: false) - -| `--backup`, `-b` -| bool -| No -| Create backups (default: true) -|=== - -**Pattern Examples**: - -[source,bash] ----- -# Single file ---pattern "LICENSE" - -# Workflow files ---pattern ".github/workflows/*.yml" - -# All Rust files ---pattern "**/*.rs" - -# Specific filename anywhere ---pattern "**/config.toml" ----- - -**Example output**: ----- -File Replace Operation - Pattern: .github/workflows/old-ci.yml - Replacement: templates/new-ci.yml - Targets: @rsr-repos - Dry Run: false - Backup: true - -Executing formally verified file replace... - -=== Batch Operation Results === -Success: 123 -Failure: 0 -Skipped: 379 -Total: 502 - -Message: Files replaced successfully ----- - -== Target Selection - -Multiple ways to specify target repositories: - -=== Explicit List - -[source,bash] ----- ---targets "repo1,repo2,repo3" ----- - -=== All Repositories - -[source,bash] ----- ---targets "@all-repos" ----- - -Scans: `~/Documents/hyperpolymath-repos/` - -=== Pattern Matching - -[source,bash] ----- ---targets "@rsr-*" # All RSR-compliant repos ---targets "@lithoglyph-*" # All lithoglyph repos ----- - -=== From File - -[source,bash] ----- ---targets "$(cat repos.txt)" ----- - -Where `repos.txt` contains one repo path per line. - -=== Directory - -[source,bash] ----- ---targets "/path/to/repos" ----- - -Scans specified directory for git repositories. - -== Operation Modes - -=== Dry Run (Safe Preview) - -**Always start with dry-run to preview changes!** - -[source,bash] ----- -repo-batcher license-update \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" \ - --dry-run ----- - -Output shows what *would* happen without making changes. - -=== Execute Mode - -Remove `--dry-run` to execute for real: - -[source,bash] ----- -repo-batcher license-update \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" \ - --backup # Backups enabled ----- - -=== Interactive Mode (Future) - -Will prompt before each repository: - -[source,bash] ----- -repo-batcher license-update \ - --old "MIT" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" \ - --interactive ----- - -== Watch Folder Mode - -Drop operation files for automatic execution. - -=== Starting Watch Daemon - -[source,bash] ----- -# Start in background -repo-batcher watch & - -# Or use Justfile -just watch ----- - -=== Creating Operation Files - -**License Update** (`watch/update-licenses.toml`): - -[source,toml] ----- -operation = "license-update" -old_license = "AGPL-3.0" -new_license = "PMPL-1.0-or-later" -targets = "@all-repos" -dry_run = false -backup = true ----- - -**Git Sync** (`watch/sync-all.toml`): - -[source,toml] ----- -operation = "git-sync" -parallel_jobs = 4 -max_depth = 2 -commit_message = "chore: automated batch update" -dry_run = false ----- - -**File Replace** (`watch/update-workflows.toml`): - -[source,toml] ----- -operation = "file-replace" -pattern = ".github/workflows/old-ci.yml" -replacement = "templates/new-ci.yml" -targets = "@rsr-repos" -dry_run = false -backup = true ----- - -=== Execution - -Operations execute automatically when files appear in watch folder: - -[source,bash] ----- -# Drop file -cp templates/license-update.example.toml \ - ~/.config/repo-batcher/watch/ - -# Check logs -tail -f ~/.local/share/repo-batcher/logs/latest.log ----- - -== Rollback - -Undo operations using backup information. - -=== Rollback Last Operation - -[source,bash] ----- -repo-batcher rollback --last ----- - -=== Rollback Specific Operation - -[source,bash] ----- -# List operations -ls ~/.local/share/repo-batcher/logs/ - -# Rollback specific -repo-batcher rollback --log-id 20260206-143022 ----- - -== Safety Features - -All operations include: - -- **Compile-time validation** (ATS2 proofs) -- **Dry-run preview** (no filesystem changes) -- **Automatic backups** (before destructive operations) -- **Rollback support** (undo on failure) -- **Atomic per-repo** (all changes or none) -- **Detailed logging** (audit trail) - -== Performance Tips - -=== Parallel Jobs - -Adjust based on your hardware: - -[source,bash] ----- -# Fast CPU + fast network -repo-batcher git-sync --parallel 8 - -# Slower system -repo-batcher git-sync --parallel 2 - -# Default (recommended) -repo-batcher git-sync --parallel 4 ----- - -=== Max Depth - -Limit repository search depth: - -[source,bash] ----- -# Shallow (faster, misses nested repos) -repo-batcher git-sync --depth 1 - -# Default (good balance) -repo-batcher git-sync --depth 2 - -# Deep (slow, finds all repos) -repo-batcher git-sync --depth 5 ----- - -== Troubleshooting - -=== Operation Fails - -1. Check logs: `~/.local/share/repo-batcher/logs/latest.log` -2. Run with `--dry-run` to preview -3. Test on single repo first: `--targets "test-repo"` -4. Verify SPDX identifiers: `repo-batcher list-ops` - -=== Git Sync Failures - -Common causes: -- **No remote configured**: Set up remote first -- **Authentication required**: Configure SSH keys or credentials -- **Merge conflicts**: Resolve manually, then retry -- **Network issues**: Check connectivity - -=== License Update Skips Files - -If files are skipped: -- Old license not found in file -- File not matching source patterns -- File already has new license - -Check logs for details. - -== Examples - -=== Mass License Migration - -Replace AGPL-3.0 with PMPL-1.0-or-later across all repos: - -[source,bash] ----- -# Preview first -repo-batcher license-update \ - --old "AGPL-3.0" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" \ - --dry-run - -# Execute -repo-batcher license-update \ - --old "AGPL-3.0" \ - --new "PMPL-1.0-or-later" \ - --targets "@all-repos" \ - --backup ----- - -=== Update Workflow Files - -Replace outdated CI workflow: - -[source,bash] ----- -repo-batcher file-replace \ - --pattern ".github/workflows/ci.yml" \ - --replacement templates/new-ci.yml \ - --targets "@all-repos" \ - --backup ----- - -=== Daily Sync Routine - -Add to cron/systemd timer: - -[source,bash] ----- -#!/bin/bash -# ~/.local/bin/daily-repo-sync.sh - -cd ~/Documents/hyperpolymath-repos -repo-batcher git-sync \ - --parallel 4 \ - --depth 2 \ - --commit-message "chore: daily automated sync" - -# Email report (if failures) -if [ $? -ne 0 ]; then - mail -s "repo-batcher: sync failures" user@example.com \ - < ~/.local/share/repo-batcher/logs/latest.log -fi ----- - -== Further Reading - -- link:../README.adoc[README] - Project overview -- link:ARCHITECTURE.adoc[Architecture] - Design and implementation -- link:../GETTING-STARTED.adoc[Getting Started] - Setup and first steps -- link:../STATE.scm[State File] - Current project status diff --git a/scaffoldia/repo-batcher/docs/SAFETY-FEATURES.adoc b/scaffoldia/repo-batcher/docs/SAFETY-FEATURES.adoc deleted file mode 100644 index 6dfd4d41..00000000 --- a/scaffoldia/repo-batcher/docs/SAFETY-FEATURES.adoc +++ /dev/null @@ -1,665 +0,0 @@ -= Safety Features -:toc: -:toclevels: 3 -:version: 1.0.0 -:date: 2026-02-06 - -== Overview - -repo-batcher operates on hundreds of repositories simultaneously. **Safety is paramount.** This document describes the comprehensive safety system that protects your repositories from accidents. - -== Safety Architecture - -=== Multi-Layer Protection - -[source] ----- -┌─────────────────────────────────────┐ -│ Layer 1: Configuration │ -│ .safety.toml settings │ -└─────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────┐ -│ Layer 2: Pre-flight Validation │ -│ Check permissions, git status │ -└─────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────┐ -│ Layer 3: User Confirmation │ -│ Prompt before bulk operations │ -└─────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────┐ -│ Layer 4: Rate Limiting │ -│ Prevent API hammering │ -└─────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────┐ -│ Layer 5: Backup & Rollback │ -│ Create backups, enable rollback │ -└─────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────┐ -│ Layer 6: Audit Logging │ -│ Track all operations │ -└─────────────────────────────────────┘ ----- - -== Safety Levels - -=== Paranoid Mode (Maximum Safety) - -**When to use:** Production systems, critical repositories - -[source,bash] ----- -# In .safety.toml -level = "paranoid" ----- - -**Characteristics:** -* Confirms every batch operation -* Maximum 50 repos per operation -* Requires confirmation if >5 repos -* 500ms rate limit between operations -* Always creates backups -* Full audit logging - -**Example:** -[source,bash] ----- -$ repo-batcher community-setup --targets "@all-repos" - -=== OPERATION CONFIRMATION REQUIRED === -Operation: Deploy community health files -Type: local_changes -Target repos: 565 -Dry run: false - -⚠ī¸ WARNINGS: - - Paranoid mode: Confirm batch operation - - Operation targets 565 repositories - -First 5 repositories: - - repo-1 - - repo-2 - - repo-3 - - repo-4 - - repo-5 - ... and 560 more - -Proceed with this operation? [y/N]: ----- - -=== Strict Mode (Recommended) - -**When to use:** Normal development work - -[source,bash] ----- -# In .safety.toml -level = "strict" ----- - -**Characteristics:** -* Requires dry-run first for remote operations -* Maximum 100 repos per operation -* Confirms if >10 repos -* 100ms rate limit -* Creates backups -* Audit logging enabled - -=== Relaxed Mode - -**When to use:** Personal projects, testing - -[source,bash] ----- -# In .safety.toml -level = "relaxed" ----- - -**Characteristics:** -* Minimal safety checks -* Higher repo limits -* Fewer confirmation prompts -* Still logs operations - -=== Disabled Mode (⚠ī¸ DANGEROUS) - -**When to use:** Automated scripts (use with extreme caution) - -[source,bash] ----- -# In .safety.toml -level = "disabled" ----- - -**⚠ī¸ WARNING:** No safety checks! Only use in controlled environments. - -== Pre-flight Validation - -=== Validation Rules - -==== 1. Git Repository Check - -Verifies all paths are valid git repositories. - -**Severity:** ERROR (blocks operation) - -[source] ----- -❌ VALIDATION ERRORS (operation blocked): - [git_repository_check] Not a git repository: /path/to/dir ----- - -==== 2. Write Permission Check - -Verifies write access to all repositories. - -**Severity:** ERROR (blocks operation) - -[source] ----- -❌ VALIDATION ERRORS (operation blocked): - [write_permission_check] No write permission: /path/to/repo ----- - -==== 3. Uncommitted Changes Check - -Warns if repositories have uncommitted changes. - -**Severity:** WARNING (can proceed) - -[source] ----- -⚠ī¸ VALIDATION WARNINGS: - [uncommitted_changes_check] Uncommitted changes in 12 repositories ----- - -==== 4. Remote Exists Check - -For push operations, verifies remote is configured. - -**Severity:** WARNING - -[source] ----- -⚠ī¸ VALIDATION WARNINGS: - [remote_exists_check] No remote configured in 3 repositories ----- - -==== 5. Disk Space Check - -Warns if disk usage exceeds 90%. - -**Severity:** WARNING - -[source] ----- -⚠ī¸ VALIDATION WARNINGS: - [disk_space_check] Disk usage at 92% (>90%) ----- - -=== Disabling Validation - -To disable specific rules (not recommended): - -[source,toml] ----- -[validation] -enabled = true -rules = [ - "git_repository_check", # Keep essential checks - "write_permission_check", # Keep essential checks - # "uncommitted_changes_check", # Disabled - # "remote_exists_check", # Disabled - # "disk_space_check", # Disabled -] ----- - -== Exclusion Lists - -=== Pattern-Based Exclusions - -Never operate on paths matching these patterns: - -[source,toml] ----- -[exclusions] -patterns = [ - ".git", # Git internals - "node_modules", # Dependencies - "vendor", # Third-party code - ".env", # Environment variables - ".secrets", # Secret files - "*.pem", # Certificates - "*.key", # Private keys - "*.p12", # PKCS12 files - ".aws", # AWS credentials - ".ssh", # SSH keys -] ----- - -=== Repository-Specific Exclusions - -Protect critical repositories by name: - -[source,toml] ----- -[exclusions] -repositories = [ - "prod-api", - "customer-database", - "payment-service", - "auth-service", -] ----- - -**Result:** -[source] ----- -❌ VALIDATION ERRORS (operation blocked): - Some repos match exclusion list: ["prod-api", "payment-service"] ----- - -== Confirmation Prompts - -=== Automatic Triggers - -Confirmation required when: - -1. **Repo count exceeds threshold** (default: 10) -2. **Destructive operations** (delete, force-push) -3. **Remote changes** without dry-run -4. **Paranoid mode** (always) - -=== Confirmation Dialog - -[source] ----- -=== OPERATION CONFIRMATION REQUIRED === -Operation: Update licenses across repositories -Type: local_changes -Target repos: 45 -Dry run: false - -⚠ī¸ WARNINGS: - - Operation targets 45 repositories - - No dry-run performed - -Repositories: - - repo-1 - - repo-2 - ... and 43 more - -Proceed with this operation? [y/N]: _ ----- - -**Type 'y' or 'yes' to proceed. Any other input aborts.** - -=== Bypassing Confirmation - -For automation (use carefully): - -[source,bash] ----- -# Use --force flag (requires safety.level != "paranoid") -repo-batcher community-setup \ - --targets "@all-repos" \ - --force - -# Or pipe 'yes' -echo "y" | repo-batcher community-setup --targets "@all-repos" ----- - -== Rate Limiting - -=== Purpose - -Prevents hammering GitHub API and avoids rate limits. - -=== Configuration - -[source,toml] ----- -[safety] -rate_limit_ms = 100 # Wait 100ms between operations ----- - -**Values:** -* `0` = No rate limiting (not recommended) -* `100` = Standard (10 ops/second) -* `500` = Conservative (2 ops/second) -* `1000` = Very safe (1 op/second) - -=== How It Works - -[source] ----- -Operation 1 → Wait 100ms → Operation 2 → Wait 100ms → Operation 3 ----- - -For 565 repositories at 100ms: -* Total delay: 56.5 seconds -* Operations still complete quickly - -== Backup & Rollback - -=== Automatic Backups - -Before any destructive operation: - -[source,toml] ----- -[safety] -backup_before = true ----- - -**Backup location:** -[source] ----- -~/.local/share/repo-batcher/backups/ - └── operation-20260206-143022/ - ├── repo-1/ - │ └── file.txt.backup - ├── repo-2/ - │ └── file.txt.backup - └── manifest.json ----- - -=== Rollback - -Undo the last operation: - -[source,bash] ----- -# Rollback last operation -repo-batcher rollback --last - -# Rollback specific operation -repo-batcher rollback --log-id operation-20260206-143022 - -# List available rollbacks -repo-batcher rollback ----- - -== Checkpoint & Resume - -=== Automatic Checkpoints - -For long operations, automatic checkpoints enable resume: - -[source,toml] ----- -[recovery] -enable_checkpoints = true -checkpoint_every = 10 # Checkpoint every 10 repos -checkpoint_retention_days = 7 ----- - -=== How It Works - -[source] ----- -Processing 565 repositories: -[10/565] ✓ Checkpoint saved -[20/565] ✓ Checkpoint saved -[30/565] ✗ CRASH! - -Next run: -Detected incomplete operation. Resume? [Y/n] y -Resuming from checkpoint: [30/565] ----- - -=== Manual Resume - -[source,bash] ----- -# Check for incomplete operations -repo-batcher resume --list - -# Resume specific operation -repo-batcher resume --operation-id abc123 ----- - -== Audit Logging - -=== Log Location - -[source,bash] ----- -~/.local/share/repo-batcher/audit.log ----- - -=== Log Format - -[source] ----- -2026-02-06 14:30:22 | local_changes | 45 repos | community-setup | SUCCESS -2026-02-06 14:35:10 | remote_changes | 12 repos | git-sync | SUCCESS -2026-02-06 14:40:05 | read_only | 565 repos | spdx-audit | SUCCESS ----- - -=== Viewing Logs - -[source,bash] ----- -# View recent operations -tail -20 ~/.local/share/repo-batcher/audit.log - -# Search for failures -grep FAILED ~/.local/share/repo-batcher/audit.log - -# View operations on specific date -grep "2026-02-06" ~/.local/share/repo-batcher/audit.log ----- - -== Operation Types - -Safety checks vary by operation type: - -[cols="2,2,3",options="header"] -|=== -|Type |Examples |Safety Measures - -|**read_only** -|spdx-audit, list -|Minimal checks, no backups - -|**local_changes** -|community-setup, templates-setup -|Validation, backups, rate limiting - -|**remote_changes** -|git-sync, wiki-setup -|All safety measures, requires dry-run first - -|**destructive** -|(none currently) -|Maximum safety, requires --force flag -|=== - -== Best Practices - -=== 1. Always Dry-Run First - -[source,bash] ----- -# Step 1: Dry-run -repo-batcher community-setup --targets "@all-repos" --dry-run - -# Step 2: Review output -# Step 3: Execute -repo-batcher community-setup --targets "@all-repos" ----- - -=== 2. Start Small - -Test on subset before full rollout: - -[source,bash] ----- -# Test on 3 repos -repo-batcher community-setup --targets "repo1,repo2,repo3" - -# If successful, expand -repo-batcher community-setup --targets "@pattern:test-*" - -# Finally, all repos -repo-batcher community-setup --targets "@all-repos" ----- - -=== 3. Use Patterns Carefully - -[source,bash] ----- -# Good: Specific pattern ---targets "@pattern:poly-*-mcp" - -# Risky: Too broad ---targets "@all-repos" - -# Better: Dry-run first ---targets "@all-repos" --dry-run ----- - -=== 4. Monitor Audit Logs - -[source,bash] ----- -# Watch logs in real-time -tail -f ~/.local/share/repo-batcher/audit.log - -# Check for errors after operations -grep -E "ERROR|FAILED" ~/.local/share/repo-batcher/audit.log ----- - -=== 5. Keep Backups - -[source,bash] ----- -# Verify backup creation -ls -la ~/.local/share/repo-batcher/backups/ - -# Test rollback on non-critical repo -repo-batcher rollback --last ----- - -== Configuration Examples - -=== Maximum Safety (Production) - -[source,toml] ----- -[safety] -level = "paranoid" -require_dry_run = true -max_repos = 25 -confirm_threshold = 3 -rate_limit_ms = 1000 -backup_before = true -audit_log = true - -[exclusions] -repositories = [ - "prod-api", - "prod-web", - "prod-db", -] ----- - -=== Balanced (Development) - -[source,toml] ----- -[safety] -level = "strict" -require_dry_run = true -max_repos = 100 -confirm_threshold = 10 -rate_limit_ms = 100 -backup_before = true -audit_log = true ----- - -=== Fast (Testing/Personal) - -[source,toml] ----- -[safety] -level = "relaxed" -require_dry_run = false -max_repos = 200 -confirm_threshold = 50 -rate_limit_ms = 50 -backup_before = true -audit_log = true ----- - -== Emergency Procedures - -=== If Something Goes Wrong - -1. **Stop immediately:** - [source,bash] - ---- - Ctrl+C - ---- - -2. **Check what changed:** - [source,bash] - ---- - tail -50 ~/.local/share/repo-batcher/audit.log - ---- - -3. **Rollback if needed:** - [source,bash] - ---- - repo-batcher rollback --last - ---- - -4. **Verify rollback:** - [source,bash] - ---- - # Check git status in affected repos - cd ~/Documents/hyperpolymath-repos/repo-name - git status - git diff - ---- - -=== Recovering from Crashes - -[source,bash] ----- -# Check for incomplete operations -repo-batcher resume --list - -# Resume the operation -repo-batcher resume --operation-id abc123 - -# Or start fresh -rm -rf ~/.local/share/repo-batcher/checkpoints/abc123 ----- - -== Summary - -**Safety Features Checklist:** - -* ✅ Safety levels (paranoid/strict/relaxed/disabled) -* ✅ Pre-flight validation (5 rules) -* ✅ Exclusion lists (patterns + specific repos) -* ✅ User confirmation prompts -* ✅ Rate limiting (prevent API abuse) -* ✅ Automatic backups -* ✅ Rollback capability -* ✅ Checkpoint & resume -* ✅ Audit logging -* ✅ Dry-run enforcement -* ✅ Repository count limits - -**Recommended Configuration:** - -For most users, **strict mode with dry-run** provides the best balance of safety and convenience. - -**Remember:** When in doubt, dry-run first! 🛡ī¸ diff --git a/scaffoldia/repo-batcher/docs/WIKI-AND-COMMUNITY-OPERATIONS.adoc b/scaffoldia/repo-batcher/docs/WIKI-AND-COMMUNITY-OPERATIONS.adoc deleted file mode 100644 index 6de76af4..00000000 --- a/scaffoldia/repo-batcher/docs/WIKI-AND-COMMUNITY-OPERATIONS.adoc +++ /dev/null @@ -1,570 +0,0 @@ -= Wiki and Community Health Operations -:toc: -:toclevels: 3 -:version: 1.0.0 -:date: 2026-02-06 - -== Overview - -Two new operations for repository standardization: - -1. **wiki-setup** - Initialize wikis with first page (solves GitHub automation blocker) -2. **community-setup** - Deploy community health files in bulk - -== The Wiki Problem - -=== Why Wikis Block Automation - -GitHub has an architectural quirk: - -[source,bash] ----- -1. Enable wiki in settings ✅ Can do via API -2. First page creation ❌ Blocks all automation! -3. Everything else ✅ Can automate after step 2 ----- - -**The Issue:** - -* `has_wiki=true` just enables the "Wiki" tab -* The wiki git repo (`repo.wiki.git`) only exists after first page creation -* Until then, there's no repo to clone/edit -* LLMs and automation tools cannot access the wiki - -**Our Solution:** - -[source,bash] ----- -repo-batcher wiki-setup \ - --targets "@all-repos" \ - --dry-run ----- - -This operation: - -1. Enables wiki feature via API -2. Initializes local git repo -3. Creates `Home.md` with content -4. Pushes to `repo.wiki.git` (creates it!) -5. Now automation works! ✅ - -== Wiki Setup Operation - -=== Basic Usage - -[source,bash] ----- -# Initialize all wikis with default Home page -repo-batcher wiki-setup \ - --targets "@all-repos" \ - --dry-run - -# Use custom Home template -repo-batcher wiki-setup \ - --targets "@pattern:poly-*-mcp" \ - --home-template ~/templates/wiki-home.md ----- - -=== Default Home.md Content - -[source,markdown] ----- -# {repo-name} Wiki - -Welcome to the {repo-name} wiki! - -## Getting Started - -This wiki was automatically initialized by repo-batcher. - -## Contents - -- [Home](Home) - -## Contributing - -Feel free to contribute to this wiki by editing pages or creating new ones. ----- - -=== Custom Templates - -Create your own `Home.md` template: - -[source,bash] ----- -cat > ~/templates/mcp-wiki-home.md <<'EOF' -# MCP Server Wiki - -## Installation - -```bash -npm install @hyperpolymath/{repo-name} -``` - -## Configuration - -See [Configuration](Configuration) for details. - -## API Reference - -- [API Documentation](API) -- [Examples](Examples) - -## Support - -For issues, see [GitHub Issues](https://github.com/hyperpolymath/{repo-name}/issues). -EOF - -repo-batcher wiki-setup \ - --targets "@pattern:poly-*-mcp" \ - --home-template ~/templates/mcp-wiki-home.md ----- - -=== What Gets Created - -[cols="2,3,3",options="header"] -|=== -|Action |What Happens |Result - -|Enable Feature -|`PATCH /repos/{owner}/{repo}` with `has_wiki=true` -|Wiki tab appears in GitHub UI - -|Initialize Repo -|Creates temp git repo, adds `Home.md`, pushes -|`repo.wiki.git` now exists - -|Enable Automation -|LLMs can now `git clone repo.wiki.git` -|Full automation possible! ✅ -|=== - -== Community Health Files Operation - -=== What Are Community Health Files? - -GitHub recognizes these special files: - -[cols="2,3",options="header"] -|=== -|File |Purpose - -|`CODE_OF_CONDUCT.md` -|Community behavior standards - -|`CONTRIBUTING.md` -|How to contribute - -|`SECURITY.md` -|Security policy & reporting - -|`SUPPORT.md` -|How to get help - -|`FUNDING.yml` -|Sponsorship links -|=== - -These files: - -* Show up in special GitHub UI locations -* Appear when opening issues/PRs -* Signal project professionalism -* Required for GitHub verified badges - -=== Basic Usage - -[source,bash] ----- -# Deploy default community files to all repos -repo-batcher community-setup \ - --targets "@all-repos" \ - --dry-run - -# Use custom templates -repo-batcher community-setup \ - --targets "@pattern:poly-*-mcp" \ - --template-dir ~/templates/community/ ----- - -=== Default Templates - -==== CODE_OF_CONDUCT.md - -Based on Contributor Covenant 2.0: - -* Pledge for harassment-free environment -* Community standards -* Enforcement guidelines - -==== CONTRIBUTING.md - -Comprehensive contribution guide: - -* Getting started (fork, branch, commit, PR) -* Code style guidelines -* Bug reporting process -* Feature request process -* Pull request checklist - -==== SECURITY.md - -Security policy: - -* Supported versions table -* Vulnerability reporting process -* Response timeline commitments -* Security best practices - -==== SUPPORT.md - -Support resources: - -* Documentation links -* Where to ask questions -* Bug reporting guidelines -* Feature request process -* Response time expectations - -==== FUNDING.yml - -Sponsorship configuration: - -[source,yaml] ----- -github: hyperpolymath -# patreon: your-username -# open_collective: your-project ----- - -=== Custom Templates - -Create your own community files: - -[source,bash] ----- -mkdir -p ~/templates/community - -# Custom CODE_OF_CONDUCT.md -cat > ~/templates/community/CODE_OF_CONDUCT.md <<'EOF' -# Code of Conduct - -## Our Standards - -- Be respectful -- Be constructive -- Be collaborative - -## Enforcement - -Contact maintainers for issues. -EOF - -# Custom CONTRIBUTING.md -cat > ~/templates/community/CONTRIBUTING.md <<'EOF' -# Contributing - -1. Fork the repo -2. Create feature branch -3. Make changes -4. Submit PR - -See DEVELOPMENT.md for details. -EOF - -# Deploy to all MCP servers -repo-batcher community-setup \ - --targets "@pattern:poly-*-mcp" \ - --template-dir ~/templates/community/ ----- - -=== What Gets Created - -[cols="2,3,2",options="header"] -|=== -|File |Location |Size - -|CODE_OF_CONDUCT.md -|`.github/CODE_OF_CONDUCT.md` -|~1.5KB - -|CONTRIBUTING.md -|`.github/CONTRIBUTING.md` -|~2KB - -|SECURITY.md -|`.github/SECURITY.md` -|~1.5KB - -|SUPPORT.md -|`.github/SUPPORT.md` -|~1KB - -|FUNDING.yml -|`.github/FUNDING.yml` -|~200B -|=== - -== Integration with Other Operations - -=== Workflow: Bootstrap New Repository - -[source,bash] ----- -# 1. Create repository -gh repo create hyperpolymath/new-project --public - -# 2. Clone template -git clone https://github.com/hyperpolymath/rsr-template-repo new-project - -# 3. Deploy community files -repo-batcher community-setup \ - --targets "new-project" \ - --template-dir ~/templates/community/ - -# 4. Initialize wiki -repo-batcher wiki-setup \ - --targets "new-project" \ - --home-template ~/templates/wiki-home.md - -# 5. Deploy workflows -repo-batcher file-replace \ - --pattern ".github/workflows/*.yml" \ - --replacement ~/templates/workflows/ \ - --targets "new-project" - -# 6. Configure settings -repo-batcher github-settings \ - --targets "new-project" \ - --config ~/templates/repo-settings.toml - -# 7. Commit and push -cd new-project -git add . -git commit -m "feat: initialize repository with standards" -git push ----- - -=== Workflow: Standardize Existing Repos - -[source,bash] ----- -# Deploy community files to all repos without them -repo-batcher community-setup \ - --targets "@all-repos" - -# Initialize wikis for repos that lack them -repo-batcher wiki-setup \ - --targets "@all-repos" - -# Sync everything -repo-batcher git-sync \ - --parallel 8 \ - --commit-message "chore: add community health files and initialize wikis" ----- - -== Command Reference - -=== wiki-setup - -[source,bash] ----- -repo-batcher wiki-setup \ - --targets "@all-repos" \ - [--home-template PATH] \ - [--dry-run] ----- - -**Flags:** - -* `--targets`, `-t` - Target repositories (required) -* `--home-template`, `-h` - Path to Home.md template -* `--dry-run`, `-d` - Preview without executing - -**Examples:** - -[source,bash] ----- -# Default home page -repo-batcher wiki-setup -t "@all-repos" -d - -# Custom template -repo-batcher wiki-setup \ - -t "@pattern:poly-*-mcp" \ - -h ~/templates/wiki-home.md - -# Single repo -repo-batcher wiki-setup -t "hyperpolymath/repo-batcher" ----- - -=== community-setup - -[source,bash] ----- -repo-batcher community-setup \ - --targets "@all-repos" \ - [--template-dir PATH] \ - [--dry-run] ----- - -**Flags:** - -* `--targets`, `-t` - Target repositories (required) -* `--template-dir`, `-T` - Directory with custom templates -* `--dry-run`, `-d` - Preview without executing - -**Examples:** - -[source,bash] ----- -# Default templates -repo-batcher community-setup -t "@all-repos" -d - -# Custom templates -repo-batcher community-setup \ - -t "@pattern:poly-*-mcp" \ - -T ~/templates/community/ - -# Specific repos -repo-batcher community-setup \ - -t "repo1,repo2,repo3" ----- - -== Safety Features - -=== wiki-setup - -* ✅ Checks if wiki already initialized (skips if exists) -* ✅ Enables feature via API first -* ✅ Uses temporary directory (auto-cleanup) -* ✅ Dry-run support -* ✅ Requires gh CLI authentication -* ✅ Reports success/failure per repo - -=== community-setup - -* ✅ Checks if files already exist (reports status) -* ✅ Creates `.github/` directory if missing -* ✅ Skips identical files (no unnecessary changes) -* ✅ Reports created/updated/skipped counts -* ✅ Dry-run support -* ✅ Works with local repositories (no API needed) - -== Performance - -=== wiki-setup - -[cols="2,2,2",options="header"] -|=== -|Repos |Time (Sequential) |Time (Parallel) - -|10 repos -|~2 minutes -|~30 seconds - -|100 repos -|~20 minutes -|~5 minutes - -|565 repos -|~3 hours -|~45 minutes -|=== - -**Why slower than other operations?** - -* Requires git clone/push per repo -* Network latency for GitHub API + git operations -* Wiki repos can be slow to initialize - -=== community-setup - -[cols="2,2,2",options="header"] -|=== -|Repos |Time (Sequential) |Time (Parallel) - -|10 repos -|~5 seconds -|~2 seconds - -|100 repos -|~50 seconds -|~15 seconds - -|565 repos -|~5 minutes -|~90 seconds -|=== - -**Why fast?** - -* Local file operations only -* No network calls -* Minimal disk I/O - -== Troubleshooting - -=== wiki-setup Issues - -**Error: "Failed to push to wiki repo"** - -* Check gh CLI authentication: `gh auth status` -* Verify repository exists: `gh repo view owner/repo` -* Check wiki feature enabled in repo settings - -**Error: "Wiki already exists"** - -* This is normal! Operation skips already-initialized wikis -* Use `--dry-run` to preview which repos need initialization - -**Error: "Permission denied"** - -* Need write access to repository -* Need wiki feature enabled (operation does this automatically) - -=== community-setup Issues - -**Error: "Failed to create .github directory"** - -* Check repository path is correct -* Verify write permissions on repository - -**Files not appearing in GitHub UI** - -* Files must be in `.github/` directory -* Some files show up in specific contexts (e.g., CODE_OF_CONDUCT when opening issues) -* Try refreshing GitHub page or wait ~1 minute for cache - -== Future Enhancements - -=== Planned Features (Week 3-4) - -**wiki-setup:** - -* Deploy additional wiki pages from templates -* Support for wiki sidebar configuration -* Wiki footer customization -* Bulk wiki content updates - -**community-setup:** - -* Issue template deployment (`.github/ISSUE_TEMPLATE/`) -* PR template customization -* GitHub Sponsors configuration -* Custom badges in README - -== Conclusion - -These operations solve two key repository management problems: - -1. **Wiki automation blocker** - Now LLMs and tools can access wikis -2. **Community health standardization** - Deploy professional files in bulk - -Together they enable: - -* ✅ Fully automated repository setup -* ✅ Professional project appearance -* ✅ Community engagement infrastructure -* ✅ GitHub verified badges eligibility - -**Ready to use on 565+ repositories!** 🚀 diff --git a/scaffoldia/repo-batcher/examples/SafeDOMExample.res b/scaffoldia/repo-batcher/examples/SafeDOMExample.res deleted file mode 100644 index 407d5bf3..00000000 --- a/scaffoldia/repo-batcher/examples/SafeDOMExample.res +++ /dev/null @@ -1,109 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later -// Example: Using SafeDOM for formally verified DOM mounting - -open SafeDOM - -// Example 1: Basic mounting with error handling -let mountApp = () => { - mountSafe( - "#app", - "

Hello, World!

Mounted safely with proofs.

", - ~onSuccess=el => { - Console.log("✓ App mounted successfully!") - Console.log("Element:", el) - }, - ~onError=err => { - Console.error("✗ Mount failed:", err) - } - ) -} - -// Example 2: Wait for DOM ready before mounting -let mountWhenDOMReady = () => { - mountWhenReady( - "#app", - "

App Title

", - ~onSuccess=_ => Console.log("✓ Mounted after DOM ready"), - ~onError=err => Console.error("✗ Failed:", err) - ) -} - -// Example 3: Batch mounting (atomic - all or nothing) -let mountMultiple = () => { - let specs = [ - {selector: "#header", html: "

Site Title

"}, - {selector: "#nav", html: ""}, - {selector: "#main", html: "

Content here

"}, - {selector: "#footer", html: "
Š 2026
"} - ] - - switch mountBatch(specs) { - | Ok(elements) => { - Console.log(`✓ Successfully mounted ${Array.length(elements)} elements`) - elements->Array.forEach(el => Console.log(" -", el)) - } - | Error(err) => { - Console.error("✗ Batch mount failed:", err) - Console.error(" (None were mounted - atomic operation)") - } - } -} - -// Example 4: Explicit validation before mounting -let mountWithValidation = () => { - // Validate selector first - switch ProvenSelector.validate("#my-app") { - | Error(e) => Console.error(`Invalid selector: ${e}`) - | Ok(validSelector) => { - // Validate HTML - switch ProvenHTML.validate("
Content
") { - | Error(e) => Console.error(`Invalid HTML: ${e}`) - | Ok(validHtml) => { - // Now mount with proven safety - switch mount(validSelector, validHtml) { - | Mounted(el) => Console.log("✓ Mounted with validated inputs:", el) - | MountPointNotFound(s) => Console.error(`✗ Element not found: ${s}`) - | InvalidSelector(_) => Console.error("Impossible - already validated") - | InvalidHTML(_) => Console.error("Impossible - already validated") - } - } - } - } -} - -// Example 5: Integration with TEA -module MyApp = { - type model = {message: string} - type msg = NoOp - - let init = () => {message: "Hello from TEA"} - let update = (model, _msg) => model - let view = model => `

${model.message}

` -} - -let mountTEAApp = () => { - let model = MyApp.init() - let html = MyApp.view(model) - - mountWhenReady( - "#tea-app", - html, - ~onSuccess=el => { - Console.log("✓ TEA app mounted") - // Set up event handlers, subscriptions here - }, - ~onError=err => Console.error(`✗ TEA mount failed: ${err}`) - ) -} - -// Entry point -let main = () => { - Console.log("SafeDOM Examples") - Console.log("================\n") - - // Choose which example to run - mountWhenDOMReady() // Run on DOM ready -} - -// Auto-execute when module loads -main() diff --git a/scaffoldia/repo-batcher/examples/web-project-deno.json b/scaffoldia/repo-batcher/examples/web-project-deno.json deleted file mode 100644 index eaa39d95..00000000 --- a/scaffoldia/repo-batcher/examples/web-project-deno.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "// NOTE": "Example deno.json for ReScript web projects", - "tasks": { - "build": "deno run -A npm:rescript", - "clean": "deno run -A npm:rescript clean", - "watch": "deno run -A npm:rescript -w", - "serve": "deno run -A jsr:@std/http/file-server .", - "test": "deno test --allow-all" - }, - "imports": { - "rescript": "npm:rescript@^12.1.0", - "@rescript/core": "npm:@rescript/core@^1.6.0", - "safe-dom/": "https://raw.githubusercontent.com/hyperpolymath/rescript-dom-mounter/main/src/", - "proven/": "../proven/bindings/rescript/src/" - }, - "compilerOptions": { - "allowJs": true, - "checkJs": false - } -} diff --git a/scaffoldia/repo-batcher/src/ats2/ffi/c_exports.dats b/scaffoldia/repo-batcher/src/ats2/ffi/c_exports.dats deleted file mode 100644 index 7d7625a8..00000000 --- a/scaffoldia/repo-batcher/src/ats2/ffi/c_exports.dats +++ /dev/null @@ -1,244 +0,0 @@ -(* -** SPDX-License-Identifier: PMPL-1.0-or-later -** -** C FFI Exports -** Exports ATS2 functions to C for V interop -*) - -#include "share/atspre_define.hats" -#include "share/atspre_staload.hats" - -staload "../operations/types.dats" -staload "../operations/license_update.dats" -staload "../operations/git_sync.dats" -staload "../validation/spdx.dats" - -(* ========== Type Marshalling ========== *) - -(* -** C-compatible result structure -*) -typedef c_operation_result = @{ - status = int, // 0=success, 1=failure, 2=skipped - message = string -} - -(* -** C-compatible batch result structure -*) -typedef c_batch_result = @{ - success_count = int, - failure_count = int, - skipped_count = int, - message = string -} - -(* -** Converts operation_result to C-compatible structure -*) -fun operation_result_to_c(result: operation_result): c_operation_result = - case+ result of - | OpSuccess(msg) => @{ status = 0, message = msg } - | OpFailure(msg) => @{ status = 1, message = msg } - | OpSkipped(msg) => @{ status = 2, message = msg } - -(* -** Converts batch_result to C-compatible structure -*) -fun batch_result_to_c(result: batch_result): c_batch_result = @{ - success_count = result.success_count, - failure_count = result.failure_count, - skipped_count = result.skipped_count, - message = "Batch operation completed" -} - -(* ========== Exported C Functions ========== *) - -(* -** Validates SPDX identifier -** Returns 1 if valid, 0 if invalid -*) -extern -fun c_validate_spdx(license: string): int = "ext#" -implement c_validate_spdx(license) = - if is_valid_spdx(license) then 1 else 0 - -(* -** Performs license update operation -** Returns C-compatible batch result -** -** Parameters: -** old_license: Old SPDX identifier -** new_license: New SPDX identifier -** base_dir: Base directory to scan -** max_depth: Maximum recursion depth -** dry_run: 1 for dry-run, 0 for real -** backup: 1 to create backups, 0 to skip -*) -extern -fun c_license_update( - old_license: string, - new_license: string, - base_dir: string, - max_depth: int, - dry_run: int, - backup: int -): c_batch_result = "ext#" - -implement c_license_update( - old_license, - new_license, - base_dir, - max_depth, - dry_run, - backup -) = let - // Validate SPDX identifiers - val old_valid = is_valid_spdx(old_license) - val new_valid = is_valid_spdx(new_license) -in - if ~old_valid then - @{ - success_count = 0, - failure_count = 1, - skipped_count = 0, - message = "Invalid old license: " + old_license - } - else if ~new_valid then - @{ - success_count = 0, - failure_count = 1, - skipped_count = 0, - message = "Invalid new license: " + new_license - } - else let - // Find repositories - val repos = find_git_repos(base_dir, max_depth) - - // Process each repository - val do_dry_run = dry_run = 1 - val do_backup = backup = 1 - val backup_dir = "/tmp/repo-batcher-backups" - - fun process_all( - repos: List0(string), - total_success: int, - total_failure: int, - total_skipped: int - ): @(int, int, int) = - case+ repos of - | list_nil() => @(total_success, total_failure, total_skipped) - | list_cons(repo, rest) => let - val result = update_license_in_repo( - repo, - old_license, - new_license, - do_backup, - backup_dir, - do_dry_run - ) - in - process_all( - rest, - total_success + result.success_count, - total_failure + result.failure_count, - total_skipped + result.skipped_count - ) - end - - val @(s, f, sk) = process_all(repos, 0, 0, 0) - in - @{ - success_count = s, - failure_count = f, - skipped_count = sk, - message = "License update completed" - } - end -end - -(* -** Performs git sync operation -** Returns C-compatible batch result -** -** Parameters: -** base_dir: Base directory to scan -** max_depth: Maximum recursion depth -** commit_msg: Commit message -** parallel_jobs: Number of parallel jobs (handled in V) -** dry_run: 1 for dry-run, 0 for real -*) -extern -fun c_git_sync( - base_dir: string, - max_depth: int, - commit_msg: string, - parallel_jobs: int, - dry_run: int -): c_batch_result = "ext#" - -implement c_git_sync( - base_dir, - max_depth, - commit_msg, - parallel_jobs, - dry_run -) = let - val do_dry_run = dry_run = 1 - val result = execute_git_sync_operation( - base_dir, - max_depth, - commit_msg, - parallel_jobs, - do_dry_run - ) -in - batch_result_to_c(result) -end - -(* -** Performs file replace operation -** Returns C-compatible batch result -** -** Parameters: -** pattern: File pattern to match -** replacement: Path to replacement file -** base_dir: Base directory to scan -** max_depth: Maximum recursion depth -** dry_run: 1 for dry-run, 0 for real -** backup: 1 to create backups, 0 to skip -*) -extern -fun c_file_replace( - pattern: string, - replacement: string, - base_dir: string, - max_depth: int, - dry_run: int, - backup: int -): c_batch_result = "ext#" - -implement c_file_replace( - pattern, - replacement, - base_dir, - max_depth, - dry_run, - backup -) = let - // TODO: Implement file replace operation -in - @{ - success_count = 0, - failure_count = 0, - skipped_count = 0, - message = "File replace not yet implemented" - } -end - -(* -** Gets version string -*) -extern -fun c_get_version(): string = "ext#" -implement c_get_version() = "0.1.0" diff --git a/scaffoldia/repo-batcher/src/ats2/operations/file_replace.dats b/scaffoldia/repo-batcher/src/ats2/operations/file_replace.dats deleted file mode 100644 index 0da8bddf..00000000 --- a/scaffoldia/repo-batcher/src/ats2/operations/file_replace.dats +++ /dev/null @@ -1,343 +0,0 @@ -(* -** SPDX-License-Identifier: PMPL-1.0-or-later -** -** File Replace Operation Implementation -** Replaces files matching patterns across repositories -** Ensures no circular replacements and validates file integrity -*) - -#include "share/atspre_define.hats" -#include "share/atspre_staload.hats" - -staload "libats/libc/SATS/stdio.sats" -staload "libats/libc/SATS/dirent.sats" -staload "libats/libc/SATS/sys/stat.sats" - -staload "./types.dats" -staload "../utils/string_utils.sats" - -(* ========== File Operations ========== *) - -fun read_file_contents(path: string): Option(string) = let - val file = fileref_open_opt(path, file_mode_r) -in - case+ file of - | ~Some_vt(f) => let - val content = fileref_get_file_string(f) - val () = fileref_close(f) - in - Some(content) - end - | ~None_vt() => None() -end - -fun write_file_contents(path: string, content: string): bool = let - val file = fileref_open_opt(path, file_mode_w) -in - case+ file of - | ~Some_vt(f) => let - val () = fileref_put_string(f, content) - val () = fileref_close(f) - in - true - end - | ~None_vt() => false -end - -fun file_exists(path: string): bool = let - val file = fileref_open_opt(path, file_mode_r) -in - case+ file of - | ~Some_vt(f) => let val () = fileref_close(f) in true end - | ~None_vt() => false -end - -fun copy_file(source: string, dest: string): bool = let - val content_opt = read_file_contents(source) -in - case+ content_opt of - | Some(content) => write_file_contents(dest, content) - | None() => false -end - -(* ========== Pattern Matching ========== *) - -(* -** Check if file path matches pattern -** Supports: -** - Exact match: "LICENSE" -** - Wildcard: "*.md" -** - Path pattern: ".github/workflows/*.yml" -*) -fun path_matches_pattern(path: string, pattern: string): bool = let - (* Extract filename from path *) - val last_slash = string_last_index_of(path, "/") - val filename = if last_slash < 0 then path - else string_suffix(path, last_slash + 1) - - (* Check for wildcard *) - val star_pos = string_index_of(pattern, "*") -in - if star_pos < 0 then - (* Exact match *) - string_equal(filename, pattern) || string_equal(path, pattern) - else let - (* Wildcard match *) - val prefix = string_prefix(pattern, star_pos) - val suffix = string_suffix(pattern, star_pos + 1) - in - string_has_prefix(filename, prefix) && string_has_suffix(filename, suffix) - end -end - -(* ========== Directory Traversal ========== *) - -(* -** Find all files matching pattern in directory tree -** Returns list of relative paths -*) -fun find_matching_files( - base_dir: string, - pattern: string, - max_depth: int -): List0(string) = let - fun scan_directory( - dir_path: string, - depth: int, - acc: List0(string) - ): List0(string) = - if depth > max_depth then acc - else let - val dir_opt = dirptr_open_opt(dir_path) - in - case+ dir_opt of - | ~Some_vt(dir) => let - fun read_entries(acc: List0(string)): List0(string) = let - val entry = dirptr_read_opt(dir) - in - case+ entry of - | ~Some_vt(ent) => let - val name = dirent_get_name(ent) - in - if string_equal(name, ".") || string_equal(name, "..") then - read_entries(acc) - else let - val full_path = dir_path + "/" + name - (* Check if directory *) - val is_dir = dirent_is_dir(ent) - in - if is_dir then - (* Skip .git and other hidden dirs *) - if string_has_prefix(name, ".") then read_entries(acc) - else read_entries(scan_directory(full_path, depth + 1, acc)) - else - (* Check if matches pattern *) - if path_matches_pattern(full_path, pattern) - then read_entries(list0_cons(full_path, acc)) - else read_entries(acc) - end - end - | ~None_vt() => acc - end - - val files = read_entries(acc) - val () = dirptr_close(dir) - in - files - end - | ~None_vt() => acc - end -in - scan_directory(base_dir, 0, list0_nil()) -end - -(* ========== Circular Replacement Check ========== *) - -(* -** Compute FNV-1a hash of file content -** Used to detect circular replacements -*) -fun compute_file_hash(content: string): uint64 = let - val FNV_PRIME: uint64 = 0x100000001b3UL - val FNV_OFFSET: uint64 = 0xcbf29ce484222325UL - - fun hash_bytes(s: string, i: int, len: int, hash: uint64): uint64 = - if i >= len then hash - else let - val byte = string_get_at(s, i) - val new_hash = (hash XOR g0int2uint(g1ofg0(g0int2int(byte)))) * FNV_PRIME - in - hash_bytes(s, i + 1, len, new_hash) - end -in - hash_bytes(content, 0, string_length(content), FNV_OFFSET) -end - -(* -** Check if replacement would be circular -** Returns true if source and dest have same content hash -*) -fun is_circular_replacement(source: string, dest: string): bool = let - val source_content = read_file_contents(source) - val dest_content = read_file_contents(dest) -in - case+ (source_content, dest_content) of - | (Some(s), Some(d)) => - compute_file_hash(s) = compute_file_hash(d) - | (_, _) => false -end - -(* ========== Repository Operations ========== *) - -(* -** Replace files in repository -** Returns (successful_replacements, failed_replacements) -*) -fun replace_files_in_repo( - repo_path: string, - pattern: string, - replacement_file: string, - backup: bool, - dry_run: bool -): @(int, int) = let - (* Find all matching files *) - val matching_files = find_matching_files(repo_path, pattern, 5) - - (* Read replacement content once *) - val replacement_content = read_file_contents(replacement_file) -in - case+ replacement_content of - | None() => @(0, list0_length(matching_files)) - | Some(content) => let - fun process_files( - files: List0(string), - success: int, - failed: int - ): @(int, int) = - case+ files of - | list0_nil() => @(success, failed) - | list0_cons(file, rest) => - (* Check for circular replacement *) - if is_circular_replacement(replacement_file, file) then - process_files(rest, success, failed + 1) - else - if dry_run then - process_files(rest, success + 1, failed) - else let - (* Create backup if requested *) - val backup_result = - if backup then - copy_file(file, file + ".backup") - else true - - (* Write replacement *) - val write_result = - if backup_result then write_file_contents(file, content) - else false - in - if write_result - then process_files(rest, success + 1, failed) - else process_files(rest, success, failed + 1) - end - in - process_files(matching_files, 0, 0) - end -end - -(* ========== Main Operation ========== *) - -(* -** Execute file replace operation -** Replaces files matching pattern with replacement file -*) -extern -fun execute_file_replace_operation( - repos: List0(string), - pattern: string, - replacement_file: string, - backup: bool, - dry_run: bool -): batch_result = "ext#" - -implement -execute_file_replace_operation(repos, pattern, replacement_file, backup, dry_run) = let - (* Validate replacement file exists *) - val replacement_exists = file_exists(replacement_file) -in - if ~replacement_exists then - @{ - total = 0, - successful = 0, - failed = 0, - results = list0_nil() - } - else let - fun process_repos( - rs: List0(string), - success: int, - failed: int, - results: List0(operation_result) - ): batch_result = - case+ rs of - | list0_nil() => - @{ - total = success + failed, - successful = success, - failed = failed, - results = results - } - | list0_cons(repo, rest) => let - val (replaced, errors) = replace_files_in_repo(repo, pattern, replacement_file, backup, dry_run) - val result = - if replaced > 0 || errors > 0 then - @{ - repo_path = repo, - success = errors = 0, - message = "Replaced " + tostring_int(replaced) + " files, " + tostring_int(errors) + " errors", - files_affected = replaced - } - else - @{ - repo_path = repo, - success = true, - message = "No matching files found", - files_affected = 0 - } - in - if errors = 0 - then process_repos(rest, success + 1, failed, list0_cons(result, results)) - else process_repos(rest, success, failed + 1, list0_cons(result, results)) - end - in - process_repos(repos, 0, 0, list0_nil()) - end -end - -(* ========== C Exports ========== *) - -extern -fun c_file_replace( - base_dir: string, - max_depth: int, - pattern: string, - replacement_file: string, - backup: int, - dry_run: int -): c_batch_result = "ext#" - -implement -c_file_replace(base_dir, max_depth, pattern, replacement_file, backup, dry_run) = let - (* Scan for repositories *) - val repos = find_git_repos(base_dir, max_depth) - - (* Execute file replace *) - val result = execute_file_replace_operation( - repos, - pattern, - replacement_file, - backup != 0, - dry_run != 0 - ) -in - batch_result_to_c(result) -end diff --git a/scaffoldia/repo-batcher/src/ats2/operations/git_sync.dats b/scaffoldia/repo-batcher/src/ats2/operations/git_sync.dats deleted file mode 100644 index bb615512..00000000 --- a/scaffoldia/repo-batcher/src/ats2/operations/git_sync.dats +++ /dev/null @@ -1,386 +0,0 @@ -(* -** SPDX-License-Identifier: PMPL-1.0-or-later -** -** Git Batch Sync Operation -** Ports functionality from sync_repos.sh with formal verification -*) - -#include "share/atspre_define.hats" -#include "share/atspre_staload.hats" - -staload "libats/libc/SATS/stdlib.sats" -staload "libats/libc/SATS/stdio.sats" - -staload "./types.dats" -staload "../utils/string_utils.sats" - -(* ========== Git Command Execution ========== *) - -(* -** Executes git command in repository directory -** Returns exit code -*) -fun exec_git_command(repo_path: string, command: string): int = let - val full_command = "cd " + repo_path + " && git " + command -in - system(full_command) -end - -(* -** Checks if directory is a valid git repository -** Returns true if valid, false otherwise -*) -fun is_git_repo(path: string): bool = let - val git_dir = path + "/.git" - val result = test_file_exists(git_dir) -in - result -end - -(* -** Checks if repository has uncommitted changes -** Returns true if clean, false if dirty -*) -fun is_repo_clean(repo_path: string): bool = let - val exit_code = exec_git_command(repo_path, "diff --quiet") -in - exit_code = 0 -end - -(* -** Gets status of git repository -** Returns human-readable status string -*) -fun get_repo_status(repo_path: string): string = let - val status_file = "/tmp/repo_status.txt" - val cmd = "cd " + repo_path + " && git status --short > " + status_file - val _ = system(cmd) - val content = read_file_contents(status_file) -in - case+ content of - | Some(s) => s - | None() => "unknown" -end - -(* ========== Git Operations ========== *) - -(* -** Performs git add on all files in repository -** Returns operation result -*) -fun git_add_all(repo_path: string, dry_run: bool): operation_result = - if dry_run then - OpSuccess("[DRY-RUN] Would add all files in: " + repo_path) - else let - val exit_code = exec_git_command(repo_path, "add .") - in - if exit_code = 0 then - OpSuccess("Added all files in: " + repo_path) - else - OpFailure("Failed to add files in: " + repo_path) - end - -(* -** Performs git commit with message -** Returns operation result -*) -fun git_commit(repo_path: string, message: string, dry_run: bool): operation_result = - if dry_run then - OpSuccess("[DRY-RUN] Would commit in: " + repo_path) - else let - val cmd = "commit -m \"" + message + "\"" - val exit_code = exec_git_command(repo_path, cmd) - in - if exit_code = 0 then - OpSuccess("Committed in: " + repo_path) - else if exit_code = 1 then - // Exit code 1 usually means "nothing to commit" - OpSkipped("Nothing to commit in: " + repo_path) - else - OpFailure("Failed to commit in: " + repo_path) - end - -(* -** Performs git push to remote -** Returns operation result -*) -fun git_push(repo_path: string, dry_run: bool): operation_result = - if dry_run then - OpSuccess("[DRY-RUN] Would push: " + repo_path) - else let - val exit_code = exec_git_command(repo_path, "push") - in - if exit_code = 0 then - OpSuccess("Pushed: " + repo_path) - else - OpFailure("Failed to push: " + repo_path + " (check remote, auth, conflicts)") - end - -(* -** Full git sync: add, commit, push -** Returns operation result for this repository -*) -fun git_sync_repo( - repo_path: string, - commit_msg: string, - dry_run: bool -): operation_result = let - // Validate it's a git repo - val is_repo = is_git_repo(repo_path) -in - if ~is_repo then - OpFailure("Not a git repository: " + repo_path) - else let - // Step 1: git add . - val add_result = git_add_all(repo_path, dry_run) - in - case+ add_result of - | OpSuccess(_) => let - // Step 2: git commit - val commit_result = git_commit(repo_path, commit_msg, dry_run) - in - case+ commit_result of - | OpSuccess(_) => let - // Step 3: git push - val push_result = git_push(repo_path, dry_run) - in - push_result - end - | OpSkipped(msg) => OpSkipped(msg) - | OpFailure(msg) => OpFailure(msg) - end - | OpFailure(msg) => OpFailure("Add failed: " + msg) - | OpSkipped(msg) => OpSkipped(msg) - end -end - -(* ========== Repository Discovery ========== *) - -(* -** Finds all git repositories in directory up to max_depth -** Returns list of repository paths -** -** This replicates the logic from sync_repos.sh: -** find . -maxdepth 2 -name ".git" -type d -*) -fun find_git_repos( - base_dir: string, - max_depth: int -): List0(string) = let - // Use find command to locate .git directories - val find_cmd = - "find " + base_dir + - " -maxdepth " + tostring_int(max_depth) + - " -name \".git\" -type d 2>/dev/null" - - val output_file = "/tmp/repo_list.txt" - val cmd = find_cmd + " > " + output_file - val _ = system(cmd) - - // Read repository paths from output - val repos = read_repo_list_from_file(output_file) -in - repos -end - -(* -** Reads list of repository paths from file -** Each line is a path to .git directory, we extract parent -*) -and read_repo_list_from_file(path: string): List0(string) = let - val file = fileref_open_opt(path, file_mode_r) -in - case+ file of - | ~Some_vt(f) => let - fun read_lines(acc: List0(string)): List0(string) = let - val line = fileref_get_line_string(f) - in - if string_is_empty(line) then - acc - else let - // Remove ".git" suffix to get repo path - val repo_path = remove_git_suffix(line) - in - read_lines(list_cons(repo_path, acc)) - end - end - val repos = read_lines(list_nil()) - val () = fileref_close(f) - in - repos - end - | ~None_vt() => list_nil() -end - -(* -** Removes "/.git" suffix from path -*) -and remove_git_suffix(path: string): string = let - val git_suffix = "/.git" - val idx = string_index_of(path, git_suffix) -in - if idx >= 0 then - string_substring(path, 0, idx) - else - path -end - -(* ========== Parallel Execution (Placeholder) ========== *) - -(* -** Processes repositories in parallel (to be implemented in V) -** For now, this is a sequential implementation -** -** The actual parallelism will be handled by V coroutines -** This function just processes the list sequentially -*) -fun process_repos_sequential( - repos: List0(string), - commit_msg: string, - dry_run: bool, - results: List0(operation_result) -): List0(operation_result) = - case+ repos of - | list_nil() => results - | list_cons(repo, rest) => let - val result = git_sync_repo(repo, commit_msg, dry_run) - in - process_repos_sequential(rest, commit_msg, dry_run, list_cons(result, results)) - end - -(* ========== Summary Report ========== *) - -(* -** Generates summary report (like sync_repos.sh output) -** Returns formatted string -*) -fun generate_summary_report(results: List0(operation_result)): string = let - fun count_results( - results: List0(operation_result), - success: int, - failure: int, - skipped: int - ): @(int, int, int) = - case+ results of - | list_nil() => @(success, failure, skipped) - | list_cons(r, rest) => - case+ r of - | OpSuccess(_) => count_results(rest, success + 1, failure, skipped) - | OpFailure(_) => count_results(rest, success, failure + 1, skipped) - | OpSkipped(_) => count_results(rest, success, failure, skipped + 1) - - val @(s, f, sk) = count_results(results, 0, 0, 0) - val total = s + f + sk - - // Build summary string - val header = "\n--- Sync Summary ---\n" - val table_header = "Status | Count\n" - val separator = "--------------------|----------\n" - val success_line = "Fully Succeeded | " + tostring_int(s) + "\n" - val failure_line = "Failed/Partial | " + tostring_int(f) + "\n" - val skipped_line = "Skipped | " + tostring_int(sk) + "\n" - val total_line = "Total Processed | " + tostring_int(total) + "\n" -in - header + table_header + separator + success_line + failure_line + - skipped_line + total_line -end - -(* -** Generates failure details report -** Lists all failed operations -*) -fun generate_failure_report(results: List0(operation_result)): string = let - fun collect_failures( - results: List0(operation_result), - failures: List0(string) - ): List0(string) = - case+ results of - | list_nil() => failures - | list_cons(r, rest) => - case+ r of - | OpFailure(msg) => collect_failures(rest, list_cons(msg, failures)) - | _ => collect_failures(rest, failures) - - val failures = collect_failures(results, list_nil()) - val failure_count = list_length(failures) -in - if failure_count = 0 then - "" - else let - val header = "\n--- Failure Details ---\n" - fun format_failures(failures: List0(string), acc: string): string = - case+ failures of - | list_nil() => acc - | list_cons(msg, rest) => - format_failures(rest, acc + " - " + msg + "\n") - in - header + format_failures(failures, "") - end -end - -(* ========== Main Git Sync Implementation ========== *) - -(* -** Main implementation of git-sync operation -** This is the formally verified version of sync_repos.sh -*) -implement execute_git_sync_operation( - base_dir: string, - max_depth: int, - commit_msg: string, - parallel_jobs: int, - dry_run: bool -): batch_result = let - // Find all git repositories (like: find . -maxdepth 2 -name ".git") - val repos = find_git_repos(base_dir, max_depth) - - // Process repositories - // TODO: In V, this will use coroutines for parallel execution - // For now, sequential processing - val results = process_repos_sequential(repos, commit_msg, dry_run, list_nil()) - - // Count results - fun count_results( - results: List0(operation_result), - success: int, - failure: int, - skipped: int - ): @(int, int, int) = - case+ results of - | list_nil() => @(success, failure, skipped) - | list_cons(r, rest) => - case+ r of - | OpSuccess(_) => count_results(rest, success + 1, failure, skipped) - | OpFailure(_) => count_results(rest, success, failure + 1, skipped) - | OpSkipped(_) => count_results(rest, success, failure, skipped + 1) - - val @(s, f, sk) = count_results(results, 0, 0, 0) - - // Generate summary report - val summary = generate_summary_report(results) - val failure_report = generate_failure_report(results) - val () = print_string(summary) - val () = print_string(failure_report) -in - @{ - success_count = s, - failure_count = f, - skipped_count = sk, - results = results, - rollback_info = None() - } -end - -(* ========== Helper Functions ========== *) - -and test_file_exists(path: string): bool = let - val cmd = "test -e " + path - val exit_code = system(cmd) -in - exit_code = 0 -end - -and list_length(lst: List0('a)): int = - case+ lst of - | list_nil() => 0 - | list_cons(_, rest) => 1 + list_length(rest) diff --git a/scaffoldia/repo-batcher/src/ats2/operations/github_settings.dats b/scaffoldia/repo-batcher/src/ats2/operations/github_settings.dats deleted file mode 100644 index 5ddae5c8..00000000 --- a/scaffoldia/repo-batcher/src/ats2/operations/github_settings.dats +++ /dev/null @@ -1,475 +0,0 @@ -(* -** SPDX-License-Identifier: PMPL-1.0-or-later -** -** GitHub Settings Operation Implementation -** Bulk GitHub repository configuration management -** Week 1: Basic repository and merge settings -*) - -#include "share/atspre_define.hats" -#include "share/atspre_staload.hats" - -staload "libats/libc/SATS/stdio.sats" - -staload "./types.dats" -staload "../utils/string_utils.sats" - -(* ========== Settings Type Definitions ========== *) - -(* -** Repository feature settings -*) -typedef repo_features = @{ - has_issues = Option(bool), - has_wiki = Option(bool), - has_projects = Option(bool), - has_downloads = Option(bool) -} - -(* -** Merge strategy settings -*) -typedef merge_settings = @{ - allow_squash_merge = Option(bool), - allow_merge_commit = Option(bool), - allow_rebase_merge = Option(bool), - delete_branch_on_merge = Option(bool), - allow_auto_merge = Option(bool) -} - -(* -** Complete settings configuration -*) -typedef github_settings = @{ - repo_features = repo_features, - merge_settings = merge_settings -} - -(* -** Settings change result -*) -typedef settings_result = @{ - repo_path = string, - success = bool, - changes_applied = int, - message = string -} - -(* ========== Default Settings ========== *) - -(* -** Create default repository features -** All options are None (no changes) -*) -fun default_repo_features(): repo_features = - @{ - has_issues = None(), - has_wiki = None(), - has_projects = None(), - has_downloads = None() - } - -(* -** Create default merge settings -** All options are None (no changes) -*) -fun default_merge_settings(): merge_settings = - @{ - allow_squash_merge = None(), - allow_merge_commit = None(), - allow_rebase_merge = None(), - delete_branch_on_merge = None(), - allow_auto_merge = None() - } - -(* -** Create default settings configuration -*) -fun default_github_settings(): github_settings = - @{ - repo_features = default_repo_features(), - merge_settings = default_merge_settings() - } - -(* ========== Settings Validation ========== *) - -(* -** Count number of settings changes -** Returns count of non-None settings -*) -fun count_changes_repo_features(features: repo_features): int = let - fun count_option(opt: Option(bool), acc: int): int = - case+ opt of - | Some(_) => acc + 1 - | None() => acc -in - count_option(features.has_issues, 0) + - count_option(features.has_wiki, 0) + - count_option(features.has_projects, 0) + - count_option(features.has_downloads, 0) -end - -fun count_changes_merge_settings(settings: merge_settings): int = let - fun count_option(opt: Option(bool), acc: int): int = - case+ opt of - | Some(_) => acc + 1 - | None() => acc -in - count_option(settings.allow_squash_merge, 0) + - count_option(settings.allow_merge_commit, 0) + - count_option(settings.allow_rebase_merge, 0) + - count_option(settings.delete_branch_on_merge, 0) + - count_option(settings.allow_auto_merge, 0) -end - -fun count_total_changes(settings: github_settings): int = - count_changes_repo_features(settings.repo_features) + - count_changes_merge_settings(settings.merge_settings) - -(* -** Validate settings before applying -** Returns true if settings are safe to apply -*) -fun validate_settings(settings: github_settings): bool = let - val change_count = count_total_changes(settings) -in - (* Must have at least one change *) - if change_count = 0 then false - (* Too many changes at once might be dangerous *) - else if change_count > 20 then false - else true -end - -(* ========== Settings Application (Stub for V FFI) ========== *) - -(* -** Apply repository feature settings -** This will be implemented in V with GitHub API calls -*) -extern -fun apply_repo_features( - repo: string, - features: repo_features, - dry_run: bool -): @(bool, int) = "ext#" - -(* -** Apply merge settings -** This will be implemented in V with GitHub API calls -*) -extern -fun apply_merge_settings( - repo: string, - settings: merge_settings, - dry_run: bool -): @(bool, int) = "ext#" - -(* ========== Main Operation ========== *) - -(* -** Apply GitHub settings to a single repository -*) -fun apply_settings_to_repo( - repo: string, - settings: github_settings, - dry_run: bool -): settings_result = let - (* Validate settings first *) - val valid = validate_settings(settings) -in - if ~valid then - @{ - repo_path = repo, - success = false, - changes_applied = 0, - message = "Invalid settings configuration" - } - else let - (* Apply repository features *) - val (features_ok, features_count) = apply_repo_features(repo, settings.repo_features, dry_run) - - (* Apply merge settings *) - val (merge_ok, merge_count) = apply_merge_settings(repo, settings.merge_settings, dry_run) - - (* Compute overall success *) - val success = features_ok && merge_ok - val total_changes = features_count + merge_count - - (* Generate message *) - val message = - if dry_run then - "[DRY RUN] Would apply " + tostring_int(total_changes) + " changes" - else if success then - "Applied " + tostring_int(total_changes) + " changes successfully" - else - "Failed to apply settings" - in - @{ - repo_path = repo, - success = success, - changes_applied = total_changes, - message = message - } - end -end - -(* -** Execute GitHub settings operation across multiple repositories -*) -extern -fun execute_github_settings_operation( - repos: List0(string), - settings: github_settings, - dry_run: bool -): List0(settings_result) = "ext#" - -implement -execute_github_settings_operation(repos, settings, dry_run) = let - fun process_repos( - rs: List0(string), - results: List0(settings_result) - ): List0(settings_result) = - case+ rs of - | list0_nil() => results - | list0_cons(repo, rest) => let - val result = apply_settings_to_repo(repo, settings, dry_run) - in - process_repos(rest, list0_cons(result, results)) - end -in - process_repos(repos, list0_nil()) -end - -(* ========== Settings Parsing (from TOML) ========== *) - -(* -** Parse boolean setting from string -*) -fun parse_bool_setting(value: string): Option(bool) = - if string_equal(value, "true") then Some(true) - else if string_equal(value, "false") then Some(false) - else None() - -(* -** Set repository feature from key-value pair -*) -fun set_repo_feature( - features: repo_features, - key: string, - value: string -): repo_features = let - val bool_val = parse_bool_setting(value) -in - if string_equal(key, "has_issues") then - @{ - has_issues = bool_val, - has_wiki = features.has_wiki, - has_projects = features.has_projects, - has_downloads = features.has_downloads - } - else if string_equal(key, "has_wiki") then - @{ - has_issues = features.has_issues, - has_wiki = bool_val, - has_projects = features.has_projects, - has_downloads = features.has_downloads - } - else if string_equal(key, "has_projects") then - @{ - has_issues = features.has_issues, - has_wiki = features.has_wiki, - has_projects = bool_val, - has_downloads = features.has_downloads - } - else if string_equal(key, "has_downloads") then - @{ - has_issues = features.has_issues, - has_wiki = features.has_wiki, - has_projects = features.has_projects, - has_downloads = bool_val - } - else features -end - -(* -** Set merge setting from key-value pair -*) -fun set_merge_setting( - settings: merge_settings, - key: string, - value: string -): merge_settings = let - val bool_val = parse_bool_setting(value) -in - if string_equal(key, "allow_squash_merge") then - @{ - allow_squash_merge = bool_val, - allow_merge_commit = settings.allow_merge_commit, - allow_rebase_merge = settings.allow_rebase_merge, - delete_branch_on_merge = settings.delete_branch_on_merge, - allow_auto_merge = settings.allow_auto_merge - } - else if string_equal(key, "allow_merge_commit") then - @{ - allow_squash_merge = settings.allow_squash_merge, - allow_merge_commit = bool_val, - allow_rebase_merge = settings.allow_rebase_merge, - delete_branch_on_merge = settings.delete_branch_on_merge, - allow_auto_merge = settings.allow_auto_merge - } - else if string_equal(key, "allow_rebase_merge") then - @{ - allow_squash_merge = settings.allow_squash_merge, - allow_merge_commit = settings.allow_merge_commit, - allow_rebase_merge = bool_val, - delete_branch_on_merge = settings.delete_branch_on_merge, - allow_auto_merge = settings.allow_auto_merge - } - else if string_equal(key, "delete_branch_on_merge") then - @{ - allow_squash_merge = settings.allow_squash_merge, - allow_merge_commit = settings.allow_merge_commit, - allow_rebase_merge = settings.allow_rebase_merge, - delete_branch_on_merge = bool_val, - allow_auto_merge = settings.allow_auto_merge - } - else if string_equal(key, "allow_auto_merge") then - @{ - allow_squash_merge = settings.allow_squash_merge, - allow_merge_commit = settings.allow_merge_commit, - allow_rebase_merge = settings.allow_rebase_merge, - delete_branch_on_merge = settings.delete_branch_on_merge, - allow_auto_merge = bool_val - } - else settings -end - -(* ========== Summary Statistics ========== *) - -(* -** Compute summary statistics from results -*) -fun compute_summary(results: List0(settings_result)): @(int, int, int) = let - fun loop(rs: List0(settings_result), success: int, failed: int, changes: int): @(int, int, int) = - case+ rs of - | list0_nil() => @(success, failed, changes) - | list0_cons(r, rest) => - if r.success then - loop(rest, success + 1, failed, changes + r.changes_applied) - else - loop(rest, success, failed + 1, changes) -in - loop(results, 0, 0, 0) -end - -(* -** Print summary of settings operation -*) -fun print_summary(results: List0(settings_result)): void = let - val (success, failed, changes) = compute_summary(results) - val total = success + failed -in - println("=== GitHub Settings Summary ==="); - println("Total repositories: " + tostring_int(total)); - println("Successful: " + tostring_int(success)); - println("Failed: " + tostring_int(failed)); - println("Total changes applied: " + tostring_int(changes)); - println("") -end - -(* ========== C Exports for FFI ========== *) - -(* -** C-compatible settings structure -*) -typedef c_github_settings = @{ - (* Repository features *) - has_issues = int, (* -1=none, 0=false, 1=true *) - has_wiki = int, - has_projects = int, - has_downloads = int, - (* Merge settings *) - allow_squash_merge = int, - allow_merge_commit = int, - allow_rebase_merge = int, - delete_branch_on_merge = int, - allow_auto_merge = int -} - -(* -** Convert Option(bool) to C int representation -*) -fun option_bool_to_int(opt: Option(bool)): int = - case+ opt of - | Some(true) => 1 - | Some(false) => 0 - | None() => ~1 - -(* -** Convert C int to Option(bool) -*) -fun int_to_option_bool(n: int): Option(bool) = - if n = 1 then Some(true) - else if n = 0 then Some(false) - else None() - -(* -** Convert github_settings to C-compatible structure -*) -fun settings_to_c(settings: github_settings): c_github_settings = - @{ - has_issues = option_bool_to_int(settings.repo_features.has_issues), - has_wiki = option_bool_to_int(settings.repo_features.has_wiki), - has_projects = option_bool_to_int(settings.repo_features.has_projects), - has_downloads = option_bool_to_int(settings.repo_features.has_downloads), - allow_squash_merge = option_bool_to_int(settings.merge_settings.allow_squash_merge), - allow_merge_commit = option_bool_to_int(settings.merge_settings.allow_merge_commit), - allow_rebase_merge = option_bool_to_int(settings.merge_settings.allow_rebase_merge), - delete_branch_on_merge = option_bool_to_int(settings.merge_settings.delete_branch_on_merge), - allow_auto_merge = option_bool_to_int(settings.merge_settings.allow_auto_merge) - } - -(* -** Main C export for github-settings operation -*) -extern -fun c_github_settings( - base_dir: string, - max_depth: int, - settings: c_github_settings, - dry_run: int -): int = "ext#" - -implement -c_github_settings(base_dir, max_depth, c_settings, dry_run) = let - (* Convert C settings to ATS2 settings *) - val settings = @{ - repo_features = @{ - has_issues = int_to_option_bool(c_settings.has_issues), - has_wiki = int_to_option_bool(c_settings.has_wiki), - has_projects = int_to_option_bool(c_settings.has_projects), - has_downloads = int_to_option_bool(c_settings.has_downloads) - }, - merge_settings = @{ - allow_squash_merge = int_to_option_bool(c_settings.allow_squash_merge), - allow_merge_commit = int_to_option_bool(c_settings.allow_merge_commit), - allow_rebase_merge = int_to_option_bool(c_settings.allow_rebase_merge), - delete_branch_on_merge = int_to_option_bool(c_settings.delete_branch_on_merge), - allow_auto_merge = int_to_option_bool(c_settings.allow_auto_merge) - } - } - - (* Scan for repositories *) - val repos = find_git_repos(base_dir, max_depth) - - (* Execute operation *) - val results = execute_github_settings_operation(repos, settings, dry_run != 0) - - (* Print summary *) - val () = print_summary(results) - - (* Return success count *) - val (success, failed, _) = compute_summary(results) -in - success -end diff --git a/scaffoldia/repo-batcher/src/ats2/operations/license_update.dats b/scaffoldia/repo-batcher/src/ats2/operations/license_update.dats deleted file mode 100644 index c379589c..00000000 --- a/scaffoldia/repo-batcher/src/ats2/operations/license_update.dats +++ /dev/null @@ -1,390 +0,0 @@ -(* -** SPDX-License-Identifier: PMPL-1.0-or-later -** -** License Update Operation Implementation -** Replaces license headers and LICENSE files across repositories -*) - -#include "share/atspre_define.hats" -#include "share/atspre_staload.hats" - -staload "libats/libc/SATS/stdio.sats" -staload "libats/libc/SATS/dirent.sats" - -staload "./types.dats" -staload "../validation/spdx.dats" -staload "../utils/string_utils.sats" - -(* ========== File Operations ========== *) - -(* -** Reads entire file into string -** Returns Some(content) if successful, None otherwise -*) -fun read_file_contents(path: string): Option(string) = let - val file = fileref_open_opt(path, file_mode_r) -in - case+ file of - | ~Some_vt(f) => let - val content = fileref_get_file_string(f) - val () = fileref_close(f) - in - Some(content) - end - | ~None_vt() => None() -end - -(* -** Writes string to file -** Returns true if successful, false otherwise -*) -fun write_file_contents(path: string, content: string): bool = let - val file = fileref_open_opt(path, file_mode_w) -in - case+ file of - | ~Some_vt(f) => let - val () = fileref_put_string(f, content) - val () = fileref_close(f) - in - true - end - | ~None_vt() => false -end - -(* -** Creates backup of file -** Returns backup path if successful, None otherwise -*) -fun backup_file(path: string, backup_dir: string): Option(string) = let - val timestamp = // TODO: get actual timestamp - "backup" - val filename = // Extract filename from path - path - val backup_path = backup_dir + "/" + filename + "." + timestamp - - // Read original - val content = read_file_contents(path) -in - case+ content of - | Some(c) => - if write_file_contents(backup_path, c) then - Some(backup_path) - else - None() - | None() => None() -end - -(* ========== SPDX Header Update ========== *) - -(* -** Replaces SPDX header in file content -** Returns updated content -*) -fun replace_spdx_header( - content: string, - old_license: string, - new_license: string, - comment_style: string -): string = let - val old_header = generate_spdx_header(old_license, comment_style) - val new_header = generate_spdx_header(new_license, comment_style) -in - // Replace old header with new header - string_replace(content, old_header, new_header) -end - -(* -** Updates SPDX header in a single file -** Returns operation result -*) -fun update_spdx_in_file( - path: string, - old_license: string, - new_license: string, - do_backup: bool, - backup_dir: string, - dry_run: bool -): operation_result = let - // Get file extension to determine comment style - val ext = get_file_extension(path) - val comment_style = get_comment_style_for_ext(ext) - - // Read file - val content = read_file_contents(path) -in - case+ content of - | Some(original) => let - // Check if file has old license - val has_old = string_contains(original, old_license) - in - if ~has_old then - OpSkipped("File does not contain old license: " + path) - else let - val updated = replace_spdx_header(original, old_license, new_license, comment_style) - in - if dry_run then - OpSuccess("[DRY-RUN] Would update: " + path) - else let - // Create backup if required - val backup_result = - if do_backup then - backup_file(path, backup_dir) - else - Some("no-backup") - - val write_result = - case+ backup_result of - | Some(backup_path) => - if write_file_contents(path, updated) then - OpSuccess("Updated: " + path + " (backup: " + backup_path + ")") - else - OpFailure("Failed to write: " + path) - | None() => - OpFailure("Failed to create backup for: " + path) - in - write_result - end - end - end - | None() => OpFailure("Failed to read file: " + path) -end - -(* ========== LICENSE File Update ========== *) - -(* -** Reads LICENSE file content for given SPDX identifier -** Returns license text or error -*) -fun get_license_text(spdx_id: string): Option(string) = - case+ spdx_id of - | "PMPL-1.0-or-later" => Some( - "Palimpsest License (PMPL-1.0-or-later)\n\n" + - "Full license text at: https://github.com/hyperpolymath/palimpsest-license\n" - ) - | "MIT" => Some( - "MIT License\n\n" + - "Permission is hereby granted, free of charge...\n" - ) - | "Apache-2.0" => Some( - "Apache License, Version 2.0\n\n" + - "Licensed under the Apache License...\n" - ) - | _ => None() // TODO: Fetch from SPDX database - -(* -** Updates LICENSE file in repository -** Returns operation result -*) -fun update_license_file( - repo_path: string, - new_license: string, - do_backup: bool, - backup_dir: string, - dry_run: bool -): operation_result = let - val license_path = repo_path + "/LICENSE" - val license_text = get_license_text(new_license) -in - case+ license_text of - | Some(text) => - if dry_run then - OpSuccess("[DRY-RUN] Would update LICENSE in: " + repo_path) - else let - // Backup if required - val backup_result = - if do_backup then - backup_file(license_path, backup_dir) - else - Some("no-backup") - - val write_result = - case+ backup_result of - | Some(_) => - if write_file_contents(license_path, text) then - OpSuccess("Updated LICENSE in: " + repo_path) - else - OpFailure("Failed to write LICENSE in: " + repo_path) - | None() => - OpFailure("Failed to backup LICENSE in: " + repo_path) - in - write_result - end - | None() => - OpFailure("Unknown license text for: " + new_license) -end - -(* ========== Repository Scanning ========== *) - -(* -** Finds all source files in repository matching patterns -** Returns list of file paths -*) -fun find_source_files( - repo_path: string, - patterns: List0(string) -): List0(string) = let - // TODO: Implement recursive directory traversal - // For now, return empty list as placeholder -in - list_nil() -end - -(* -** Source file patterns to update -*) -val default_source_patterns = @[ - "**/*.rs", - "**/*.v", - "**/*.dats", - "**/*.idr", - "**/*.zig", - "**/*.scm", - "**/*.toml", - "**/*.yml", - "**/*.yaml" -] : List0(string) - -(* ========== Main License Update Logic ========== *) - -(* -** Performs license update on a single repository -** Returns batch_result for this repo -*) -fun update_license_in_repo( - repo_path: string, - old_license: string, - new_license: string, - do_backup: bool, - backup_dir: string, - dry_run: bool -): batch_result = let - // Find all source files - val source_files = find_source_files(repo_path, default_source_patterns) - - // Update LICENSE file - val license_result = update_license_file( - repo_path, new_license, do_backup, backup_dir, dry_run - ) - - // Update SPDX headers in all source files - fun update_files(files: List0(string), results: List0(operation_result)): List0(operation_result) = - case+ files of - | list_nil() => results - | list_cons(file, rest) => let - val result = update_spdx_in_file( - file, old_license, new_license, do_backup, backup_dir, dry_run - ) - in - update_files(rest, list_cons(result, results)) - end - - val file_results = update_files(source_files, list_cons(license_result, list_nil())) - - // Count results - fun count_results( - results: List0(operation_result), - success: int, - failure: int, - skipped: int - ): @(int, int, int) = - case+ results of - | list_nil() => @(success, failure, skipped) - | list_cons(r, rest) => - case+ r of - | OpSuccess(_) => count_results(rest, success + 1, failure, skipped) - | OpFailure(_) => count_results(rest, success, failure + 1, skipped) - | OpSkipped(_) => count_results(rest, success, failure, skipped + 1) - - val @(s, f, sk) = count_results(file_results, 0, 0, 0) -in - @{ - success_count = s, - failure_count = f, - skipped_count = sk, - results = file_results, - rollback_info = None() - } -end - -(* -** Main entry point for license update operation -** Processes all target repositories -*) -implement execute_batch_operation(op, ctx) = - case+ op of - | OpLicenseUpdate(lic_op) => let - // Extract operation parameters - // TODO: Pattern match on lic_op structure - val old_license = "AGPL-3.0" // Placeholder - val new_license = "PMPL-1.0-or-later" // Placeholder - - // Get target repositories - val repos = resolve_targets(ctx.targets) - - // Process each repository - fun process_repos( - repos: List0(string), - results: List0(batch_result) - ): List0(batch_result) = - case+ repos of - | list_nil() => results - | list_cons(repo, rest) => let - val result = update_license_in_repo( - repo, - old_license, - new_license, - true, // backup enabled - ctx.backup_dir, - ctx.dry_run - ) - in - process_repos(rest, list_cons(result, results)) - end - - val all_results = process_repos(repos, list_nil()) - - // Aggregate results - // TODO: Sum up all batch_results - in - // Return aggregated result - @{ - success_count = 0, - failure_count = 0, - skipped_count = 0, - results = list_nil(), - rollback_info = None() - } - end - | _ => - // Other operations handled elsewhere - @{ - success_count = 0, - failure_count = 0, - skipped_count = 0, - results = list_nil(), - rollback_info = None() - } - -(* ========== Helper Functions ========== *) - -(* -** Resolves target specification to list of repository paths -*) -and resolve_targets(target: repo_target): List0(string) = - case+ target of - | RepoList(repos) => repos - | RepoPattern(pattern) => list_nil() // TODO: Implement pattern matching - | RepoFile(file) => list_nil() // TODO: Read from file - | RepoDirectory(dir) => list_nil() // TODO: Scan directory - -(* -** Gets file extension from path -*) -and get_file_extension(path: string): string = let - val idx = string_rindex_of(path, '.') -in - if idx >= 0 then - string_suffix(path, idx) - else - "" -end diff --git a/scaffoldia/repo-batcher/src/ats2/operations/spdx_audit.dats b/scaffoldia/repo-batcher/src/ats2/operations/spdx_audit.dats deleted file mode 100644 index 3f813ee7..00000000 --- a/scaffoldia/repo-batcher/src/ats2/operations/spdx_audit.dats +++ /dev/null @@ -1,387 +0,0 @@ -(* -** SPDX-License-Identifier: PMPL-1.0-or-later -** -** SPDX Audit Operation Implementation -** Audits SPDX license headers across all source files -** Ensures compliance with hyperpolymath licensing standards -*) - -#include "share/atspre_define.hats" -#include "share/atspre_staload.hats" - -staload "libats/libc/SATS/stdio.sats" -staload "libats/libc/SATS/dirent.sats" - -staload "./types.dats" -staload "../validation/spdx.dats" -staload "../utils/string_utils.sats" - -(* ========== File Extensions ========== *) - -(* -** Source file extensions that should have SPDX headers -*) -val source_extensions: List0(string) = list0_make( - ".rs", (* Rust *) - ".v", (* V *) - ".c", (* C *) - ".h", (* C headers *) - ".cpp", (* C++ *) - ".hpp", (* C++ headers *) - ".js", (* JavaScript *) - ".jsx", (* React *) - ".ts", (* TypeScript *) - ".tsx", (* React TypeScript *) - ".py", (* Python *) - ".rb", (* Ruby *) - ".go", (* Go *) - ".java", (* Java *) - ".kt", (* Kotlin *) - ".scala", (* Scala *) - ".ml", (* OCaml *) - ".mli", (* OCaml interface *) - ".ex", (* Elixir *) - ".exs", (* Elixir script *) - ".gleam", (* Gleam *) - ".dats", (* ATS2 *) - ".sats", (* ATS2 *) - ".idr", (* Idris2 *) - ".zig", (* Zig *) - ".sh", (* Shell *) - ".bash", (* Bash *) - ".yml", (* YAML *) - ".yaml", (* YAML *) - ".toml", (* TOML *) - ".scm", (* Scheme *) - ".rkt", (* Racket *) - ".el", (* Emacs Lisp *) - ".jl", (* Julia *) - ".ad", (* Ada body *) - ".ads" (* Ada spec *) -) - -fun has_source_extension(path: string): bool = let - fun check_extension(exts: List0(string)): bool = - case+ exts of - | list0_nil() => false - | list0_cons(ext, rest) => - if string_has_suffix(path, ext) then true - else check_extension(rest) -in - check_extension(source_extensions) -end - -(* ========== File Operations ========== *) - -fun read_file_contents(path: string): Option(string) = let - val file = fileref_open_opt(path, file_mode_r) -in - case+ file of - | ~Some_vt(f) => let - val content = fileref_get_file_string(f) - val () = fileref_close(f) - in - Some(content) - end - | ~None_vt() => None() -end - -(* ========== SPDX Header Detection ========== *) - -(* -** Check if content has valid SPDX header -** Looks for "SPDX-License-Identifier:" in first 20 lines -*) -fun has_spdx_header(content: string): bool = let - val lines = string_split(content, "\n") - - fun check_lines(ls: List0(string), count: int): bool = - if count >= 20 then false - else - case+ ls of - | list0_nil() => false - | list0_cons(line, rest) => - if string_contains(line, "SPDX-License-Identifier:") then true - else check_lines(rest, count + 1) -in - check_lines(lines, 0) -end - -(* -** Extract SPDX identifier from content -** Returns the license identifier (e.g., "PMPL-1.0-or-later") -*) -fun extract_spdx_id(content: string): Option(string) = let - val lines = string_split(content, "\n") - - fun find_in_lines(ls: List0(string)): Option(string) = - case+ ls of - | list0_nil() => None() - | list0_cons(line, rest) => - if string_contains(line, "SPDX-License-Identifier:") then let - val id_pos = string_index_of(line, "SPDX-License-Identifier:") - val after_id = string_suffix(line, id_pos + 24) - val trimmed = string_trim(after_id) - in - if string_length(trimmed) > 0 then Some(trimmed) - else None() - end - else find_in_lines(rest) -in - find_in_lines(lines) -end - -(* ========== Audit Results ========== *) - -typedef audit_stats = @{ - total_files = int, - with_spdx = int, - without_spdx = int, - invalid_spdx = int, - pmpl_license = int, - other_licenses = int -} - -typedef audit_result = @{ - repo_path = string, - stats = audit_stats, - missing_files = List0(string), - invalid_files = List0(string) -} - -(* ========== Directory Scanning ========== *) - -(* -** Scan repository for source files and audit SPDX headers -*) -fun audit_repository(repo_path: string): audit_result = let - fun scan_directory( - dir_path: string, - depth: int, - stats: audit_stats, - missing: List0(string), - invalid: List0(string) - ): @(audit_stats, List0(string), List0(string)) = - if depth > 5 then @(stats, missing, invalid) - else let - val dir_opt = dirptr_open_opt(dir_path) - in - case+ dir_opt of - | ~Some_vt(dir) => let - fun read_entries( - s: audit_stats, - m: List0(string), - i: List0(string) - ): @(audit_stats, List0(string), List0(string)) = let - val entry = dirptr_read_opt(dir) - in - case+ entry of - | ~Some_vt(ent) => let - val name = dirent_get_name(ent) - in - if string_equal(name, ".") || string_equal(name, "..") then - read_entries(s, m, i) - else let - val full_path = dir_path + "/" + name - val is_dir = dirent_is_dir(ent) - in - if is_dir then - (* Skip hidden directories *) - if string_has_prefix(name, ".") then read_entries(s, m, i) - else let - val (new_stats, new_missing, new_invalid) = - scan_directory(full_path, depth + 1, s, m, i) - in - read_entries(new_stats, new_missing, new_invalid) - end - else - (* Check if source file *) - if has_source_extension(full_path) then let - val content_opt = read_file_contents(full_path) - in - case+ content_opt of - | Some(content) => - if has_spdx_header(content) then let - val spdx_id_opt = extract_spdx_id(content) - in - case+ spdx_id_opt of - | Some(spdx_id) => - if is_valid_spdx(spdx_id) then let - val is_pmpl = string_has_prefix(spdx_id, "PMPL-1.0") - val new_stats = @{ - total_files = s.total_files + 1, - with_spdx = s.with_spdx + 1, - without_spdx = s.without_spdx, - invalid_spdx = s.invalid_spdx, - pmpl_license = if is_pmpl then s.pmpl_license + 1 else s.pmpl_license, - other_licenses = if is_pmpl then s.other_licenses else s.other_licenses + 1 - } - in - read_entries(new_stats, m, i) - end - else let - val new_stats = @{ - total_files = s.total_files + 1, - with_spdx = s.with_spdx, - without_spdx = s.without_spdx, - invalid_spdx = s.invalid_spdx + 1, - pmpl_license = s.pmpl_license, - other_licenses = s.other_licenses - } - in - read_entries(new_stats, m, list0_cons(full_path, i)) - end - | None() => let - val new_stats = @{ - total_files = s.total_files + 1, - with_spdx = s.with_spdx, - without_spdx = s.without_spdx, - invalid_spdx = s.invalid_spdx + 1, - pmpl_license = s.pmpl_license, - other_licenses = s.other_licenses - } - in - read_entries(new_stats, m, list0_cons(full_path, i)) - end - end - else let - val new_stats = @{ - total_files = s.total_files + 1, - with_spdx = s.with_spdx, - without_spdx = s.without_spdx + 1, - invalid_spdx = s.invalid_spdx, - pmpl_license = s.pmpl_license, - other_licenses = s.other_licenses - } - in - read_entries(new_stats, list0_cons(full_path, m), i) - end - | None() => read_entries(s, m, i) - end - else read_entries(s, m, i) - end - end - | ~None_vt() => @(s, m, i) - end - - val (final_stats, final_missing, final_invalid) = read_entries(stats, missing, invalid) - val () = dirptr_close(dir) - in - @(final_stats, final_missing, final_invalid) - end - | ~None_vt() => @(stats, missing, invalid) - end - - val initial_stats = @{ - total_files = 0, - with_spdx = 0, - without_spdx = 0, - invalid_spdx = 0, - pmpl_license = 0, - other_licenses = 0 - } - - val (stats, missing, invalid) = scan_directory(repo_path, 0, initial_stats, list0_nil(), list0_nil()) -in - @{ - repo_path = repo_path, - stats = stats, - missing_files = missing, - invalid_files = invalid - } -end - -(* ========== Main Operation ========== *) - -(* -** Execute SPDX audit operation -** Audits all source files for SPDX license headers -*) -extern -fun execute_spdx_audit_operation( - repos: List0(string) -): List0(audit_result) = "ext#" - -implement -execute_spdx_audit_operation(repos) = let - fun process_repos( - rs: List0(string), - results: List0(audit_result) - ): List0(audit_result) = - case+ rs of - | list0_nil() => results - | list0_cons(repo, rest) => let - val result = audit_repository(repo) - in - process_repos(rest, list0_cons(result, results)) - end -in - process_repos(repos, list0_nil()) -end - -(* ========== C Exports ========== *) - -(* -** C-compatible audit result for FFI -*) -typedef c_audit_stats = @{ - total_files = int, - with_spdx = int, - without_spdx = int, - invalid_spdx = int, - pmpl_license = int, - other_licenses = int -} - -typedef c_audit_result = @{ - repo_path = string, - stats = c_audit_stats, - compliance_percent = int -} - -extern -fun c_spdx_audit( - base_dir: string, - max_depth: int -): @{ - total_repos = int, - results = ptr -} = "ext#" - -implement -c_spdx_audit(base_dir, max_depth) = let - (* Scan for repositories *) - val repos = find_git_repos(base_dir, max_depth) - - (* Execute audit *) - val results = execute_spdx_audit_operation(repos) - - (* Calculate compliance percentages *) - fun compute_compliance(stats: audit_stats): int = - if stats.total_files = 0 then 100 - else (stats.with_spdx * 100) / stats.total_files - - (* Convert to C results *) - fun to_c_results( - rs: List0(audit_result), - acc: List0(c_audit_result) - ): List0(c_audit_result) = - case+ rs of - | list0_nil() => acc - | list0_cons(r, rest) => let - val c_result = @{ - repo_path = r.repo_path, - stats = r.stats, - compliance_percent = compute_compliance(r.stats) - } - in - to_c_results(rest, list0_cons(c_result, acc)) - end - - val c_results = to_c_results(results, list0_nil()) -in - @{ - total_repos = list0_length(c_results), - results = $UNSAFE.cast{ptr}(c_results) - } -end diff --git a/scaffoldia/repo-batcher/src/ats2/operations/types.dats b/scaffoldia/repo-batcher/src/ats2/operations/types.dats deleted file mode 100644 index 71526fe7..00000000 --- a/scaffoldia/repo-batcher/src/ats2/operations/types.dats +++ /dev/null @@ -1,261 +0,0 @@ -(* -** SPDX-License-Identifier: PMPL-1.0-or-later -** -** Operation type definitions with dependent type proofs -** for formally verified batch repository operations -*) - -#include "share/atspre_define.hats" -#include "share/atspre_staload.hats" - -staload "libats/SATS/stringbuf.sats" - -(* ========== String validation proofs ========== *) - -(* Proof that a string is non-empty *) -abstype nonempty_string(s:string) = string - -(* Proof that a string is a valid SPDX identifier *) -abstype spdx_id(s:string) = string - -(* Proof that a path exists *) -abstype existing_path(p:string) = string - -(* Proof that a path is a valid git repository *) -abstype git_repo(p:string) = string - -(* ========== Repository target ========== *) - -datatype repo_target = - | RepoList of List0(string) (* Explicit list of repos *) - | RepoFile of string (* Path to file containing repo list *) - | RepoPattern of string (* Pattern matching (@all-repos, @rsr-*) *) - | RepoDirectory of string (* Scan directory for repos *) - -(* ========== Operation types ========== *) - -datatype operation_result = - | OpSuccess of string (* Success message *) - | OpFailure of string (* Error message *) - | OpSkipped of string (* Skipped with reason *) - -datatype backup_policy = - | NoBackup (* No backup required *) - | RequireBackup of string (* Backup to specified directory *) - | AutoBackup (* Automatic backup location *) - -datatype operation_mode = - | DryRun (* Preview only, no changes *) - | Execute (* Execute for real *) - | Interactive (* Prompt before each repo *) - -(* -** License Update Operation -** Replaces license headers and LICENSE files across repositories -** -** Invariants (enforced by dependent types): -** - old_license must be a valid SPDX identifier -** - new_license must be a valid SPDX identifier -** - backup_dir must exist if backup is required -*) -datatype license_update_op( - old:string, new:string -) = - | LicenseUpdate(old, new) of ( - spdx_id(old), (* Validated old license *) - spdx_id(new), (* Validated new license *) - backup_policy, (* Backup strategy *) - operation_mode (* Execution mode *) - ) - -(* -** File Replace Operation -** Replaces files matching a pattern with new content -** -** Invariants: -** - pattern must be a non-empty string -** - replacement_path must exist -** - No circular replacements (file A → file B → file A) -*) -datatype file_replace_op( - pattern:string, replacement:string -) = - | FileReplace(pattern, replacement) of ( - nonempty_string(pattern), (* Valid pattern *) - existing_path(replacement), (* Replacement file exists *) - backup_policy, (* Backup strategy *) - operation_mode (* Execution mode *) - ) - -(* -** Git Batch Sync Operation -** Performs git add, commit, and push across multiple repos -** -** Invariants: -** - All targets must be valid git repositories -** - Commit message must be non-empty -** - Remote must be reachable (checked at runtime) -*) -datatype git_sync_op( - msg:string -) = - | GitBatchSync(msg) of ( - nonempty_string(msg), (* Valid commit message *) - int, (* Parallel job count *) - int, (* Max depth for repo search *) - operation_mode (* Execution mode *) - ) - -(* -** Workflow Update Operation -** Updates GitHub Actions workflow files with validation -** -** Invariants: -** - workflow_file must exist -** - action_sha must be a valid git commit hash (40 hex chars) -** - YAML must be valid (checked at runtime) -*) -datatype workflow_update_op( - file:string, action:string, sha:string -) = - | WorkflowUpdate(file, action, sha) of ( - existing_path(file), (* Workflow file exists *) - nonempty_string(action), (* Action name *) - nonempty_string(sha), (* Valid SHA *) - backup_policy, (* Backup strategy *) - operation_mode (* Execution mode *) - ) - -(* -** Custom Operation -** Executes operation from template file -** -** Invariants: -** - template_path must exist -** - template must be valid TOML (checked at runtime) -*) -datatype custom_op( - template:string -) = - | CustomOp(template) of ( - existing_path(template), (* Template exists *) - List0(@(string, string)), (* Arguments *) - operation_mode (* Execution mode *) - ) - -(* -** Unified operation type -** Tagged union of all operation types -*) -datatype batch_operation = - | OpLicenseUpdate of [old:string, new:string] license_update_op(old, new) - | OpFileReplace of [pat:string, repl:string] file_replace_op(pat, repl) - | OpGitSync of [msg:string] git_sync_op(msg) - | OpWorkflowUpdate of [file:string, action:string, sha:string] - workflow_update_op(file, action, sha) - | OpCustom of [tmpl:string] custom_op(tmpl) - -(* ========== Operation context ========== *) - -typedef operation_context = @{ - targets = repo_target, (* Target repositories *) - dry_run = bool, (* Preview mode *) - parallel_jobs = int, (* Number of parallel workers *) - log_path = string, (* Log file path *) - backup_dir = string (* Backup directory *) -} - -(* ========== Operation result with proofs ========== *) - -typedef batch_result = @{ - success_count = [n:nat] int(n), (* Number of successful operations *) - failure_count = [n:nat] int(n), (* Number of failed operations *) - skipped_count = [n:nat] int(n), (* Number of skipped operations *) - results = List0(operation_result), (* Detailed results *) - rollback_info = Option(string) (* Rollback information if needed *) -} - -(* ========== Validation functions ========== *) - -(* -** Validates that a string is a valid SPDX identifier -** Returns Some(spdx_id) if valid, None otherwise -*) -fun validate_spdx_id(s: string): Option(spdx_id(s)) - -(* -** Validates that a path exists on filesystem -** Returns Some(existing_path) if valid, None otherwise -*) -fun validate_path_exists(p: string): Option(existing_path(p)) - -(* -** Validates that a path is a git repository -** Returns Some(git_repo) if valid, None otherwise -*) -fun validate_git_repo(p: string): Option(git_repo(p)) - -(* -** Validates that a string is non-empty -** Returns Some(nonempty_string) if valid, None otherwise -*) -fun validate_nonempty(s: string): Option(nonempty_string(s)) - -(* ========== Operation constructors with validation ========== *) - -(* -** Creates a license update operation with validation -** Returns None if validation fails -*) -fun make_license_update_op( - old: string, - new: string, - backup: backup_policy, - mode: operation_mode -): Option([old:string, new:string] license_update_op(old, new)) - -(* -** Creates a file replace operation with validation -** Returns None if validation fails -*) -fun make_file_replace_op( - pattern: string, - replacement: string, - backup: backup_policy, - mode: operation_mode -): Option([pat:string, repl:string] file_replace_op(pat, repl)) - -(* -** Creates a git sync operation with validation -** Returns None if validation fails -*) -fun make_git_sync_op( - msg: string, - parallel: int, - depth: int, - mode: operation_mode -): Option([msg:string] git_sync_op(msg)) - -(* ========== Operation execution ========== *) - -(* -** Executes a batch operation across target repositories -** Returns batch_result with detailed success/failure information -** -** Proof obligations: -** - If mode is DryRun, no filesystem changes are made -** - If backup is required, backups are created before any changes -** - Operation is atomic per repository (all changes or none) -*) -fun execute_batch_operation( - op: batch_operation, - ctx: operation_context -): batch_result - -(* -** Rolls back a batch operation using rollback information -** Returns true if rollback succeeded, false otherwise -*) -fun rollback_batch_operation( - rollback_info: string -): bool diff --git a/scaffoldia/repo-batcher/src/ats2/operations/workflow_update.dats b/scaffoldia/repo-batcher/src/ats2/operations/workflow_update.dats deleted file mode 100644 index 50ce0ee5..00000000 --- a/scaffoldia/repo-batcher/src/ats2/operations/workflow_update.dats +++ /dev/null @@ -1,362 +0,0 @@ -(* -** SPDX-License-Identifier: PMPL-1.0-or-later -** -** Workflow Update Operation Implementation -** Updates GitHub Actions workflows with SHA pinning -** Ensures all actions use commit SHAs instead of tags for security -*) - -#include "share/atspre_define.hats" -#include "share/atspre_staload.hats" - -staload "libats/libc/SATS/stdio.sats" -staload "libats/libc/SATS/dirent.sats" - -staload "./types.dats" -staload "../utils/string_utils.sats" - -(* ========== SHA Pinning Database ========== *) - -(* -** Standard GitHub Actions with their SHA pins (2026-02-04) -** Format: (action_ref, version_tag, commit_sha) -*) -typedef action_pin = @{ - action = string, (* e.g., "actions/checkout" *) - version = string, (* e.g., "v4" *) - sha = string (* Full commit SHA *) -} - -val sha_pins: List0(action_pin) = list0_make( - @{action = "actions/checkout", version = "v4", sha = "34e114876b0b11c390a56381ad16ebd13914f8d5"}, - @{action = "actions/checkout", version = "v5", sha = "93cb6efe18208431cddfb8368fd83d5badbf9bfd"}, - @{action = "github/codeql-action/init", version = "v3", sha = "6624720a57d4c312633c7b953db2f2da5bcb4c3a"}, - @{action = "github/codeql-action/autobuild", version = "v3", sha = "6624720a57d4c312633c7b953db2f2da5bcb4c3a"}, - @{action = "github/codeql-action/analyze", version = "v3", sha = "6624720a57d4c312633c7b953db2f2da5bcb4c3a"}, - @{action = "ossf/scorecard-action", version = "v2.4.0", sha = "62b2cac7ed8198b15735ed49ab1e5cf35480ba46"}, - @{action = "trufflesecurity/trufflehog", version = "main", sha = "7ee2e0fdffec27d19ccbb8fb3dcf8a83b9d7f9e8"}, - @{action = "dtolnay/rust-toolchain", version = "stable", sha = "4be9e76fd7c4901c61fb841f559994984270fce7"}, - @{action = "Swatinem/rust-cache", version = "v2", sha = "779680da715d629ac1d338a641029a2f4372abb5"}, - @{action = "codecov/codecov-action", version = "v5", sha = "671740ac38dd9b0130fbe1cec585b89eea48d3de"}, - @{action = "editorconfig-checker/action-editorconfig-checker", version = "main", sha = "4054fa83a075fdf090bd098bdb1c09aaf64a4169"}, - @{action = "slsa-framework/slsa-github-generator", version = "v2.1.0", sha = "f7dd8c54c2067bafc12ca7a55595d5ee9b75204a"}, - @{action = "webfactory/ssh-agent", version = "v0.9.0", sha = "dc588b651fe13675774614f8e6a936a468676387"}, - @{action = "ocaml/setup-ocaml", version = "v3", sha = "dec6499fef64fc5d7ed43d43a87251b7b1c306f5"}, - @{action = "softprops/action-gh-release", version = "v2", sha = "a06a81a03ee405af7f2048a818ed3f03bbf83c7b"}, - @{action = "actions/configure-pages", version = "v5", sha = "983d7736d9b0ae728b81ab479565c72886d7745b"}, - @{action = "actions/jekyll-build-pages", version = "v1", sha = "44a6e6beabd48582f863aeeb6cb2151cc1716697"}, - @{action = "actions/upload-pages-artifact", version = "v3", sha = "56afc609e74202658d3ffba0e8f6dda462b719fa"}, - @{action = "actions/deploy-pages", version = "v4", sha = "d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e"}, - @{action = "ruby/setup-ruby", version = "v1", sha = "09a7688d3b55cf0e976497ff046b70949eeaccfd"} -) - -(* ========== File Operations ========== *) - -fun read_file_contents(path: string): Option(string) = let - val file = fileref_open_opt(path, file_mode_r) -in - case+ file of - | ~Some_vt(f) => let - val content = fileref_get_file_string(f) - val () = fileref_close(f) - in - Some(content) - end - | ~None_vt() => None() -end - -fun write_file_contents(path: string, content: string): bool = let - val file = fileref_open_opt(path, file_mode_w) -in - case+ file of - | ~Some_vt(f) => let - val () = fileref_put_string(f, content) - val () = fileref_close(f) - in - true - end - | ~None_vt() => false -end - -(* ========== Workflow Analysis ========== *) - -(* -** Find SHA pin for action reference -** Looks up action@version and returns the commit SHA -*) -fun find_sha_pin(action_ref: string): Option(string) = let - (* Parse action@version *) - val at_pos = string_index_of(action_ref, "@") -in - if at_pos < 0 then None() - else let - val action = string_prefix(action_ref, at_pos) - val version = string_suffix(action_ref, at_pos + 1) - - fun loop(pins: List0(action_pin)): Option(string) = - case+ pins of - | list0_nil() => None() - | list0_cons(pin, rest) => - if string_equal(pin.action, action) && string_equal(pin.version, version) - then Some(pin.sha) - else loop(rest) - in - loop(sha_pins) - end -end - -(* -** Check if action reference is already SHA-pinned -** Returns true if uses full commit SHA (40 hex chars) -*) -fun is_sha_pinned(action_ref: string): bool = let - val at_pos = string_index_of(action_ref, "@") -in - if at_pos < 0 then false - else let - val ref_part = string_suffix(action_ref, at_pos + 1) - val len = string_length(ref_part) - in - (* SHA must be 40 hex characters *) - if len != 40 then false - else let - fun is_hex_char(c: char): bool = - (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') - - fun check_all_hex(s: string, i: int): bool = - if i >= len then true - else if is_hex_char(string_get_at(s, i)) then check_all_hex(s, i + 1) - else false - in - check_all_hex(ref_part, 0) - end - end -end - -(* -** Replace action reference with SHA-pinned version -** e.g., "actions/checkout@v4" -> "actions/checkout@34e114876b0b..." -** Preserves comment showing original version -*) -fun pin_action_reference(line: string): string = let - val uses_pos = string_index_of(line, "uses:") -in - if uses_pos < 0 then line (* Not a uses: line *) - else let - (* Extract the action reference *) - val after_uses = string_suffix(line, uses_pos + 5) - val trimmed = string_trim(after_uses) - - (* Check if already pinned *) - in - if is_sha_pinned(trimmed) then line - else - case+ find_sha_pin(trimmed) of - | Some(sha) => let - val at_pos = string_index_of(trimmed, "@") - val action = string_prefix(trimmed, at_pos) - val version = string_suffix(trimmed, at_pos + 1) - val indentation = string_prefix(line, uses_pos) - val pinned = action + "@" + sha - val comment = " # " + version - in - indentation + "uses: " + pinned + comment - end - | None() => line (* No pin found, leave unchanged *) - end -end - -(* -** Process entire workflow file -** Updates all action references with SHA pins -*) -fun update_workflow_file(content: string): string = let - val lines = string_split(content, "\n") - - fun process_lines(ls: List0(string)): List0(string) = - case+ ls of - | list0_nil() => list0_nil() - | list0_cons(line, rest) => - list0_cons(pin_action_reference(line), process_lines(rest)) - - val updated_lines = process_lines(lines) -in - string_join(updated_lines, "\n") -end - -(* ========== Repository Operations ========== *) - -(* -** Find all workflow files in .github/workflows/ -** Returns list of relative paths -*) -fun find_workflow_files(repo_path: string): List0(string) = let - val workflows_dir = repo_path + "/.github/workflows" - - (* Check if directory exists *) - fun dir_exists(path: string): bool = let - val dir = dirptr_open_opt(path) - in - case+ dir of - | ~Some_vt(d) => let val () = dirptr_close(d) in true end - | ~None_vt() => false - end -in - if ~dir_exists(workflows_dir) then list0_nil() - else let - (* List all .yml and .yaml files *) - val dir = dirptr_open_exn(workflows_dir) - - fun read_entries(acc: List0(string)): List0(string) = let - val entry = dirptr_read_opt(dir) - in - case+ entry of - | ~Some_vt(ent) => let - val name = dirent_get_name(ent) - val is_yaml = string_has_suffix(name, ".yml") || string_has_suffix(name, ".yaml") - in - if is_yaml - then read_entries(list0_cons(workflows_dir + "/" + name, acc)) - else read_entries(acc) - end - | ~None_vt() => acc - end - - val files = read_entries(list0_nil()) - val () = dirptr_close(dir) - in - files - end -end - -(* -** Update all workflows in repository -** Returns number of files updated -*) -fun update_repository_workflows( - repo_path: string, - backup: bool, - dry_run: bool -): @(int, int) = let (* (updated_count, total_count) *) - val workflow_files = find_workflow_files(repo_path) - - fun process_file(file_path: string): bool = let - val content_opt = read_file_contents(file_path) - in - case+ content_opt of - | Some(content) => let - val updated = update_workflow_file(content) - val has_changes = ~string_equal(content, updated) - in - if has_changes then - if dry_run then true - else let - (* Create backup if requested *) - val backup_result = - if backup then let - val backup_path = file_path + ".backup" - in - write_file_contents(backup_path, content) - end - else true - - (* Write updated content *) - val write_result = - if backup_result then write_file_contents(file_path, updated) - else false - in - write_result - end - else false - end - | None() => false - end - - fun count_updates(files: List0(string), updated: int, total: int): @(int, int) = - case+ files of - | list0_nil() => @(updated, total) - | list0_cons(file, rest) => - if process_file(file) - then count_updates(rest, updated + 1, total + 1) - else count_updates(rest, updated, total + 1) -in - count_updates(workflow_files, 0, 0) -end - -(* ========== Main Operation ========== *) - -(* -** Execute workflow update operation -** Updates GitHub Actions with SHA pinning across repositories -*) -extern -fun execute_workflow_update_operation( - repos: List0(string), - backup: bool, - dry_run: bool -): batch_result = "ext#" - -implement -execute_workflow_update_operation(repos, backup, dry_run) = let - fun process_repos( - rs: List0(string), - success: int, - failed: int, - results: List0(operation_result) - ): batch_result = - case+ rs of - | list0_nil() => - @{ - total = success + failed, - successful = success, - failed = failed, - results = results - } - | list0_cons(repo, rest) => let - val (updated, total) = update_repository_workflows(repo, backup, dry_run) - val result = - if updated > 0 || total > 0 then - @{ - repo_path = repo, - success = true, - message = "Updated " + tostring_int(updated) + "/" + tostring_int(total) + " workflows", - files_affected = updated - } - else - @{ - repo_path = repo, - success = true, - message = "No workflows found", - files_affected = 0 - } - in - process_repos(rest, success + 1, failed, list0_cons(result, results)) - end -in - process_repos(repos, 0, 0, list0_nil()) -end - -(* ========== C Exports ========== *) - -extern -fun c_workflow_update( - base_dir: string, - max_depth: int, - backup: int, - dry_run: int -): c_batch_result = "ext#" - -implement -c_workflow_update(base_dir, max_depth, backup, dry_run) = let - (* Scan for repositories *) - val repos = find_git_repos(base_dir, max_depth) - - (* Execute workflow update *) - val result = execute_workflow_update_operation( - repos, - backup != 0, - dry_run != 0 - ) -in - batch_result_to_c(result) -end diff --git a/scaffoldia/repo-batcher/src/ats2/utils/string_utils.dats b/scaffoldia/repo-batcher/src/ats2/utils/string_utils.dats deleted file mode 100644 index 3f4b0d53..00000000 --- a/scaffoldia/repo-batcher/src/ats2/utils/string_utils.dats +++ /dev/null @@ -1,331 +0,0 @@ -(* -** SPDX-License-Identifier: PMPL-1.0-or-later -** -** String Utility Functions -** Core string manipulation helpers for batch operations -*) - -#include "share/atspre_define.hats" -#include "share/atspre_staload.hats" - -(* ========== String Search ========== *) - -(* -** Finds index of needle in haystack -** Returns -1 if not found, position >= 0 if found -*) -implement string_index_of(haystack, needle) = let - val haystack_len = string_length(haystack) - val needle_len = string_length(needle) - - fun loop(i: int): int = - if i + needle_len > haystack_len then - ~1 // Not found - else let - val match = string_equal_at(haystack, i, needle) - in - if match then i - else loop(i + 1) - end -in - if needle_len = 0 then 0 // Empty needle found at position 0 - else if needle_len > haystack_len then ~1 // Needle longer than haystack - else loop(0) -end - -(* -** Checks if haystack starts with needle at position i -*) -and string_equal_at(haystack: string, i: int, needle: string): bool = let - val needle_len = string_length(needle) - - fun loop(j: int): bool = - if j >= needle_len then true - else if haystack[i + j] != needle[j] then false - else loop(j + 1) -in - loop(0) -end - -(* -** Finds last index of character in string -** Returns -1 if not found -*) -implement string_rindex_of(haystack, needle) = let - val len = string_length(haystack) - - fun loop(i: int): int = - if i < 0 then ~1 - else if haystack[i] = needle then i - else loop(i - 1) -in - loop(len - 1) -end - -(* -** Checks if haystack contains needle -*) -implement string_contains(haystack, needle) = - string_index_of(haystack, needle) >= 0 - -(* ========== String Extraction ========== *) - -(* -** Extracts substring starting at position with given length -** Returns empty string if out of bounds -*) -implement string_substring(s, start, len) = let - val s_len = string_length(s) -in - if start < 0 orelse start >= s_len then - "" - else if len <= 0 then - "" - else let - val actual_len = min(len, s_len - start) - val buf = string_make_substring(s, start, actual_len) - in - buf - end -end - -(* -** Gets suffix of string starting at position -*) -implement string_suffix(s, start) = let - val len = string_length(s) -in - if start < 0 orelse start >= len then "" - else string_substring(s, start, len - start) -end - -(* -** Gets prefix of string up to length -*) -fun string_prefix(s: string, len: int): string = - string_substring(s, 0, len) - -(* ========== String Trimming ========== *) - -(* -** Checks if character is whitespace -*) -fun is_whitespace(c: char): bool = - c = ' ' orelse c = '\t' orelse c = '\n' orelse c = '\r' - -(* -** Trims whitespace from start of string -*) -fun string_ltrim(s: string): string = let - val len = string_length(s) - - fun find_start(i: int): int = - if i >= len then len - else if is_whitespace(s[i]) then find_start(i + 1) - else i - - val start = find_start(0) -in - if start >= len then "" - else string_suffix(s, start) -end - -(* -** Trims whitespace from end of string -*) -fun string_rtrim(s: string): string = let - val len = string_length(s) - - fun find_end(i: int): int = - if i < 0 then 0 - else if is_whitespace(s[i]) then find_end(i - 1) - else i + 1 - - val end_pos = find_end(len - 1) -in - if end_pos <= 0 then "" - else string_prefix(s, end_pos) -end - -(* -** Trims whitespace from both ends of string -*) -implement string_trim(s) = - string_rtrim(string_ltrim(s)) - -(* ========== String Replacement ========== *) - -(* -** Replaces first occurrence of old with new -*) -fun string_replace_first(str: string, old: string, new: string): string = let - val idx = string_index_of(str, old) -in - if idx < 0 then - str // Not found, return original - else let - val old_len = string_length(old) - val before = string_prefix(str, idx) - val after = string_suffix(str, idx + old_len) - in - before + new + after - end -end - -(* -** Replaces all occurrences of old with new -*) -implement string_replace(str, old, new) = let - val old_len = string_length(old) - - fun loop(s: string, acc: string): string = let - val idx = string_index_of(s, old) - in - if idx < 0 then - acc + s // No more occurrences - else let - val before = string_prefix(s, idx) - val after = string_suffix(s, idx + old_len) - val new_acc = acc + before + new - in - loop(after, new_acc) - end - end -in - if old_len = 0 then str // Don't replace empty string - else loop(str, "") -end - -(* ========== String Building ========== *) - -(* -** Joins list of strings with separator -*) -fun string_join(strings: List0(string), sep: string): string = let - fun loop(lst: List0(string), first: bool, acc: string): string = - case+ lst of - | list_nil() => acc - | list_cons(s, rest) => - if first then - loop(rest, false, s) - else - loop(rest, false, acc + sep + s) -in - loop(strings, true, "") -end - -(* -** Splits string by separator -*) -fun string_split(s: string, sep: string): List0(string) = let - val sep_len = string_length(sep) - - fun loop(str: string, acc: List0(string)): List0(string) = let - val idx = string_index_of(str, sep) - in - if idx < 0 then - list_cons(str, acc) // Last part - else let - val before = string_prefix(str, idx) - val after = string_suffix(str, idx + sep_len) - in - loop(after, list_cons(before, acc)) - end - end - - val parts = loop(s, list_nil()) - - // Reverse to get correct order - fun reverse(lst: List0(string), acc: List0(string)): List0(string) = - case+ lst of - | list_nil() => acc - | list_cons(x, rest) => reverse(rest, list_cons(x, acc)) -in - reverse(parts, list_nil()) -end - -(* ========== Integer Conversion ========== *) - -(* -** Converts integer to string -*) -implement tostring_int(n) = let - fun int2string(i: int): string = - if i = 0 then "0" - else if i < 0 then "-" + int2string_pos(~i) - else int2string_pos(i) - - and int2string_pos(i: int): string = - if i = 0 then "" - else int2string_pos(i / 10) + digit2char(i mod 10) - - and digit2char(d: int): string = - case+ d of - | 0 => "0" | 1 => "1" | 2 => "2" | 3 => "3" | 4 => "4" - | 5 => "5" | 6 => "6" | 7 => "7" | 8 => "8" | 9 => "9" - | _ => "?" -in - int2string(n) -end - -(* ========== Helper Functions ========== *) - -fun min(a: int, b: int): int = - if a < b then a else b - -fun max(a: int, b: int): int = - if a > b then a else b - -(* ========== String Validation ========== *) - -(* -** Checks if string is empty -*) -fun string_is_empty(s: string): bool = - string_length(s) = 0 - -(* -** Checks if string is non-empty -*) -fun string_is_nonempty(s: string): bool = - string_length(s) > 0 - -(* -** Checks if string contains only whitespace -*) -fun string_is_whitespace(s: string): bool = let - val len = string_length(s) - - fun loop(i: int): bool = - if i >= len then true - else if is_whitespace(s[i]) then loop(i + 1) - else false -in - loop(0) -end - -(* ========== String Comparison ========== *) - -(* -** Case-insensitive string comparison -*) -fun string_equal_ci(s1: string, s2: string): bool = let - val len1 = string_length(s1) - val len2 = string_length(s2) -in - if len1 != len2 then false - else let - fun loop(i: int): bool = - if i >= len1 then true - else if tolower(s1[i]) != tolower(s2[i]) then false - else loop(i + 1) - in - loop(0) - end -end - -and tolower(c: char): char = - if c >= 'A' && c <= 'Z' then - char_of_int(int_of_char(c) + 32) - else - c diff --git a/scaffoldia/repo-batcher/src/ats2/utils/string_utils.sats b/scaffoldia/repo-batcher/src/ats2/utils/string_utils.sats deleted file mode 100644 index 36485eab..00000000 --- a/scaffoldia/repo-batcher/src/ats2/utils/string_utils.sats +++ /dev/null @@ -1,47 +0,0 @@ -(* -** SPDX-License-Identifier: PMPL-1.0-or-later -** -** String Utility Functions - Interface -*) - -(* ========== String Search ========== *) - -fun string_index_of(haystack: string, needle: string): int -fun string_rindex_of(haystack: string, needle: char): int -fun string_contains(haystack: string, needle: string): bool - -(* ========== String Extraction ========== *) - -fun string_substring(s: string, start: int, len: int): string -fun string_suffix(s: string, start: int): string -fun string_prefix(s: string, len: int): string - -(* ========== String Trimming ========== *) - -fun string_trim(s: string): string -fun string_ltrim(s: string): string -fun string_rtrim(s: string): string - -(* ========== String Replacement ========== *) - -fun string_replace(str: string, old: string, new: string): string -fun string_replace_first(str: string, old: string, new: string): string - -(* ========== String Building ========== *) - -fun string_join(strings: List0(string), sep: string): string -fun string_split(s: string, sep: string): List0(string) - -(* ========== Integer Conversion ========== *) - -fun tostring_int(n: int): string - -(* ========== String Validation ========== *) - -fun string_is_empty(s: string): bool -fun string_is_nonempty(s: string): bool -fun string_is_whitespace(s: string): bool - -(* ========== String Comparison ========== *) - -fun string_equal_ci(s1: string, s2: string): bool diff --git a/scaffoldia/repo-batcher/src/ats2/validation/spdx.dats b/scaffoldia/repo-batcher/src/ats2/validation/spdx.dats deleted file mode 100644 index e70c3520..00000000 --- a/scaffoldia/repo-batcher/src/ats2/validation/spdx.dats +++ /dev/null @@ -1,133 +0,0 @@ -(* -** SPDX-License-Identifier: PMPL-1.0-or-later -** -** SPDX license identifier validation -** Validates license identifiers against SPDX specification -*) - -#include "share/atspre_define.hats" -#include "share/atspre_staload.hats" - -staload "../operations/types.dats" - -(* ========== SPDX License List ========== *) - -(* -** Common SPDX license identifiers -** Full list: https://spdx.org/licenses/ -*) -val common_spdx_licenses = @[ - "PMPL-1.0-or-later", - "MIT", - "Apache-2.0", - "GPL-3.0-only", - "GPL-3.0-or-later", - "LGPL-3.0-only", - "LGPL-3.0-or-later", - "BSD-2-Clause", - "BSD-3-Clause", - "ISC", - "MPL-2.0", - "AGPL-3.0-only", - "AGPL-3.0-or-later", - "Unlicense", - "0BSD" -] : List0(string) - -(* -** Validates if a string is a valid SPDX identifier -** Returns true if valid, false otherwise -*) -fun is_valid_spdx(s: string): bool = let - fun check_list(licenses: List0(string)): bool = - case+ licenses of - | list_nil() => false - | list_cons(lic, rest) => - if s = lic then true - else check_list(rest) -in - if string_is_empty(s) then false - else check_list(common_spdx_licenses) -end - -(* -** Validates SPDX identifier and returns proof type -** Returns Some(spdx_id) if valid, None otherwise -*) -implement validate_spdx_id(s) = - if is_valid_spdx(s) then Some(s) - else None() - -(* -** Extracts SPDX identifier from header line -** Format: "// SPDX-License-Identifier: MIT" -** Returns extracted license or empty string -*) -fun extract_spdx_from_header(line: string): string = let - val prefix = "SPDX-License-Identifier:" - val idx = string_index_of(line, prefix) -in - if idx >= 0 then let - val start = idx + string_length(prefix) - val rest = string_suffix(line, start) - val trimmed = string_trim(rest) - in - trimmed - end - else "" -end - -(* -** Checks if a file contains SPDX header -** Returns Some(license) if found, None otherwise -*) -fun find_spdx_in_file(path: string): Option(string) = let - val file = fileref_open_opt(path, file_mode_r) -in - case+ file of - | ~Some_vt(f) => let - fun loop(): Option(string) = let - val line = fileref_get_line_string(f) - in - if string_is_empty(line) then None() - else let - val spdx = extract_spdx_from_header(line) - in - if string_is_empty(spdx) then loop() - else Some(spdx) - end - end - val result = loop() - val () = fileref_close(f) - in - result - end - | ~None_vt() => None() -end - -(* -** Generates SPDX header for given license and comment style -** comment_style: "//", "#", "--", etc. -*) -fun generate_spdx_header(license: string, comment_style: string): string = - comment_style + " SPDX-License-Identifier: " + license + "\n" - -(* -** Determines comment style for file extension -** Returns comment prefix ("//", "#", "--", etc.) -*) -fun get_comment_style_for_ext(ext: string): string = - case+ ext of - | ".rs" => "//" - | ".v" => "//" - | ".dats" => "(*" - | ".sats" => "(*" - | ".idr" => "--" - | ".zig" => "//" - | ".scm" => ";;" - | ".toml" => "#" - | ".yml" => "#" - | ".yaml" => "#" - | ".md" => "