diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 42a3170..2ad2ce9 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -1,9 +1,12 @@ -name: Deploy GitHub Pages +name: Deploy Hugo Documentation on: push: branches: - main + paths: + - 'hugo-docs/**' + - '.github/workflows/pages.yml' workflow_dispatch: permissions: @@ -15,26 +18,53 @@ concurrency: group: "pages" cancel-in-progress: false +# Default shell +defaults: + run: + shell: bash + jobs: build: runs-on: ubuntu-latest + env: + HUGO_VERSION: "0.147.0" steps: - - name: Checkout + - name: Checkout repository uses: actions/checkout@v4 - - - name: Setup Pages - uses: actions/configure-pages@v5 with: - enablement: true + fetch-depth: 0 # Full history for .GitInfo / LastMod - - name: Build with Jekyll - uses: actions/jekyll-build-pages@v1 + - name: Setup Go (required for Hugo modules) + uses: actions/setup-go@v5 with: - source: ./ - destination: ./_site + go-version: "1.22" - - name: Upload artifact + - name: Install Hugo CLI + run: | + wget -O ${{ runner.temp }}/hugo.deb \ + https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb + sudo dpkg -i ${{ runner.temp }}/hugo.deb + + - name: Setup GitHub Pages + id: pages + uses: actions/configure-pages@v5 + + - name: Download Hugo modules + working-directory: ./hugo-docs + run: hugo mod tidy + + - name: Build Hugo site + working-directory: ./hugo-docs + env: + HUGO_ENVIRONMENT: production + # Use the Pages URL configured above + HUGO_BASEURL: ${{ steps.pages.outputs.base_url }}/ + run: hugo --gc --minify + + - name: Upload Pages artifact uses: actions/upload-pages-artifact@v3 + with: + path: ./hugo-docs/public deploy: environment: diff --git a/.gitignore b/.gitignore index ed26ee5..a42fc15 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,8 @@ _site/ .venv/ venv/ docs/test-report.html + +# Hugo build artifacts +hugo-docs/public/ +hugo-docs/resources/ +hugo-docs/.hugo_build.lock diff --git a/README-INDEX.md b/README-INDEX.md index a7cb2de..0446a54 100644 --- a/README-INDEX.md +++ b/README-INDEX.md @@ -6,40 +6,63 @@ Welcome to the DevOps-OS documentation! This set of guides will help you use and | Guide | Description | |-------|-------------| -| [Creating DevOps-OS Using Dev Container](.devcontainer/DEVOPS-OS-README.md) | How to set up and customize the DevOps-OS development container | -| [DevOps-OS Quick Start Guide](.devcontainer/DEVOPS-OS-QUICKSTART.md) | Essential CLI commands for all functionality in the project | -| [Creating Customized GitHub Actions Templates](.devcontainer/GITHUB-ACTIONS-README.md) | How to generate and customize GitHub Actions workflows | -| [Creating Customized Jenkins Templates](.devcontainer/JENKINS-PIPELINE-README.md) | How to generate and customize Jenkins pipelines | -| [Creating Kubernetes Deployments](.devcontainer/KUBERNETES-DEPLOYMENT-README.md) | How to generate and manage Kubernetes deployment configurations | -| [Implementing CI/CD for Technology Stacks](.devcontainer/CICD-TECH-STACK-README.md) | How to implement CI/CD pipelines for specific technology stacks | -| [CI/CD Generators Usage Guide](.devcontainer/CI-CD-GENERATORS-USAGE.md) | Detailed options and examples for the CI/CD generators | +| [**CLI Commands Reference**](docs/CLI-COMMANDS-REFERENCE.md) | **Complete reference** — every option, input file, and output location for all CLI commands | +| [Getting Started](docs/GETTING-STARTED.md) | First pipeline in 5 minutes | +| [Quick Start Guide](docs/DEVOPS-OS-QUICKSTART.md) | Essential CLI commands for all functionality | +| [GitHub Actions Generator](docs/GITHUB-ACTIONS-README.md) | Generate and customize GitHub Actions workflows | +| [GitLab CI Generator](docs/GITLAB-CI-README.md) | Generate and customize GitLab CI pipelines | +| [Jenkins Pipeline Generator](docs/JENKINS-PIPELINE-README.md) | Generate and customize Jenkins pipelines | +| [ArgoCD / Flux CD Generator](docs/ARGOCD-README.md) | Generate ArgoCD and Flux CD GitOps configs | +| [SRE Configuration Generator](docs/SRE-CONFIGURATION-README.md) | Generate Prometheus, Grafana, SLO, and Alertmanager configs | +| [Kubernetes Deployment](docs/KUBERNETES-DEPLOYMENT-README.md) | Generate and manage Kubernetes deployment configurations | +| [CI/CD for Technology Stacks](docs/CICD-TECH-STACK-README.md) | Implement CI/CD for specific technology stacks | +| [DevOps-OS Dev Container](docs/DEVOPS-OS-README.md) | Set up and customize the dev container | + +## CLI Generator Quick Reference + +| Command | Default output | +|---------|---------------| +| `python -m cli.scaffold_gha` | `.github/workflows/-.yml` | +| `python -m cli.scaffold_gitlab` | `.gitlab-ci.yml` | +| `python -m cli.scaffold_jenkins` | `Jenkinsfile` | +| `python -m cli.scaffold_argocd` | `argocd/` directory | +| `python -m cli.scaffold_argocd --method flux` | `flux/` directory | +| `python -m cli.scaffold_sre` | `sre/` directory | +| `python -m cli.scaffold_devcontainer` | `.devcontainer/` directory | + +See [CLI Commands Reference](docs/CLI-COMMANDS-REFERENCE.md) for the full option tables, input files, and output path details. -## CI/CD Generator Tools +## Quick Start -DevOps-OS includes powerful generators for creating CI/CD configurations: +```bash +git clone https://github.com/cloudengine-labs/devops_os.git +cd devops_os +pip install -r cli/requirements.txt -1. **GitHub Actions Generator**: `github-actions-generator-improved.py` -2. **Jenkins Pipeline Generator**: `jenkins-pipeline-generator-improved.py` -3. **Kubernetes Config Generator**: `k8s-config-generator.py` -4. **Unified CI/CD Generator**: `generate-cicd.py` +# GitHub Actions complete pipeline +python -m cli.scaffold_gha --name my-app --languages python,javascript --type complete +# Output: .github/workflows/my-app-complete.yml -## Quick Start +# GitLab CI pipeline +python -m cli.scaffold_gitlab --name my-app --languages python +# Output: .gitlab-ci.yml -To quickly generate both GitHub Actions and Jenkins pipelines: - -```bash -.devcontainer/generate-cicd.py --name "My Project" --languages python,javascript --kubernetes +# SRE monitoring stack +python -m cli.scaffold_sre --name my-app --team platform +# Output: sre/ directory ``` -For more examples and detailed usage, see the [DevOps-OS Quick Start Guide](DEVOPS-OS-QUICKSTART.md). +For more examples and detailed usage, see the [Getting Started guide](docs/GETTING-STARTED.md). ## Features - **Multi-language Support**: Python, Java, JavaScript/TypeScript, Go -- **CI/CD Configuration**: GitHub Actions, Jenkins +- **CI/CD Configuration**: GitHub Actions, GitLab CI, Jenkins +- **GitOps**: ArgoCD, Flux CD +- **SRE Observability**: Prometheus alert rules, Grafana dashboards, SLO manifests, Alertmanager configs - **Kubernetes Deployment**: kubectl, kustomize, ArgoCD, Flux - **Customization**: Environment variables, command-line options, custom JSON configurations -- **Integration**: Container registries, credentials management, external services +- **AI Integration**: Claude MCP server, OpenAI function calling ## Getting Help diff --git a/README.md b/README.md index 33e5165..f300a56 100644 --- a/README.md +++ b/README.md @@ -103,26 +103,30 @@ python -m cli.scaffold_gha --name my-app --languages python --kubernetes --k8s-m ### 3 — Generate other pipelines & configs ```bash -# Jenkins pipeline +# Jenkins pipeline → Jenkinsfile python -m cli.scaffold_jenkins --name my-app --languages java --type complete -# GitLab CI pipeline +# GitLab CI pipeline → .gitlab-ci.yml python -m cli.scaffold_gitlab --name my-app --languages python,go --type complete -# ArgoCD / Flux GitOps configs +# ArgoCD GitOps configs → argocd/application.yaml + argocd/appproject.yaml python -m cli.scaffold_argocd --name my-app --repo https://github.com/myorg/my-app.git + +# Flux GitOps configs → flux/git-repository.yaml + flux/kustomization.yaml + flux/image-update-automation.yaml python -m cli.scaffold_argocd --name my-app --method flux --repo https://github.com/myorg/my-app.git -# SRE configs (Prometheus, Grafana, SLO) +# SRE configs (Prometheus, Grafana, SLO) → sre/ directory python -m cli.scaffold_sre --name my-app --team platform --slo-target 99.9 -# Dev container configuration +# Dev container configuration → .devcontainer/devcontainer.json + .devcontainer/devcontainer.env.json python -m cli.scaffold_devcontainer --languages python,go --cicd-tools docker,terraform --kubernetes-tools k9s,flux # Kubernetes manifests python kubernetes/k8s-config-generator.py --name my-app --image ghcr.io/myorg/my-app:v1 ``` +> See [CLI Commands Reference](docs/CLI-COMMANDS-REFERENCE.md) for the full option tables and every default output path. + ### 4 — Interactive wizard (all-in-one) ```bash @@ -234,6 +238,7 @@ You can also customize `.devcontainer/devcontainer.env.json` directly to enable | Guide | Description | |-------|-------------| | [🚀 Getting Started](docs/GETTING-STARTED.md) | Easy step-by-step guide — **start here** | +| [📖 CLI Commands Reference](docs/CLI-COMMANDS-REFERENCE.md) | **Complete reference** — every option, input file, and output location | | [📦 Dev Container Setup](docs/DEVOPS-OS-README.md) | Set up and customize the dev container | | [⚡ Quick Start Reference](docs/DEVOPS-OS-QUICKSTART.md) | Essential CLI commands for all features | | [⚙️ GitHub Actions Generator](docs/GITHUB-ACTIONS-README.md) | Generate and customize GitHub Actions workflows | diff --git a/docs/ARGOCD-README.md b/docs/ARGOCD-README.md index 7b91519..08c9d34 100644 --- a/docs/ARGOCD-README.md +++ b/docs/ARGOCD-README.md @@ -43,6 +43,10 @@ python -m cli.scaffold_argocd \ | `--rollouts` | off | Add an Argo Rollouts canary resource | | `--image` | `ghcr.io/myorg/my-app` | Image for Flux image automation | | `--output-dir` | `.` | Root directory for output files | +| `--allow-any-source-repo` | off | Add `*` to AppProject `sourceRepos` (grants access to any repo — use with caution) | + +All options can also be set via environment variables prefixed with `DEVOPS_OS_ARGOCD_` +(e.g. `DEVOPS_OS_ARGOCD_AUTO_SYNC=true`). ## ArgoCD Output Files diff --git a/docs/CLI-COMMANDS-REFERENCE.md b/docs/CLI-COMMANDS-REFERENCE.md new file mode 100644 index 0000000..70be0ac --- /dev/null +++ b/docs/CLI-COMMANDS-REFERENCE.md @@ -0,0 +1,687 @@ +# DevOps-OS CLI Commands Reference + +This page is the single source of truth for every DevOps-OS CLI command. +For each command you will find the **complete option table**, the **input files** it reads, and the **exact output files / directories** it writes. + +--- + +## Table of Contents + +- [Installation](#installation) +- [Command Overview](#command-overview) +- [scaffold_gha — GitHub Actions Generator](#scaffold_gha--github-actions-generator) +- [scaffold_gitlab — GitLab CI Generator](#scaffold_gitlab--gitlab-ci-generator) +- [scaffold_jenkins — Jenkins Pipeline Generator](#scaffold_jenkins--jenkins-pipeline-generator) +- [scaffold_argocd — ArgoCD / Flux CD Generator](#scaffold_argocd--argocd--flux-cd-generator) +- [scaffold_sre — SRE Config Generator](#scaffold_sre--sre-config-generator) +- [scaffold_devcontainer — Dev Container Generator](#scaffold_devcontainer--dev-container-generator) +- [devopsos — Unified CLI](#devopsos--unified-cli) +- [Environment Variable Reference](#environment-variable-reference) +- [Input File Formats](#input-file-formats) + +--- + +## Installation + +```bash +git clone https://github.com/cloudengine-labs/devops_os.git +cd devops_os +python -m venv .venv +source .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -r cli/requirements.txt +``` + +--- + +## Command Overview + +| Command | Invocation | Output location | +|---------|-----------|-----------------| +| GitHub Actions | `python -m cli.scaffold_gha` | `.github/workflows/-.yml` | +| GitLab CI | `python -m cli.scaffold_gitlab` | `.gitlab-ci.yml` | +| Jenkins | `python -m cli.scaffold_jenkins` | `Jenkinsfile` | +| ArgoCD | `python -m cli.scaffold_argocd` | `argocd/` directory | +| Flux CD | `python -m cli.scaffold_argocd --method flux` | `flux/` directory | +| SRE configs | `python -m cli.scaffold_sre` | `sre/` directory | +| Dev Container | `python -m cli.scaffold_devcontainer` | `.devcontainer/` directory | +| Interactive wizard | `python -m cli.devopsos` | varies (see below) | + +All generators also accept environment variables as an alternative to flags — +see [Environment Variable Reference](#environment-variable-reference). + +--- + +## scaffold_gha — GitHub Actions Generator + +Generates a GitHub Actions workflow YAML file. + +### Invocation + +```bash +python -m cli.scaffold_gha [options] +``` + +### Options + +| Option | Env var | Default | Description | +|--------|---------|---------|-------------| +| `--name NAME` | `DEVOPS_OS_GHA_NAME` | `DevOps-OS` | Workflow name | +| `--type TYPE` | `DEVOPS_OS_GHA_TYPE` | `complete` | Workflow type: `build` \| `test` \| `deploy` \| `complete` \| `reusable` | +| `--languages LANGS` | `DEVOPS_OS_GHA_LANGUAGES` | `python,javascript` | Comma-separated languages: `python`, `java`, `javascript`, `go` | +| `--kubernetes` | `DEVOPS_OS_GHA_KUBERNETES` | `false` | Include Kubernetes deployment steps | +| `--registry URL` | `DEVOPS_OS_GHA_REGISTRY` | `ghcr.io` | Container registry URL | +| `--k8s-method METHOD` | `DEVOPS_OS_GHA_K8S_METHOD` | `kubectl` | K8s deploy method: `kubectl` \| `kustomize` \| `argocd` \| `flux` | +| `--output DIR` | `DEVOPS_OS_GHA_OUTPUT` | `.github/workflows` | Output directory | +| `--custom-values FILE` | `DEVOPS_OS_GHA_CUSTOM_VALUES` | _(none)_ | Path to a custom values JSON file | +| `--image IMAGE` | `DEVOPS_OS_GHA_IMAGE` | `ghcr.io/yourorg/devops-os:latest` | DevOps-OS container image | +| `--branches BRANCHES` | `DEVOPS_OS_GHA_BRANCHES` | `main` | Comma-separated branches that trigger the workflow | +| `--matrix` | `DEVOPS_OS_GHA_MATRIX` | `false` | Enable matrix builds across OS/architectures | +| `--env-file FILE` | `DEVOPS_OS_GHA_ENV_FILE` | _(cli dir)_ | Path to `devcontainer.env.json` for tool selection | +| `--reusable` | `DEVOPS_OS_GHA_REUSABLE` | `false` | Generate a reusable workflow (also set by `--type reusable`) | + +### Input files (optional) + +| File | Flag | Purpose | +|------|------|---------| +| `devcontainer.env.json` | `--env-file` | Pre-loads language and tool selection from your dev container config | +| `custom-values.json` | `--custom-values` | Overrides any generated value (build timeouts, matrix config, etc.) | + +### Output files + +| File | Condition | Description | +|------|-----------|-------------| +| `/-.yml` | always | Generated GitHub Actions workflow (YAML) | + +**Default output path:** + +``` +.github/workflows/devops-os-complete.yml +``` + +The filename is built as `-.yml`. +Example: `--name "My App" --type build` → `.github/workflows/my-app-build.yml` + +### Examples + +```bash +# Complete CI/CD pipeline for Python + JavaScript, written to default location +python -m cli.scaffold_gha --name my-app --languages python,javascript --type complete +# Output: .github/workflows/my-app-complete.yml + +# Deploy pipeline with Kubernetes via ArgoCD +python -m cli.scaffold_gha --name my-app --type deploy --kubernetes --k8s-method argocd +# Output: .github/workflows/my-app-deploy.yml + +# Matrix build written to a custom directory +python -m cli.scaffold_gha --name my-app --matrix --output /tmp/workflows +# Output: /tmp/workflows/my-app-complete.yml + +# Reusable workflow +python -m cli.scaffold_gha --name shared --type reusable +# Output: .github/workflows/shared-reusable.yml +``` + +--- + +## scaffold_gitlab — GitLab CI Generator + +Generates a `.gitlab-ci.yml` pipeline file. + +### Invocation + +```bash +python -m cli.scaffold_gitlab [options] +``` + +### Options + +| Option | Env var | Default | Description | +|--------|---------|---------|-------------| +| `--name NAME` | `DEVOPS_OS_GITLAB_NAME` | `my-app` | Application / pipeline name | +| `--type TYPE` | `DEVOPS_OS_GITLAB_TYPE` | `complete` | Pipeline type: `build` \| `test` \| `deploy` \| `complete` | +| `--languages LANGS` | `DEVOPS_OS_GITLAB_LANGUAGES` | `python` | Comma-separated: `python`, `java`, `javascript`, `go` | +| `--kubernetes` | `DEVOPS_OS_GITLAB_KUBERNETES` | `false` | Add a Kubernetes deploy stage | +| `--k8s-method METHOD` | `DEVOPS_OS_GITLAB_K8S_METHOD` | `kubectl` | K8s method: `kubectl` \| `kustomize` \| `argocd` \| `flux` | +| `--output FILE` | `DEVOPS_OS_GITLAB_OUTPUT` | `.gitlab-ci.yml` | Output file path | +| `--image IMAGE` | `DEVOPS_OS_GITLAB_IMAGE` | `docker:24` | Default Docker image for pipeline jobs | +| `--branches BRANCHES` | `DEVOPS_OS_GITLAB_BRANCHES` | `main` | Comma-separated protected branches (used for deploy rules) | +| `--kube-namespace NS` | `DEVOPS_OS_GITLAB_KUBE_NAMESPACE` | _(empty)_ | Kubernetes namespace; empty means use `$KUBE_NAMESPACE` GitLab variable | +| `--custom-values FILE` | _(not in env)_ | _(none)_ | Path to a custom values JSON file | + +### Input files (optional) + +| File | Flag | Purpose | +|------|------|---------| +| `custom-values.json` | `--custom-values` | Overrides any generated value | + +### Output files + +| File | Condition | Description | +|------|-----------|-------------| +| `` | always | Generated GitLab CI pipeline (YAML) | + +**Default output path:** + +``` +.gitlab-ci.yml +``` + +Use `--output path/to/my-pipeline.yml` to write to a different location. + +### Examples + +```bash +# Complete Python pipeline (default output) +python -m cli.scaffold_gitlab --name flask-api --languages python --type complete +# Output: .gitlab-ci.yml + +# Java build + test, written to a sub-directory +python -m cli.scaffold_gitlab --name java-api --languages java --type test \ + --output ci/gitlab-ci.yml +# Output: ci/gitlab-ci.yml + +# Complete pipeline with ArgoCD deploy on main and production branches +python -m cli.scaffold_gitlab --name my-app --languages python,go \ + --kubernetes --k8s-method argocd --branches main,production +# Output: .gitlab-ci.yml +``` + +--- + +## scaffold_jenkins — Jenkins Pipeline Generator + +Generates a `Jenkinsfile` (declarative pipeline). + +### Invocation + +```bash +python -m cli.scaffold_jenkins [options] +``` + +### Options + +| Option | Env var | Default | Description | +|--------|---------|---------|-------------| +| `--name NAME` | `DEVOPS_OS_JENKINS_NAME` | `DevOps-OS` | Pipeline name | +| `--type TYPE` | `DEVOPS_OS_JENKINS_TYPE` | `complete` | Pipeline type: `build` \| `test` \| `deploy` \| `complete` \| `parameterized` | +| `--languages LANGS` | `DEVOPS_OS_JENKINS_LANGUAGES` | `python,javascript` | Comma-separated: `python`, `java`, `javascript`, `go` | +| `--kubernetes` | `DEVOPS_OS_JENKINS_KUBERNETES` | `false` | Add Kubernetes deploy stage | +| `--registry URL` | `DEVOPS_OS_JENKINS_REGISTRY` | `docker.io` | Container registry URL | +| `--k8s-method METHOD` | `DEVOPS_OS_JENKINS_K8S_METHOD` | `kubectl` | K8s method: `kubectl` \| `kustomize` \| `argocd` \| `flux` | +| `--output FILE` | `DEVOPS_OS_JENKINS_OUTPUT` | `Jenkinsfile` | Output file path | +| `--custom-values FILE` | `DEVOPS_OS_JENKINS_CUSTOM_VALUES` | _(none)_ | Path to a custom values JSON file | +| `--image IMAGE` | `DEVOPS_OS_JENKINS_IMAGE` | `docker.io/yourorg/devops-os:latest` | DevOps-OS container image | +| `--scm SCM` | `DEVOPS_OS_JENKINS_SCM` | `git` | Source control: `git` \| `svn` \| `none` | +| `--parameters` | `DEVOPS_OS_JENKINS_PARAMETERS` | `false` | Add runtime parameters (auto-enabled for `--type parameterized`) | +| `--env-file FILE` | `DEVOPS_OS_JENKINS_ENV_FILE` | _(cli dir)_ | Path to `devcontainer.env.json` | + +### Input files (optional) + +| File | Flag | Purpose | +|------|------|---------| +| `devcontainer.env.json` | `--env-file` | Pre-loads language and tool selection | +| `custom-values.json` | `--custom-values` | Overrides generated defaults (credentials IDs, timeouts, etc.) | + +### Output files + +| File | Condition | Description | +|------|-----------|-------------| +| `` | always | Generated Jenkinsfile (Groovy declarative pipeline) | + +**Default output path:** + +``` +Jenkinsfile +``` + +Use `--output path/to/Jenkinsfile` to write to a different location. + +### Examples + +```bash +# Complete Java pipeline with default output +python -m cli.scaffold_jenkins --name java-api --languages java --type complete +# Output: Jenkinsfile + +# Parameterized pipeline for Python +python -m cli.scaffold_jenkins --name my-app --languages python --type parameterized +# Output: Jenkinsfile + +# Pipeline with Kubernetes deploy, written to a sub-directory +python -m cli.scaffold_jenkins --name my-app --languages go \ + --kubernetes --k8s-method kustomize --output pipelines/Jenkinsfile +# Output: pipelines/Jenkinsfile +``` + +--- + +## scaffold_argocd — ArgoCD / Flux CD Generator + +Generates GitOps configuration files for ArgoCD (Application, AppProject, optional Rollout) or Flux CD (GitRepository, Kustomization, image automation). + +### Invocation + +```bash +python -m cli.scaffold_argocd [options] +``` + +### Options + +| Option | Env var | Default | Description | +|--------|---------|---------|-------------| +| `--name NAME` | `DEVOPS_OS_ARGOCD_NAME` | `my-app` | Application name | +| `--method METHOD` | `DEVOPS_OS_ARGOCD_METHOD` | `argocd` | GitOps tool: `argocd` \| `flux` | +| `--repo URL` | `DEVOPS_OS_ARGOCD_REPO` | `https://github.com/myorg/my-app.git` | Git repository URL | +| `--revision REV` | `DEVOPS_OS_ARGOCD_REVISION` | `HEAD` | Git branch, tag, or commit SHA | +| `--path PATH` | `DEVOPS_OS_ARGOCD_PATH` | `k8s` | Path inside the repository to manifests | +| `--namespace NS` | `DEVOPS_OS_ARGOCD_NAMESPACE` | `default` | Target Kubernetes namespace | +| `--project PROJECT` | `DEVOPS_OS_ARGOCD_PROJECT` | `default` | ArgoCD project name | +| `--server URL` | `DEVOPS_OS_ARGOCD_SERVER` | `https://kubernetes.default.svc` | Destination Kubernetes API server | +| `--auto-sync` | `DEVOPS_OS_ARGOCD_AUTO_SYNC` | `false` | Enable automated sync (prune + self-heal) | +| `--rollouts` | `DEVOPS_OS_ARGOCD_ROLLOUTS` | `false` | Add an Argo Rollouts canary strategy resource | +| `--image IMAGE` | `DEVOPS_OS_ARGOCD_IMAGE` | `ghcr.io/myorg/my-app` | Container image (used in Rollouts / Flux image automation) | +| `--output-dir DIR` | `DEVOPS_OS_ARGOCD_OUTPUT_DIR` | `.` | Root directory for all output files | +| `--allow-any-source-repo` | `DEVOPS_OS_ARGOCD_ALLOW_ANY_SOURCE_REPO` | `false` | Add `*` to AppProject `sourceRepos` (opt-in; grants access to any repo) | + +### Input files + +This command does not read any input files. All configuration is provided via flags or environment variables. + +### Output files — ArgoCD mode (`--method argocd`) + +All files are written under `/argocd/`: + +| File | Condition | Description | +|------|-----------|-------------| +| `argocd/application.yaml` | always | ArgoCD `Application` Custom Resource | +| `argocd/appproject.yaml` | always | ArgoCD `AppProject` Custom Resource | +| `argocd/rollout.yaml` | only with `--rollouts` | Argo Rollouts canary `Rollout` resource | + +**Default output paths (with `--output-dir .`):** + +``` +argocd/ +├── application.yaml +├── appproject.yaml +└── rollout.yaml ← only when --rollouts is set +``` + +### Output files — Flux CD mode (`--method flux`) + +All files are written under `/flux/`: + +| File | Condition | Description | +|------|-----------|-------------| +| `flux/git-repository.yaml` | always | Flux `GitRepository` source | +| `flux/kustomization.yaml` | always | Flux `Kustomization` resource | +| `flux/image-update-automation.yaml` | always | `ImageRepository` + `ImagePolicy` + `ImageUpdateAutomation` resources (multi-doc YAML) | + +**Default output paths (with `--output-dir .`):** + +``` +flux/ +├── git-repository.yaml +├── kustomization.yaml +└── image-update-automation.yaml +``` + +### Examples + +```bash +# ArgoCD Application + AppProject in a custom directory +python -m cli.scaffold_argocd --name my-app \ + --repo https://github.com/myorg/my-app.git \ + --namespace production \ + --output-dir gitops +# Output: gitops/argocd/application.yaml +# gitops/argocd/appproject.yaml + +# ArgoCD with automated sync and canary rollout +python -m cli.scaffold_argocd --name my-app \ + --repo https://github.com/myorg/my-app.git \ + --auto-sync --rollouts +# Output: argocd/application.yaml +# argocd/appproject.yaml +# argocd/rollout.yaml + +# Flux CD with image automation +python -m cli.scaffold_argocd --name my-app --method flux \ + --repo https://github.com/myorg/my-app.git \ + --image ghcr.io/myorg/my-app \ + --output-dir gitops +# Output: gitops/flux/git-repository.yaml +# gitops/flux/kustomization.yaml +# gitops/flux/image-update-automation.yaml +``` + +--- + +## scaffold_sre — SRE Config Generator + +Generates production-grade SRE configuration files: Prometheus alert rules, Grafana dashboard, SLO manifest, and Alertmanager routing config. + +### Invocation + +```bash +python -m cli.scaffold_sre [options] +``` + +### Options + +| Option | Env var | Default | Description | +|--------|---------|---------|-------------| +| `--name NAME` | `DEVOPS_OS_SRE_NAME` | `my-app` | Application / service name | +| `--team TEAM` | `DEVOPS_OS_SRE_TEAM` | `platform` | Owning team (used in labels and alert routing) | +| `--namespace NS` | `DEVOPS_OS_SRE_NAMESPACE` | `default` | Kubernetes namespace where the app runs | +| `--slo-type TYPE` | `DEVOPS_OS_SRE_SLO_TYPE` | `all` | SLO type: `availability` \| `latency` \| `error_rate` \| `all` | +| `--slo-target PCT` | `DEVOPS_OS_SRE_SLO_TARGET` | `99.9` | SLO target as a percentage (e.g. `99.5`) | +| `--latency-threshold SEC` | `DEVOPS_OS_SRE_LATENCY_THRESHOLD` | `0.5` | Latency SLI threshold in seconds | +| `--pagerduty-key KEY` | `DEVOPS_OS_SRE_PAGERDUTY_KEY` | _(empty)_ | PagerDuty integration key; omit to skip PagerDuty routing | +| `--slack-channel CHANNEL` | `DEVOPS_OS_SRE_SLACK_CHANNEL` | `#alerts` | Slack channel for alert routing | +| `--output-dir DIR` | `DEVOPS_OS_SRE_OUTPUT_DIR` | `sre` | Directory where all output files are written | + +### Input files + +This command does not read any input files. All configuration is provided via flags or environment variables. + +### Output files + +All files are written to `/`: + +| File | Condition | Description | +|------|-----------|-------------| +| `alert-rules.yaml` | always | Prometheus `PrometheusRule` Custom Resource with availability, latency, and infrastructure alert groups | +| `grafana-dashboard.json` | always | Importable Grafana dashboard JSON with six metric panels | +| `slo.yaml` | always | [Sloth](https://sloth.dev)-compatible SLO manifest | +| `alertmanager-config.yaml` | always | Alertmanager routing config stub (Slack + optional PagerDuty) | + +**Default output paths (with `--output-dir sre`):** + +``` +sre/ +├── alert-rules.yaml +├── grafana-dashboard.json +├── slo.yaml +└── alertmanager-config.yaml +``` + +### Examples + +```bash +# All SRE configs with defaults +python -m cli.scaffold_sre --name my-app --team platform +# Output: sre/alert-rules.yaml +# sre/grafana-dashboard.json +# sre/slo.yaml +# sre/alertmanager-config.yaml + +# Availability-only SLO, written to a custom directory +python -m cli.scaffold_sre --name my-app --slo-type availability \ + --slo-target 99.9 --output-dir monitoring +# Output: monitoring/alert-rules.yaml (etc.) + +# Latency SLO with 200ms threshold and PagerDuty alerting +python -m cli.scaffold_sre --name my-api --slo-type latency \ + --latency-threshold 0.2 \ + --pagerduty-key YOUR_PD_KEY \ + --slack-channel "#platform-alerts" +# Output: sre/alert-rules.yaml (etc.) +``` + +--- + +## scaffold_devcontainer — Dev Container Generator + +Generates VS Code Dev Container configuration files. + +### Invocation + +```bash +python -m cli.scaffold_devcontainer [options] +``` + +### Options + +| Option | Env var | Default | Description | +|--------|---------|---------|-------------| +| `--languages LANGS` | `DEVOPS_OS_DEVCONTAINER_LANGUAGES` | `python` | Comma-separated languages: `python`, `java`, `node`, `ruby`, `csharp`, `php`, `rust`, `typescript`, `kotlin`, `c`, `cpp`, `javascript`, `go` | +| `--cicd-tools TOOLS` | `DEVOPS_OS_DEVCONTAINER_CICD_TOOLS` | `docker,github_actions` | Comma-separated CI/CD tools: `docker`, `terraform`, `kubectl`, `helm`, `github_actions`, `jenkins` | +| `--kubernetes-tools TOOLS` | `DEVOPS_OS_DEVCONTAINER_KUBERNETES_TOOLS` | _(none)_ | Comma-separated K8s tools: `k9s`, `kustomize`, `argocd_cli`, `lens`, `kubeseal`, `flux`, `kind`, `minikube`, `openshift_cli` | +| `--build-tools TOOLS` | `DEVOPS_OS_DEVCONTAINER_BUILD_TOOLS` | _(none)_ | Comma-separated build tools: `gradle`, `maven`, `ant`, `make`, `cmake` | +| `--code-analysis TOOLS` | `DEVOPS_OS_DEVCONTAINER_CODE_ANALYSIS` | _(none)_ | Comma-separated analysis tools: `sonarqube`, `checkstyle`, `pmd`, `eslint`, `pylint` | +| `--devops-tools TOOLS` | `DEVOPS_OS_DEVCONTAINER_DEVOPS_TOOLS` | _(none)_ | Comma-separated DevOps tools: `nexus`, `prometheus`, `grafana`, `elk`, `jenkins` | +| `--python-version VER` | `DEVOPS_OS_DEVCONTAINER_PYTHON_VERSION` | `3.11` | Python version | +| `--java-version VER` | `DEVOPS_OS_DEVCONTAINER_JAVA_VERSION` | `17` | Java JDK version | +| `--node-version VER` | `DEVOPS_OS_DEVCONTAINER_NODE_VERSION` | `20` | Node.js version | +| `--go-version VER` | `DEVOPS_OS_DEVCONTAINER_GO_VERSION` | `1.21` | Go version | +| `--k9s-version VER` | `DEVOPS_OS_DEVCONTAINER_K9S_VERSION` | `0.29.1` | K9s version | +| `--argocd-version VER` | `DEVOPS_OS_DEVCONTAINER_ARGOCD_VERSION` | `2.8.4` | ArgoCD CLI version | +| `--flux-version VER` | `DEVOPS_OS_DEVCONTAINER_FLUX_VERSION` | `2.1.2` | Flux version | +| `--kustomize-version VER` | `DEVOPS_OS_DEVCONTAINER_KUSTOMIZE_VERSION` | `5.2.1` | Kustomize version | +| `--nexus-version VER` | `DEVOPS_OS_DEVCONTAINER_NEXUS_VERSION` | `3.50.0` | Nexus version | +| `--prometheus-version VER` | `DEVOPS_OS_DEVCONTAINER_PROMETHEUS_VERSION` | `2.45.0` | Prometheus version | +| `--grafana-version VER` | `DEVOPS_OS_DEVCONTAINER_GRAFANA_VERSION` | `10.0.0` | Grafana version | +| `--output-dir DIR` | `DEVOPS_OS_DEVCONTAINER_OUTPUT_DIR` | `.` | Root output directory; files are always written to `/.devcontainer/` | + +### Input files + +This command does not read any input files. All configuration is provided via flags or environment variables. + +### Output files + +Both files are always written to `/.devcontainer/`: + +| File | Description | +|------|-------------| +| `.devcontainer/devcontainer.json` | VS Code dev container configuration (build args, extensions, forwarded ports) | +| `.devcontainer/devcontainer.env.json` | Tool / language selection used by the Dockerfile and other generators | + +**Default output paths (with `--output-dir .`):** + +``` +.devcontainer/ +├── devcontainer.json +└── devcontainer.env.json +``` + +> **Tip:** The generated `devcontainer.env.json` can be fed into the GitHub Actions and Jenkins generators via their `--env-file` option to ensure your CI/CD environment mirrors your local dev container. + +### Examples + +```bash +# Python + Go project with Docker and kubectl +python -m cli.scaffold_devcontainer \ + --languages python,go \ + --cicd-tools docker,kubectl,helm \ + --python-version 3.12 \ + --go-version 1.22 +# Output: .devcontainer/devcontainer.json +# .devcontainer/devcontainer.env.json + +# Full-stack project with Kubernetes tools, written to a custom directory +python -m cli.scaffold_devcontainer \ + --languages python,java,javascript \ + --cicd-tools docker,terraform,kubectl,helm \ + --kubernetes-tools k9s,argocd_cli,flux \ + --devops-tools prometheus,grafana \ + --output-dir /path/to/myproject +# Output: /path/to/myproject/.devcontainer/devcontainer.json +# /path/to/myproject/.devcontainer/devcontainer.env.json +``` + +--- + +## devopsos — Unified CLI + +The `devopsos` CLI wraps all generators and provides an interactive wizard. + +### Invocation + +```bash +python -m cli.devopsos COMMAND [options] +``` + +### Commands + +| Command | Description | +|---------|-------------| +| `init` | Interactive wizard to select tools/languages and optionally generate dev container files | +| `scaffold TARGET` | Non-interactive generator (delegates to the corresponding `scaffold_*` module) | + +### `devopsos init` + +Prompts you to select languages, CI/CD tools, Kubernetes tools, build tools, code analysis tools, and DevOps tools. Then writes a dev container config. + +**Output files:** + +| File | Description | +|------|-------------| +| `.devcontainer/devcontainer.env.json` | Tool / language selection | +| `.devcontainer/devcontainer.json` | Dev container build configuration (created / updated when you confirm the generation prompt) | + +### `devopsos scaffold TARGET` + +Delegates to the individual `scaffold_*` modules. Accepts the same environment variables as each module (no interactive prompts). + +| Target | Module | Output | +|--------|--------|--------| +| `gha` | `scaffold_gha` | `.github/workflows/-.yml` | +| `gitlab` | `scaffold_gitlab` | `.gitlab-ci.yml` | +| `jenkins` | `scaffold_jenkins` | `Jenkinsfile` | +| `argocd` | `scaffold_argocd` | `argocd/` or `flux/` directory | +| `sre` | `scaffold_sre` | `sre/` directory | +| `devcontainer` | `scaffold_devcontainer` | `.devcontainer/` directory | + +```bash +# Examples +python -m cli.devopsos scaffold gha # uses DEVOPS_OS_GHA_* env vars +python -m cli.devopsos scaffold gitlab # uses DEVOPS_OS_GITLAB_* env vars +python -m cli.devopsos scaffold argocd # uses DEVOPS_OS_ARGOCD_* env vars +python -m cli.devopsos scaffold sre # uses DEVOPS_OS_SRE_* env vars +python -m cli.devopsos scaffold devcontainer +``` + +--- + +## Environment Variable Reference + +Every flag for every command has a corresponding environment variable. The prefix varies by command: + +| Command | Env var prefix | Example | +|---------|---------------|---------| +| `scaffold_gha` | `DEVOPS_OS_GHA_` | `DEVOPS_OS_GHA_TYPE=complete` | +| `scaffold_gitlab` | `DEVOPS_OS_GITLAB_` | `DEVOPS_OS_GITLAB_LANGUAGES=python,go` | +| `scaffold_jenkins` | `DEVOPS_OS_JENKINS_` | `DEVOPS_OS_JENKINS_KUBERNETES=true` | +| `scaffold_argocd` | `DEVOPS_OS_ARGOCD_` | `DEVOPS_OS_ARGOCD_AUTO_SYNC=true` | +| `scaffold_sre` | `DEVOPS_OS_SRE_` | `DEVOPS_OS_SRE_SLO_TARGET=99.5` | +| `scaffold_devcontainer` | `DEVOPS_OS_DEVCONTAINER_` | `DEVOPS_OS_DEVCONTAINER_LANGUAGES=python,go` | + +Environment variables are looked up at startup and used as default values when the corresponding flag is not supplied. Explicit flags always take precedence over environment variables. + +**CI/CD usage example** (no interactive prompts needed): + +```bash +export DEVOPS_OS_GHA_NAME="my-service" +export DEVOPS_OS_GHA_TYPE="complete" +export DEVOPS_OS_GHA_LANGUAGES="python,go" +export DEVOPS_OS_GHA_KUBERNETES="true" +export DEVOPS_OS_GHA_K8S_METHOD="argocd" +python -m cli.scaffold_gha +# Output: .github/workflows/my-service-complete.yml +``` + +--- + +## Input File Formats + +### devcontainer.env.json + +Used by `scaffold_gha` (`--env-file`) and `scaffold_jenkins` (`--env-file`) to align CI/CD tooling with the local dev container. + +```json +{ + "languages": { + "python": true, + "java": false, + "javascript": true, + "go": false + }, + "cicd": { + "docker": true, + "terraform": false, + "kubectl": true, + "helm": true, + "github_actions": true, + "jenkins": false + }, + "kubernetes": { + "k9s": false, + "kustomize": true, + "argocd_cli": false, + "flux": false, + "kind": false, + "minikube": false, + "openshift_cli": false + }, + "build_tools": { "gradle": false, "maven": true, "ant": false, "make": true, "cmake": false }, + "code_analysis": { "sonarqube": false, "checkstyle": false, "pmd": false, "eslint": true, "pylint": true }, + "devops_tools": { "nexus": false, "prometheus": false, "grafana": false, "elk": false, "jenkins": false }, + "versions": { + "python": "3.11", + "java": "17", + "node": "20", + "go": "1.21" + } +} +``` + +> **Tip:** Generate this file automatically with `python -m cli.scaffold_devcontainer`, then pass it to other generators with `--env-file .devcontainer/devcontainer.env.json`. + +### custom-values.json + +Accepted by `scaffold_gha`, `scaffold_jenkins`, and `scaffold_gitlab` via `--custom-values`. + +```json +{ + "build": { + "cache": true, + "timeout_minutes": 30, + "artifact_paths": ["dist/**", "build/**"] + }, + "test": { + "coverage": true, + "junit_reports": true, + "parallel": 4, + "timeout_minutes": 20 + }, + "deploy": { + "environments": ["dev", "staging", "prod"], + "approval_required": true, + "rollback_enabled": true + }, + "credentials": { + "docker": "docker-registry-credentials", + "kubernetes": "k8s-kubeconfig", + "git": "git-credentials", + "argocd": "argocd-token" + }, + "matrix": { + "os": ["ubuntu-latest", "windows-latest", "macos-latest"], + "architecture": ["x86_64", "arm64"] + } +} +``` + +--- + +## Related Documentation + +| Topic | Document | +|-------|---------| +| Getting started (first pipeline in 5 min) | [GETTING-STARTED.md](GETTING-STARTED.md) | +| GitHub Actions deep dive | [GITHUB-ACTIONS-README.md](GITHUB-ACTIONS-README.md) | +| GitLab CI deep dive | [GITLAB-CI-README.md](GITLAB-CI-README.md) | +| Jenkins deep dive | [JENKINS-PIPELINE-README.md](JENKINS-PIPELINE-README.md) | +| ArgoCD / Flux deep dive | [ARGOCD-README.md](ARGOCD-README.md) | +| SRE configuration deep dive | [SRE-CONFIGURATION-README.md](SRE-CONFIGURATION-README.md) | +| MCP server (AI integration) | [../mcp_server/README.md](../mcp_server/README.md) | diff --git a/docs/DEVOPS-OS-QUICKSTART.md b/docs/DEVOPS-OS-QUICKSTART.md index a183ccf..34c8193 100644 --- a/docs/DEVOPS-OS-QUICKSTART.md +++ b/docs/DEVOPS-OS-QUICKSTART.md @@ -2,20 +2,25 @@ This guide provides the essential CLI commands for using all functionalities of the DevOps-OS development container project. +> 📖 **Full reference:** For the complete list of every option, input file, and output location see [CLI-COMMANDS-REFERENCE.md](CLI-COMMANDS-REFERENCE.md). + ## Table of Contents - [Setting Up DevOps-OS](#setting-up-devops-os) -- [Creating CI/CD Configurations](#creating-cicd-configurations) - [GitHub Actions Workflows](#github-actions-workflows) +- [GitLab CI Pipelines](#gitlab-ci-pipelines) - [Jenkins Pipelines](#jenkins-pipelines) -- [Kubernetes Deployments](#kubernetes-deployments) +- [GitOps / ArgoCD & Flux CD](#gitops--argocd--flux-cd) +- [SRE Configuration](#sre-configuration) - [Container Configuration](#container-configuration) +- [Common Options for All Generators](#common-options-for-all-generators) +- [Troubleshooting](#troubleshooting) ## Setting Up DevOps-OS ### Clone the DevOps-OS Repository ```bash -git clone https://github.com/yourusername/devops-os.git -cd devops-os +git clone https://github.com/cloudengine-labs/devops_os.git +cd devops_os ``` ### Set Up a Python Virtual Environment (Recommended) @@ -44,6 +49,8 @@ To deactivate it later, simply run `deactivate`. ### Configure Development Container ```bash # Generate dev container config via CLI (recommended) +# Output: .devcontainer/devcontainer.json +# .devcontainer/devcontainer.env.json python -m cli.scaffold_devcontainer \ --languages python,go \ --cicd-tools docker,terraform,kubectl,helm \ @@ -57,198 +64,236 @@ code . # Then use Command Palette (Cmd+Shift+P): "Remote-Containers: Reopen in Container" ``` -### Manually Configure Container (After Building) -```bash -# Run configuration script manually -python3 .devcontainer/configure.py - -# Check installed tools -docker --version -kubectl version --client -helm version -terraform --version -``` - -## Creating CI/CD Configurations - -### Unified CI/CD Generator (Simplest Option) -```bash -# Generate both GitHub Actions and Jenkins pipelines with defaults -cicd/generate-cicd.py - -# Generate build-only configurations for specific languages -cicd/generate-cicd.py --type build --languages python,javascript - -# Generate with Kubernetes deployment -cicd/generate-cicd.py --kubernetes --k8s-method kustomize - -# Generate with matrix builds for GitHub Actions -cicd/generate-cicd.py --github --matrix - -# Generate with parameters for Jenkins -cicd/generate-cicd.py --jenkins --parameters - -# Generate with custom values file -cicd/generate-cicd.py --custom-values my-config.json - -# Generate for specific output location -cicd/generate-cicd.py --output-dir /path/to/my/project -``` - ## GitHub Actions Workflows ### Generate GitHub Actions Workflows ```bash -# Basic GitHub Actions workflow -python3 cicd/github-actions-generator-improved.py +# Basic complete CI/CD workflow +# Output: .github/workflows/devops-os-complete.yml +python -m cli.scaffold_gha # Build-only workflow -python3 cicd/github-actions-generator-improved.py --type build +# Output: .github/workflows/devops-os-build.yml +python -m cli.scaffold_gha --type build -# Complete CI/CD workflow -python3 cicd/github-actions-generator-improved.py --type complete +# Complete CI/CD workflow for a named application +# Output: .github/workflows/my-app-complete.yml +python -m cli.scaffold_gha --name my-app --type complete # Workflow with Kubernetes deployment -python3 cicd/github-actions-generator-improved.py --kubernetes --k8s-method kubectl +# Output: .github/workflows/my-app-complete.yml +python -m cli.scaffold_gha --name my-app --kubernetes --k8s-method kubectl # Matrix build across multiple platforms -python3 cicd/github-actions-generator-improved.py --matrix +# Output: .github/workflows/devops-os-complete.yml +python -m cli.scaffold_gha --matrix # Reusable workflow -python3 cicd/github-actions-generator-improved.py --type reusable +# Output: .github/workflows/devops-os-reusable.yml +python -m cli.scaffold_gha --type reusable # Specify languages to enable -python3 cicd/github-actions-generator-improved.py --languages python,java,go +# Output: .github/workflows/devops-os-complete.yml +python -m cli.scaffold_gha --languages python,java,go # Custom container image -python3 cicd/github-actions-generator-improved.py --image ghcr.io/myorg/devops-os:latest +python -m cli.scaffold_gha --image ghcr.io/myorg/devops-os:latest # Custom output location -python3 cicd/github-actions-generator-improved.py --output ./my-workflows +# Output: my-workflows/devops-os-complete.yml +python -m cli.scaffold_gha --output my-workflows ``` ### Use Environment Variables Instead ```bash # Set environment variables +export DEVOPS_OS_GHA_NAME=my-app export DEVOPS_OS_GHA_TYPE=complete export DEVOPS_OS_GHA_LANGUAGES=python,javascript export DEVOPS_OS_GHA_KUBERNETES=true export DEVOPS_OS_GHA_K8S_METHOD=kustomize # Run generator (will use environment variables) -python3 cicd/github-actions-generator-improved.py +# Output: .github/workflows/my-app-complete.yml +python -m cli.scaffold_gha +``` + +## GitLab CI Pipelines + +### Generate GitLab CI Pipelines +```bash +# Complete pipeline for a Python project +# Output: .gitlab-ci.yml +python -m cli.scaffold_gitlab --name my-app --languages python --type complete + +# Build + test for Java +# Output: .gitlab-ci.yml +python -m cli.scaffold_gitlab --name java-api --languages java --type test + +# Pipeline with Kubernetes deploy via ArgoCD +# Output: .gitlab-ci.yml +python -m cli.scaffold_gitlab --name my-app --languages python,go \ + --kubernetes --k8s-method argocd + +# Custom output path +# Output: ci/my-pipeline.yml +python -m cli.scaffold_gitlab --name my-app --output ci/my-pipeline.yml +``` + +### Use Environment Variables Instead +```bash +export DEVOPS_OS_GITLAB_NAME=my-app +export DEVOPS_OS_GITLAB_TYPE=complete +export DEVOPS_OS_GITLAB_LANGUAGES=python,javascript +export DEVOPS_OS_GITLAB_KUBERNETES=true +export DEVOPS_OS_GITLAB_K8S_METHOD=kustomize + +# Output: .gitlab-ci.yml +python -m cli.scaffold_gitlab ``` ## Jenkins Pipelines ### Generate Jenkins Pipelines ```bash -# Basic Jenkins pipeline -python3 cicd/jenkins-pipeline-generator-improved.py +# Basic complete CI/CD pipeline +# Output: Jenkinsfile +python -m cli.scaffold_jenkins # Build-only pipeline -python3 cicd/jenkins-pipeline-generator-improved.py --type build +# Output: Jenkinsfile +python -m cli.scaffold_jenkins --type build -# Complete CI/CD pipeline -python3 cicd/jenkins-pipeline-generator-improved.py --type complete +# Complete CI/CD pipeline for a named application +# Output: Jenkinsfile +python -m cli.scaffold_jenkins --name my-app --type complete # Pipeline with Kubernetes deployment -python3 cicd/jenkins-pipeline-generator-improved.py --kubernetes --k8s-method kubectl +# Output: Jenkinsfile +python -m cli.scaffold_jenkins --name my-app --kubernetes --k8s-method kubectl # Parameterized pipeline -python3 cicd/jenkins-pipeline-generator-improved.py --parameters +# Output: Jenkinsfile +python -m cli.scaffold_jenkins --parameters # Specify languages to enable -python3 cicd/jenkins-pipeline-generator-improved.py --languages java,go +# Output: Jenkinsfile +python -m cli.scaffold_jenkins --languages java,go # Specify SCM type -python3 cicd/jenkins-pipeline-generator-improved.py --scm git - -# Custom container image -python3 cicd/jenkins-pipeline-generator-improved.py --image docker.io/myorg/devops-os:latest +python -m cli.scaffold_jenkins --scm git # Custom output location -python3 cicd/jenkins-pipeline-generator-improved.py --output ./Jenkinsfile +# Output: pipelines/Jenkinsfile +python -m cli.scaffold_jenkins --output pipelines/Jenkinsfile ``` ### Use Environment Variables Instead ```bash -# Set environment variables +export DEVOPS_OS_JENKINS_NAME=my-app export DEVOPS_OS_JENKINS_TYPE=complete export DEVOPS_OS_JENKINS_LANGUAGES=python,javascript export DEVOPS_OS_JENKINS_KUBERNETES=true export DEVOPS_OS_JENKINS_K8S_METHOD=kustomize export DEVOPS_OS_JENKINS_PARAMETERS=true -# Run generator (will use environment variables) -python3 cicd/jenkins-pipeline-generator-improved.py +# Output: Jenkinsfile +python -m cli.scaffold_jenkins ``` -## Kubernetes Deployments +## GitOps / ArgoCD & Flux CD -### Generate Kubernetes Configurations +### Generate ArgoCD Configurations ```bash -# Generate basic Kubernetes configuration -python3 kubernetes/k8s-config-generator.py +# ArgoCD Application CR + AppProject CR +# Output: argocd/application.yaml +# argocd/appproject.yaml +python -m cli.scaffold_argocd --name my-app \ + --repo https://github.com/myorg/my-app.git \ + --namespace production + +# ArgoCD with automated sync and canary rollout +# Output: argocd/application.yaml +# argocd/appproject.yaml +# argocd/rollout.yaml +python -m cli.scaffold_argocd --name my-app \ + --repo https://github.com/myorg/my-app.git \ + --auto-sync --rollouts -# Specify application name -python3 kubernetes/k8s-config-generator.py --name my-app - -# Multi-container application -python3 kubernetes/k8s-config-generator.py --containers app,db,cache +# Custom output directory +# Output: gitops/argocd/application.yaml (etc.) +python -m cli.scaffold_argocd --name my-app \ + --repo https://github.com/myorg/my-app.git \ + --output-dir gitops +``` -# Generate with specific namespace -python3 kubernetes/k8s-config-generator.py --namespace my-namespace +### Generate Flux CD Configurations +```bash +# Flux GitRepository + Kustomization + Image Automation +# Output: flux/git-repository.yaml +# flux/kustomization.yaml +# flux/image-update-automation.yaml +python -m cli.scaffold_argocd --name my-app --method flux \ + --repo https://github.com/myorg/my-app.git \ + --image ghcr.io/myorg/my-app +``` -# Generate with Ingress -python3 kubernetes/k8s-config-generator.py --ingress +## SRE Configuration -# Generate with specific resource limits -python3 kubernetes/k8s-config-generator.py --cpu 500m --memory 512Mi +### Generate SRE Configs +```bash +# All SRE configs +# Output: sre/alert-rules.yaml +# sre/grafana-dashboard.json +# sre/slo.yaml +# sre/alertmanager-config.yaml +python -m cli.scaffold_sre --name my-app --team platform -# Generate with persistent storage -python3 kubernetes/k8s-config-generator.py --storage +# Availability-only SLO +# Output: sre/alert-rules.yaml (etc.) +python -m cli.scaffold_sre --name my-app --slo-type availability --slo-target 99.9 -# Generate with Kustomize support -python3 kubernetes/k8s-config-generator.py --kustomize +# Latency SLO with 200ms threshold +python -m cli.scaffold_sre --name my-app --slo-type latency --latency-threshold 0.2 -# Generate configurations for multiple environments -python3 kubernetes/k8s-config-generator.py --environments dev,test,prod +# With PagerDuty integration +python -m cli.scaffold_sre --name my-app \ + --pagerduty-key YOUR_PD_KEY \ + --slack-channel "#platform-alerts" # Custom output directory -python3 kubernetes/k8s-config-generator.py --output ./k8s-configs -``` - -### Deploy Kubernetes Resources -```bash -# Apply generated configurations -kubectl apply -f k8s/deployment.yaml -kubectl apply -f k8s/service.yaml - -# Apply with Kustomize -kubectl apply -k k8s/overlays/dev - -# Deploy with ArgoCD -argocd app create my-app --repo https://github.com/myorg/myrepo.git --path k8s --dest-server https://kubernetes.default.svc --dest-namespace default -argocd app sync my-app - -# Deploy with Flux -flux create source git my-app --url=https://github.com/myorg/myrepo.git --branch=main -flux create kustomization my-app --source=my-app --path="./k8s" --prune=true --interval=10m +# Output: monitoring/alert-rules.yaml (etc.) +python -m cli.scaffold_sre --name my-app --output-dir monitoring ``` ## Container Configuration ### Generate Dev Container Config via CLI ```bash -# Generate devcontainer.json and devcontainer.env.json with selected tools +# Python + Go dev container +# Output: .devcontainer/devcontainer.json +# .devcontainer/devcontainer.env.json python -m cli.scaffold_devcontainer \ - --languages python,java,go \ + --languages python,go \ + --cicd-tools docker,kubectl,helm + +# Full-stack container with Kubernetes tools +# Output: .devcontainer/devcontainer.json +# .devcontainer/devcontainer.env.json +python -m cli.scaffold_devcontainer \ + --languages python,java,javascript \ --cicd-tools docker,terraform,kubectl,helm \ --kubernetes-tools k9s,kustomize,argocd_cli,flux \ --devops-tools prometheus,grafana \ --python-version 3.12 +# Write to a specific project directory +# Output: /path/to/myproject/.devcontainer/devcontainer.json +# /path/to/myproject/.devcontainer/devcontainer.env.json +python -m cli.scaffold_devcontainer \ + --languages python,go \ + --output-dir /path/to/myproject + # Or via the unified CLI python -m cli.devopsos scaffold devcontainer @@ -290,81 +335,40 @@ cat > .devcontainer/devcontainer.env.json << EOF EOF ``` -### Build Custom DevOps-OS Container -```bash -# Build container with custom name and tag -docker build -t myorg/devops-os:latest -f .devcontainer/Dockerfile . - -# Push custom container to registry -docker push myorg/devops-os:latest - -# Use custom container in generators -python3 .devcontainer/github-actions-generator-improved.py --image myorg/devops-os:latest -python3 .devcontainer/jenkins-pipeline-generator-improved.py --image myorg/devops-os:latest -``` - -## Examples for Common Technology Stacks - -### Python Web Application (FastAPI) -```bash -# Generate Python-focused CI/CD -cicd/generate-cicd.py --languages python --name "FastAPI CI/CD" - -# Add Python test stage with coverage -python3 cicd/github-actions-generator-improved.py --type test --languages python -``` - -### Java Spring Boot Application -```bash -# Generate Java-focused CI/CD -cicd/generate-cicd.py --languages java --name "Spring Boot CI/CD" - -# Generate Kubernetes deployment for Spring Boot -python3 cicd/k8s-config-generator.py --name spring-app --port 8080 -``` - -### JavaScript/Node.js Application -```bash -# Generate JavaScript-focused CI/CD -cicd/generate-cicd.py --languages javascript --name "Node.js CI/CD" - -# Generate with npm caching -python3 cicd/github-actions-generator-improved.py --languages javascript --custom-values node-config.json -``` - -### Microservices Project -```bash -# Generate complete CI/CD for multiple services -cicd/generate-cicd.py --kubernetes --k8s-method kustomize --custom-values microservices.json -``` - ## Common Options for All Generators -|Option|GitHub Actions|Jenkins|Kubernetes|Description| -|------|--------------|-------|----------|-----------| -|`--name`|✓|✓|✓|Name of the workflow/pipeline/app| -|`--type`|✓|✓|✗|Type of workflow/pipeline| -|`--languages`|✓|✓|✗|Languages to enable| -|`--kubernetes`|✓|✓|✗|Include K8s deployment steps| -|`--k8s-method`|✓|✓|✗|K8s deployment method| -|`--output`|✓|✓|✓|Output directory/file path| -|`--custom-values`|✓|✓|✓|Custom configuration file| -|`--image`|✓|✓|✗|Container image to use| +| Option | `scaffold_gha` | `scaffold_gitlab` | `scaffold_jenkins` | `scaffold_argocd` | `scaffold_sre` | `scaffold_devcontainer` | Description | +|--------|:-:|:-:|:-:|:-:|:-:|:-:|-------------| +| `--name` | ✓ | ✓ | ✓ | ✓ | ✓ | — | Name of the workflow/pipeline/app | +| `--type` | ✓ | ✓ | ✓ | — | — | — | Type of workflow/pipeline | +| `--languages` | ✓ | ✓ | ✓ | — | — | ✓ | Languages to enable | +| `--kubernetes` | ✓ | ✓ | ✓ | — | — | — | Include K8s deployment steps | +| `--k8s-method` | ✓ | ✓ | ✓ | — | — | — | K8s deployment method | +| `--output` | ✓ | ✓ | ✓ | — | — | — | Output file path | +| `--output-dir` | — | — | — | ✓ | ✓ | ✓ | Output directory | +| `--custom-values` | ✓ | ✓ | ✓ | — | — | — | Custom configuration JSON file | +| `--image` | ✓ | ✓ | ✓ | ✓ | — | — | Container image to use | ## Troubleshooting ```bash # Show help for each generator -python3 cicd/github-actions-generator-improved.py --help -python3 cicd/jenkins-pipeline-generator-improved.py --help -python3 kubernetes/k8s-config-generator.py --help -python3 cicd/generate-cicd.py --help +python -m cli.scaffold_gha --help +python -m cli.scaffold_gitlab --help +python -m cli.scaffold_jenkins --help +python -m cli.scaffold_argocd --help +python -m cli.scaffold_sre --help +python -m cli.scaffold_devcontainer --help -# Verify container configuration -cat .devcontainer/devcontainer.env.json +# Verify generated output locations +ls -la .github/workflows/ # GitHub Actions +cat .gitlab-ci.yml # GitLab CI +cat Jenkinsfile # Jenkins +ls -la argocd/ # ArgoCD +ls -la flux/ # Flux CD +ls -la sre/ # SRE configs +ls -la .devcontainer/ # Dev container -# Check generator output -ls -la .github/workflows/ -cat Jenkinsfile -ls -la k8s/ +# Verify dev container configuration +cat .devcontainer/devcontainer.env.json ``` diff --git a/docs/GETTING-STARTED.md b/docs/GETTING-STARTED.md index b59d52f..7342238 100644 --- a/docs/GETTING-STARTED.md +++ b/docs/GETTING-STARTED.md @@ -87,7 +87,9 @@ python -m cli.scaffold_gha --name my-app --languages python --kubernetes --k8s-m python -m cli.scaffold_gha --name my-app --languages python --kubernetes --k8s-method kustomize ``` -Output: `.github/workflows/my-app-complete.yml` +**Output:** `.github/workflows/my-app-complete.yml` +The filename is built as `-.yml` inside the output directory (default: `.github/workflows/`). +Override the directory with `--output `. --- @@ -101,7 +103,8 @@ python -m cli.scaffold_gitlab --name my-app --languages python --type complete python -m cli.scaffold_gitlab --name my-app --languages python,go --kubernetes --k8s-method argocd ``` -Output: `.gitlab-ci.yml` +**Output:** `.gitlab-ci.yml` (default) +Override the file path with `--output ` (e.g. `--output ci/my-pipeline.yml`). --- @@ -115,7 +118,8 @@ python -m cli.scaffold_jenkins --name my-app --languages java --type complete python -m cli.scaffold_jenkins --name my-app --languages python --type parameterized ``` -Output: `Jenkinsfile` +**Output:** `Jenkinsfile` (default) +Override the file path with `--output ` (e.g. `--output pipelines/Jenkinsfile`). --- @@ -125,27 +129,37 @@ Output: `Jenkinsfile` # Plain kubectl Deployment + Service python kubernetes/k8s-config-generator.py --name my-app --image ghcr.io/myorg/my-app:v1 -# ArgoCD Application CR +# ArgoCD Application CR + AppProject python -m cli.scaffold_argocd --name my-app --repo https://github.com/myorg/my-app.git \ --namespace production -# Flux Kustomization +# Flux Kustomization + GitRepository + Image Automation python -m cli.scaffold_argocd --name my-app --method flux --repo https://github.com/myorg/my-app.git ``` +**Output — ArgoCD:** `argocd/application.yaml` and `argocd/appproject.yaml` (in current directory) +**Output — Flux:** `flux/git-repository.yaml`, `flux/kustomization.yaml`, `flux/image-update-automation.yaml` +Override the root directory with `--output-dir `. + --- ## 4 — Generate SRE configs ```bash -# Prometheus alert rules + Grafana dashboard + SLO manifest +# Prometheus alert rules + Grafana dashboard + SLO manifest + Alertmanager config python -m cli.scaffold_sre --name my-app --team platform # Latency SLO only python -m cli.scaffold_sre --name my-app --slo-type latency --slo-target 99.5 ``` -Output: `sre/` directory with `alert-rules.yaml`, `grafana-dashboard.json`, `slo.yaml` +**Output:** `sre/` directory containing: +- `sre/alert-rules.yaml` — Prometheus PrometheusRule CR +- `sre/grafana-dashboard.json` — Grafana importable dashboard +- `sre/slo.yaml` — Sloth-compatible SLO manifest +- `sre/alertmanager-config.yaml` — Alertmanager routing stub + +Override the directory with `--output-dir `. --- @@ -217,7 +231,16 @@ devops_os/ A: No. Docker is only needed if you want to run the dev container. The generators are plain Python scripts. **Q: Where do the generated files go?** -A: By default they are written into your current working directory. Use `--output` / `--output-dir` to change the location. +A: Each generator has its own default output location: +- `scaffold_gha` → `.github/workflows/-.yml` +- `scaffold_gitlab` → `.gitlab-ci.yml` +- `scaffold_jenkins` → `Jenkinsfile` +- `scaffold_argocd` → `argocd/` (ArgoCD) or `flux/` (Flux CD) directory in the current directory +- `scaffold_sre` → `sre/` directory +- `scaffold_devcontainer` → `.devcontainer/` directory + +Use `--output` / `--output-dir` to change the location. +See [CLI-COMMANDS-REFERENCE.md](CLI-COMMANDS-REFERENCE.md) for the full details. **Q: Can I customise the generated output?** A: Yes — use `--custom-values path/to/values.json` to override any default value. @@ -231,6 +254,7 @@ A: Copy the generated file(s) to your project repository, commit, and push. No f | I want to… | Read | |-----------|------| +| See every option and output path | [CLI-COMMANDS-REFERENCE.md](CLI-COMMANDS-REFERENCE.md) | | Deep-dive GitHub Actions options | [GITHUB-ACTIONS-README.md](GITHUB-ACTIONS-README.md) | | Deep-dive GitLab CI options | [GITLAB-CI-README.md](GITLAB-CI-README.md) | | Deep-dive Jenkins options | [JENKINS-PIPELINE-README.md](JENKINS-PIPELINE-README.md) | diff --git a/docs/GITHUB-ACTIONS-README.md b/docs/GITHUB-ACTIONS-README.md index 37cfc32..5e619f9 100644 --- a/docs/GITHUB-ACTIONS-README.md +++ b/docs/GITHUB-ACTIONS-README.md @@ -24,11 +24,16 @@ The GitHub Actions generator (`github-actions-generator-improved.py`) creates YA To generate a basic GitHub Actions workflow: ```bash -python github-actions-generator-improved.py --name "My Workflow" --type complete +python -m cli.scaffold_gha --name "My Workflow" --type complete ``` This generates a complete CI/CD workflow including build, test, and deploy stages. +**Output:** `.github/workflows/my-workflow-complete.yml` + +The filename is built as `-.yml` inside the output directory. +Change the output directory with `--output ` (default: `.github/workflows/`). + ## Workflow Types The generator supports several types of workflows: @@ -50,17 +55,21 @@ The generator supports several types of workflows: ### Example with Basic Options ```bash -python github-actions-generator-improved.py --name "Python API" --languages python --output ./.github/workflows +python -m cli.scaffold_gha --name "Python API" --languages python --output ./.github/workflows ``` +**Output:** `.github/workflows/python-api-complete.yml` + ## Matrix Builds Matrix builds allow you to run your workflows across multiple configurations, such as different operating systems or language versions: ```bash -python github-actions-generator-improved.py --matrix +python -m cli.scaffold_gha --matrix ``` +**Output:** `.github/workflows/devops-os-complete.yml` + This creates a workflow with a matrix strategy that runs on multiple platforms. ### Custom Matrix Configuration @@ -83,7 +92,7 @@ For more advanced matrix configurations, you can provide a custom values file: ``` ```bash -python github-actions-generator-improved.py --custom-values matrix-config.json +python -m cli.scaffold_gha --custom-values matrix-config.json ``` ## Kubernetes Integration @@ -91,7 +100,7 @@ python github-actions-generator-improved.py --custom-values matrix-config.json To include Kubernetes deployment steps in your workflow: ```bash -python github-actions-generator-improved.py --kubernetes --k8s-method kubectl +python -m cli.scaffold_gha --kubernetes --k8s-method kubectl ``` ### Kubernetes Deployment Methods @@ -108,9 +117,11 @@ Four Kubernetes deployment methods are supported: Reusable workflows can be called from other workflows, making them ideal for creating standardized CI/CD templates: ```bash -python github-actions-generator-improved.py --type reusable +python -m cli.scaffold_gha --type reusable ``` +**Output:** `.github/workflows/devops-os-reusable.yml` + ### Using a Reusable Workflow The generated reusable workflow can be called from another workflow: @@ -136,7 +147,8 @@ export DEVOPS_OS_GHA_KUBERNETES="true" export DEVOPS_OS_GHA_K8S_METHOD="kustomize" export DEVOPS_OS_GHA_MATRIX="true" -python github-actions-generator-improved.py +python -m cli.scaffold_gha +# Output: .github/workflows/api-service-complete.yml ``` ## Advanced Customization @@ -178,7 +190,7 @@ For advanced customization, create a custom values JSON file: ``` ```bash -python github-actions-generator-improved.py --custom-values advanced-config.json +python -m cli.scaffold_gha --custom-values advanced-config.json ``` ### Integration with DevOps-OS Configuration @@ -186,7 +198,7 @@ python github-actions-generator-improved.py --custom-values advanced-config.json The generator integrates with the DevOps-OS `devcontainer.env.json` file to ensure consistency between your development environment and CI/CD workflows: ```bash -python github-actions-generator-improved.py --env-file ./devcontainer.env.json +python -m cli.scaffold_gha --env-file ./devcontainer.env.json ``` ## Examples @@ -194,31 +206,36 @@ python github-actions-generator-improved.py --env-file ./devcontainer.env.json ### Basic Python Application Workflow ```bash -python github-actions-generator-improved.py --name "Python App" --languages python --type complete +python -m cli.scaffold_gha --name "Python App" --languages python --type complete +# Output: .github/workflows/python-app-complete.yml ``` ### Java Application with Maven ```bash -python github-actions-generator-improved.py --name "Java Service" --languages java --custom-values maven-config.json +python -m cli.scaffold_gha --name "Java Service" --languages java --custom-values maven-config.json +# Output: .github/workflows/java-service-complete.yml ``` ### Multi-language Microservices ```bash -python github-actions-generator-improved.py --name "Microservices" --languages python,javascript,go --kubernetes --k8s-method kustomize +python -m cli.scaffold_gha --name "Microservices" --languages python,javascript,go --kubernetes --k8s-method kustomize +# Output: .github/workflows/microservices-complete.yml ``` ### Cross-platform Node.js Application ```bash -python github-actions-generator-improved.py --name "Node.js App" --languages javascript --matrix --custom-values node-matrix.json +python -m cli.scaffold_gha --name "Node.js App" --languages javascript --matrix --custom-values node-matrix.json +# Output: .github/workflows/node-js-app-complete.yml ``` ### Complete Docker and Kubernetes Workflow ```bash -python github-actions-generator-improved.py --name "Container Deploy" --languages go --kubernetes --k8s-method argocd --registry ghcr.io +python -m cli.scaffold_gha --name "Container Deploy" --languages go --kubernetes --k8s-method argocd --registry ghcr.io +# Output: .github/workflows/container-deploy-complete.yml ``` ## Understanding the Generated Workflow diff --git a/docs/JENKINS-PIPELINE-README.md b/docs/JENKINS-PIPELINE-README.md index e39766a..78aa5af 100644 --- a/docs/JENKINS-PIPELINE-README.md +++ b/docs/JENKINS-PIPELINE-README.md @@ -24,11 +24,14 @@ The Jenkins pipeline generator (`jenkins-pipeline-generator-improved.py`) create To generate a basic Jenkins pipeline: ```bash -python jenkins-pipeline-generator-improved.py --name "My Pipeline" --type complete +python -m cli.scaffold_jenkins --name "My Pipeline" --type complete ``` This generates a complete CI/CD pipeline including build, test, and deploy stages. +**Output:** `Jenkinsfile` (default) +Change the output file path with `--output ` (e.g. `--output pipelines/Jenkinsfile`). + ## Pipeline Types The generator supports several types of pipelines: @@ -50,7 +53,8 @@ The generator supports several types of pipelines: ### Example with Basic Options ```bash -python jenkins-pipeline-generator-improved.py --name "Python API" --languages python --output ./Jenkinsfile +python -m cli.scaffold_jenkins --name "Python API" --languages python --output ./Jenkinsfile +# Output: Jenkinsfile ``` ## Parameterized Pipelines @@ -58,7 +62,8 @@ python jenkins-pipeline-generator-improved.py --name "Python API" --languages py Parameterized pipelines allow for runtime configuration of the pipeline: ```bash -python jenkins-pipeline-generator-improved.py --parameters +python -m cli.scaffold_jenkins --parameters +# Output: Jenkinsfile ``` This creates a pipeline with parameters that can be configured when the pipeline is run. @@ -86,7 +91,8 @@ parameters { To include Kubernetes deployment steps in your pipeline: ```bash -python jenkins-pipeline-generator-improved.py --kubernetes --k8s-method kubectl +python -m cli.scaffold_jenkins --kubernetes --k8s-method kubectl +# Output: Jenkinsfile ``` ### Kubernetes Deployment Methods @@ -103,7 +109,8 @@ Four Kubernetes deployment methods are supported: The pipeline generator integrates with Jenkins credentials for secure management of sensitive information: ```bash -python jenkins-pipeline-generator-improved.py --type complete --kubernetes +python -m cli.scaffold_jenkins --type complete --kubernetes +# Output: Jenkinsfile ``` ### Credentials in Generated Pipelines @@ -143,7 +150,8 @@ export DEVOPS_OS_JENKINS_KUBERNETES="true" export DEVOPS_OS_JENKINS_K8S_METHOD="kustomize" export DEVOPS_OS_JENKINS_PARAMETERS="true" -python jenkins-pipeline-generator-improved.py +python -m cli.scaffold_jenkins +# Output: Jenkinsfile ``` ## Advanced Customization @@ -200,7 +208,8 @@ For advanced customization, create a custom values JSON file: ``` ```bash -python jenkins-pipeline-generator-improved.py --custom-values advanced-config.json +python -m cli.scaffold_jenkins --custom-values advanced-config.json +# Output: Jenkinsfile ``` ### Integration with DevOps-OS Configuration @@ -208,7 +217,8 @@ python jenkins-pipeline-generator-improved.py --custom-values advanced-config.js The generator integrates with the DevOps-OS `devcontainer.env.json` file to ensure consistency between your development environment and CI/CD pipelines: ```bash -python jenkins-pipeline-generator-improved.py --env-file ./devcontainer.env.json +python -m cli.scaffold_jenkins --env-file ./devcontainer.env.json +# Output: Jenkinsfile ``` ## Examples @@ -216,31 +226,36 @@ python jenkins-pipeline-generator-improved.py --env-file ./devcontainer.env.json ### Basic Python Application Pipeline ```bash -python jenkins-pipeline-generator-improved.py --name "Python App" --languages python --type complete +python -m cli.scaffold_jenkins --name "Python App" --languages python --type complete +# Output: Jenkinsfile ``` ### Java Application with Maven ```bash -python jenkins-pipeline-generator-improved.py --name "Java Service" --languages java --custom-values maven-config.json +python -m cli.scaffold_jenkins --name "Java Service" --languages java --custom-values maven-config.json +# Output: Jenkinsfile ``` ### Multi-language Microservices ```bash -python jenkins-pipeline-generator-improved.py --name "Microservices" --languages python,javascript,go --kubernetes --k8s-method kustomize +python -m cli.scaffold_jenkins --name "Microservices" --languages python,javascript,go --kubernetes --k8s-method kustomize +# Output: Jenkinsfile ``` ### Parameterized Deployment Pipeline ```bash -python jenkins-pipeline-generator-improved.py --name "Deployment" --languages go --kubernetes --k8s-method argocd --parameters +python -m cli.scaffold_jenkins --name "Deployment" --languages go --kubernetes --k8s-method argocd --parameters +# Output: Jenkinsfile ``` ### Complete Docker and Kubernetes Pipeline ```bash -python jenkins-pipeline-generator-improved.py --name "Container Deploy" --languages go --kubernetes --k8s-method kubectl --registry docker.io +python -m cli.scaffold_jenkins --name "Container Deploy" --languages go --kubernetes --k8s-method kubectl --registry docker.io +# Output: Jenkinsfile ``` ## Understanding the Generated Pipeline diff --git a/docs/SRE-CONFIGURATION-README.md b/docs/SRE-CONFIGURATION-README.md index 09738a0..d2e49c0 100644 --- a/docs/SRE-CONFIGURATION-README.md +++ b/docs/SRE-CONFIGURATION-README.md @@ -32,7 +32,10 @@ python -m cli.scaffold_sre --name my-app --pagerduty-key YOUR_PD_KEY \ | `--latency-threshold` | `0.5` | Latency SLI threshold in seconds | | `--pagerduty-key` | _(empty)_ | PagerDuty integration key | | `--slack-channel` | `#alerts` | Slack channel for alert routing | -| `--output-dir` | `sre` | Output directory | +| `--output-dir` | `sre` | Directory where all output files are written | + +All options can also be set via environment variables prefixed with `DEVOPS_OS_SRE_` +(e.g. `DEVOPS_OS_SRE_SLO_TARGET=99.5`). ## Generated Files diff --git a/hugo-docs/content/_index.md b/hugo-docs/content/_index.md new file mode 100644 index 0000000..bdd3c4c --- /dev/null +++ b/hugo-docs/content/_index.md @@ -0,0 +1,86 @@ +--- +title: "DevOps-OS" +type: "docs" +--- + +# 🚀 DevOps-OS + +**Automate your entire DevOps lifecycle — from CI/CD pipelines to Kubernetes deployments and SRE dashboards — using a conversational AI assistant or a single CLI command.** + +[![CI](https://github.com/cloudengine-labs/devops_os/actions/workflows/ci.yml/badge.svg)](https://github.com/cloudengine-labs/devops_os/actions/workflows/ci.yml) +[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue?logo=python&logoColor=white)](https://www.python.org/) +[![License: MIT](https://img.shields.io/badge/license-MIT-green)](https://github.com/cloudengine-labs/devops_os/blob/main/LICENSE) + +--- + +## What is DevOps-OS? + +DevOps-OS is an open-source DevOps automation platform that scaffolds production-ready CI/CD pipelines, Kubernetes configurations, and SRE observability configs — in seconds, from a single CLI command or an AI chat prompt. + +## Features + +| | Feature | Description | +|--|---------|-------------| +| 🚀 | **CI/CD Generators** | One-command scaffolding for GitHub Actions, GitLab CI, and Jenkins pipelines — [→ CI/CD Generators]({{< relref "/docs/ci-cd" >}}) | +| ☸️ | **GitOps Config Generator** | Kubernetes manifests, ArgoCD Applications, and Flux CD Kustomizations — [→ GitOps & ArgoCD]({{< relref "/docs/gitops" >}}) | +| 📊 | **SRE Config Generator** | Prometheus alert rules, Grafana dashboards, and SLO manifests — [→ SRE Configuration]({{< relref "/docs/sre" >}}) | +| 🤖 | **MCP Server** | Plug DevOps-OS tools into Claude or ChatGPT as native AI skills — [→ AI Integration]({{< relref "/docs/ai-integration" >}}) | +| 🛠️ | **Dev Container** | Pre-configured multi-language environment: Python · Java · Go · JavaScript — [→ Dev Container]({{< relref "/docs/dev-container" >}}) | +| 📖 | **CLI Reference** | Complete option tables, input files, and exact output paths for every command — [→ CLI Reference]({{< relref "/docs/reference" >}}) | + +--- + +## ⚡ Quick Start + +```bash +# 1. Clone and install +git clone https://github.com/cloudengine-labs/devops_os.git +cd devops_os +pip install -r cli/requirements.txt + +# 2. Generate a GitHub Actions workflow +python -m cli.scaffold_gha --name my-app --languages python,javascript --type complete +# Output: .github/workflows/my-app-complete.yml + +# 3. Generate a GitLab CI pipeline +python -m cli.scaffold_gitlab --name my-app --languages python --type complete +# Output: .gitlab-ci.yml + +# 4. Generate SRE configs (Prometheus + Grafana + SLO) +python -m cli.scaffold_sre --name my-app --team platform +# Output: sre/ directory +``` + +> [!NOTE] +> **New here?** Start with the [Getting Started guide]({{< relref "/docs/getting-started" >}}) for a step-by-step walkthrough. + +--- + +## Supported Platforms + +| Category | Tools | +|----------|-------| +| **CI/CD** | GitHub Actions, GitLab CI, Jenkins | +| **GitOps / Deploy** | ArgoCD, Flux CD, kubectl, Kustomize | +| **Containers** | Docker, Helm | +| **SRE / Observability** | Prometheus, Grafana, SLO (Sloth-compatible) | +| **AI Integration** | Claude MCP Server, OpenAI function calling | +| **Languages** | Python · Java · Go · JavaScript / TypeScript | + +--- + +## Documentation + +| Guide | Description | +|-------|-------------| +| [Getting Started]({{< relref "/docs/getting-started" >}}) | Zero to first pipeline in 5 minutes | +| [Quick Start Reference]({{< relref "/docs/getting-started/quickstart" >}}) | All CLI commands at a glance | +| [GitHub Actions]({{< relref "/docs/ci-cd/github-actions" >}}) | Generate GHA workflows | +| [GitLab CI]({{< relref "/docs/ci-cd/gitlab-ci" >}}) | Generate GitLab pipelines | +| [Jenkins]({{< relref "/docs/ci-cd/jenkins" >}}) | Generate Jenkinsfiles | +| [ArgoCD & Flux]({{< relref "/docs/gitops" >}}) | Generate GitOps configs | +| [SRE Configuration]({{< relref "/docs/sre" >}}) | Generate monitoring & alerting configs | +| [Kubernetes]({{< relref "/docs/kubernetes" >}}) | Generate K8s manifests | +| [Dev Container]({{< relref "/docs/dev-container" >}}) | Configure the dev container | +| [AI Integration]({{< relref "/docs/ai-integration" >}}) | MCP server & AI skills | +| [CLI Reference]({{< relref "/docs/reference" >}}) | Full CLI options and output paths | diff --git a/hugo-docs/content/docs/_index.md b/hugo-docs/content/docs/_index.md new file mode 100644 index 0000000..63640e6 --- /dev/null +++ b/hugo-docs/content/docs/_index.md @@ -0,0 +1,28 @@ +--- +title: "Documentation" +weight: 1 +bookFlatSection: false +--- + +# DevOps-OS Documentation + +Welcome to the DevOps-OS documentation. Use the sidebar to navigate between sections, or start with the guides below. + +## Where to start + +> [!NOTE] +> **New users:** Read [Getting Started]({{< relref "/docs/getting-started" >}}) first — it walks you through installation and generates your first pipeline in under 5 minutes. + +## Documentation sections + +| Section | What you'll find | +|---------|-----------------| +| [Getting Started]({{< relref "/docs/getting-started" >}}) | Prerequisites, installation, first pipeline | +| [Quick Start]({{< relref "/docs/getting-started/quickstart" >}}) | All CLI commands with output paths | +| [CI/CD Generators]({{< relref "/docs/ci-cd" >}}) | GitHub Actions, GitLab CI, Jenkins | +| [GitOps & ArgoCD]({{< relref "/docs/gitops" >}}) | ArgoCD and Flux CD configs | +| [SRE Configuration]({{< relref "/docs/sre" >}}) | Prometheus, Grafana, SLO, Alertmanager | +| [Kubernetes]({{< relref "/docs/kubernetes" >}}) | Kubernetes manifest generator | +| [Dev Container]({{< relref "/docs/dev-container" >}}) | Multi-language dev environment setup | +| [AI Integration]({{< relref "/docs/ai-integration" >}}) | MCP server for Claude / ChatGPT | +| [CLI Reference]({{< relref "/docs/reference" >}}) | Complete command options and output paths | diff --git a/hugo-docs/content/docs/ai-integration/_index.md b/hugo-docs/content/docs/ai-integration/_index.md new file mode 100644 index 0000000..b7b699e --- /dev/null +++ b/hugo-docs/content/docs/ai-integration/_index.md @@ -0,0 +1,182 @@ +--- +title: "AI Integration" +weight: 70 +--- + +# AI Integration — MCP Server & AI Skills + +DevOps-OS exposes all its pipeline automation tools as an **MCP (Model Context Protocol) server** and as **AI skill definitions** for Claude and OpenAI. + +--- + +## Available Tools + +| Tool | What it generates | +|------|-------------------| +| `generate_github_actions_workflow` | GitHub Actions workflow YAML | +| `generate_jenkins_pipeline` | Jenkins Declarative Pipeline (Jenkinsfile) | +| `generate_k8s_config` | Kubernetes Deployment + Service manifests | +| `scaffold_devcontainer` | `devcontainer.json` + `devcontainer.env.json` | +| `generate_gitlab_ci_pipeline` | GitLab CI/CD pipeline (`.gitlab-ci.yml`) | +| `generate_argocd_config` | Argo CD Application and AppProject manifests | +| `generate_sre_configs` | SRE / observability configs (Prometheus, Grafana, SLO) | + +--- + +## MCP Server + +### Installation + +```bash +pip install -r mcp_server/requirements.txt +``` + +### Running the server + +```bash +# Run as a stdio MCP server (default — for Claude Desktop and most MCP clients) +python -m mcp_server.server + +# Or directly +python mcp_server/server.py +``` + +### Connecting to Claude Desktop + +Add to `claude_desktop_config.json` +(`~/Library/Application Support/Claude/claude_desktop_config.json` on macOS): + +```json +{ + "mcpServers": { + "devops-os": { + "command": "python", + "args": ["-m", "mcp_server.server"], + "cwd": "/path/to/devops_os" + } + } +} +``` + +Restart Claude Desktop, then ask it: + +> *"Generate a complete GitHub Actions CI/CD workflow for a Python + Node.js project with Kubernetes deployment using Kustomize."* + +> *"Create a Jenkins pipeline for a Python microservice with Docker build and push stages."* + +> *"Scaffold a devcontainer for a Go + Python project with Terraform and kubectl."* + +--- + +## Architecture + +``` +AI Assistant (Claude / ChatGPT) + │ MCP / function-call request + ▼ +DevOps-OS MCP Server ←──────────┐ + │ │ + │ calls Python functions │ + ▼ │ + cli.scaffold_gha │ + cli.scaffold_gitlab (same generators + cli.scaffold_jenkins used by the CLI) + cli.scaffold_argocd │ + cli.scaffold_sre │ + │ │ + ▼ │ + Generated files ───────────────┘ +``` + +--- + +## Using with Claude (Anthropic API) + +Load `skills/claude_tools.json` as the `tools` parameter: + +```python +import json +import anthropic + +with open("skills/claude_tools.json") as fh: + tools = json.load(fh) + +client = anthropic.Anthropic() +response = client.messages.create( + model="claude-opus-4-5", + max_tokens=4096, + tools=tools, + messages=[{ + "role": "user", + "content": ( + "Generate a complete GitHub Actions CI/CD workflow for a " + "Python + Node.js project with Kubernetes deployment via Kustomize." + ), + }], +) + +for block in response.content: + if block.type == "tool_use": + print(f"Tool: {block.name}") + print(f"Input: {json.dumps(block.input, indent=2)}") +``` + +--- + +## Using with OpenAI (function calling) + +Load `skills/openai_functions.json` as the `tools` parameter: + +```python +import json +import openai + +with open("skills/openai_functions.json") as fh: + functions = json.load(fh) + +client = openai.OpenAI() +response = client.chat.completions.create( + model="gpt-4o", + tools=functions, + messages=[{ + "role": "user", + "content": "Generate a Jenkins pipeline for a Java Spring Boot app." + }], +) + +for choice in response.choices: + if choice.message.tool_calls: + for tc in choice.message.tool_calls: + print(f"Function: {tc.function.name}") + print(f"Args: {tc.function.arguments}") +``` + +--- + +## Custom GPT / GPT Actions + +Use `skills/openai_functions.json` as the OpenAPI schema for a Custom GPT Action: + +1. Open **ChatGPT → Create a GPT → Configure → Actions → Create new action** +2. Paste the contents of `skills/openai_functions.json` +3. Set the server URL to your deployed MCP server endpoint +4. Save and test the GPT + +--- + +## Example Prompts + +``` +Generate a GitHub Actions workflow for a Java Spring Boot app with kubectl deployment. + +Create a Jenkins pipeline for a Python microservice with Docker build and push stages. + +Scaffold a devcontainer for a Go + Python project with Terraform and kubectl. + +Generate Kubernetes manifests for an app called 'api-service' using image +'ghcr.io/myorg/api-service:v1.2.3' with 3 replicas on port 8080. + +Create a GitLab CI pipeline with Python testing and ArgoCD deployment. + +Generate SRE configs for my-service with a 99.9% availability SLO and PagerDuty alerts. +``` diff --git a/hugo-docs/content/docs/ci-cd/_index.md b/hugo-docs/content/docs/ci-cd/_index.md new file mode 100644 index 0000000..a06b76e --- /dev/null +++ b/hugo-docs/content/docs/ci-cd/_index.md @@ -0,0 +1,27 @@ +--- +title: "CI/CD Generators" +weight: 20 +bookCollapseSection: true +--- + +# CI/CD Generators + +DevOps-OS provides one-command scaffolding for three major CI/CD platforms: + +| Platform | Description | Guide | +|----------|-------------|-------| +| ⚙️ **GitHub Actions** | Generate complete GitHub Actions workflow YAML files with build, test, deploy, and reusable workflow types. | [GitHub Actions →]({{< relref "/docs/ci-cd/github-actions" >}}) | +| 🦊 **GitLab CI** | Generate `.gitlab-ci.yml` pipelines with multi-stage jobs, Docker build, Kubernetes deploy, and language-specific test jobs. | [GitLab CI →]({{< relref "/docs/ci-cd/gitlab-ci" >}}) | +| 🔧 **Jenkins** | Generate Jenkinsfiles using the declarative pipeline syntax, with parameterized builds and Kubernetes deployment support. | [Jenkins →]({{< relref "/docs/ci-cd/jenkins" >}}) | + +--- + +## All three generators share the same conventions + +- **Language selection:** `--languages python,java,javascript,go` +- **Kubernetes deploy:** `--kubernetes --k8s-method [kubectl|kustomize|argocd|flux]` +- **Custom overrides:** `--custom-values path/to/values.json` +- **Environment variables:** Every flag has a `DEVOPS_OS__*` env-var alternative for CI/CD usage +- **Output control:** `--output` (file path) or `--output-dir` (directory) + +See [CLI Reference]({{< relref "/docs/reference" >}}) for complete option tables. diff --git a/hugo-docs/content/docs/ci-cd/github-actions.md b/hugo-docs/content/docs/ci-cd/github-actions.md new file mode 100644 index 0000000..a359c2e --- /dev/null +++ b/hugo-docs/content/docs/ci-cd/github-actions.md @@ -0,0 +1,213 @@ +--- +title: "GitHub Actions" +weight: 21 +--- + +# GitHub Actions Workflow Generator + +The GitHub Actions generator creates YAML workflow files that orchestrate CI/CD processes using GitHub's built-in action system. Workflows leverage the DevOps-OS container for a consistent build environment. + +--- + +## Basic Usage + +```bash +python -m cli.scaffold_gha --name "my-app" --type complete +``` + +**Output:** `.github/workflows/my-app-complete.yml` + +The filename pattern is `-.yml` inside the output directory. +Change the output directory with `--output ` (default: `.github/workflows/`). + +--- + +## Options + +| Option | Default | Description | +|--------|---------|-------------| +| `--name NAME` | `DevOps-OS` | Workflow name | +| `--type TYPE` | `complete` | `build` \| `test` \| `deploy` \| `complete` \| `reusable` | +| `--languages LANGS` | `python,javascript` | Comma-separated: `python`, `java`, `javascript`, `go` | +| `--kubernetes` | off | Include Kubernetes deployment steps | +| `--registry URL` | `ghcr.io` | Container registry URL | +| `--k8s-method METHOD` | `kubectl` | `kubectl` \| `kustomize` \| `argocd` \| `flux` | +| `--output DIR` | `.github/workflows` | Output directory | +| `--custom-values FILE` | _(none)_ | Path to custom values JSON file | +| `--image IMAGE` | `ghcr.io/yourorg/devops-os:latest` | DevOps-OS container image | +| `--branches BRANCHES` | `main` | Comma-separated branches that trigger the workflow | +| `--matrix` | off | Enable matrix builds across OS/architectures | +| `--env-file FILE` | _(cli dir)_ | Path to `devcontainer.env.json` | +| `--reusable` | off | Generate a reusable workflow | + +--- + +## Workflow Types + +| Type | Description | +|------|-------------| +| `build` | Focuses on building and packaging your application | +| `test` | Focuses on running tests | +| `deploy` | Focuses on deploying to the target environment | +| `complete` | Combines build, test, and deploy stages | +| `reusable` | Creates a workflow callable from other workflows | + +--- + +## Examples + +### Python application — complete pipeline + +```bash +python -m cli.scaffold_gha --name "Python App" --languages python --type complete +# Output: .github/workflows/python-app-complete.yml +``` + +### Java with Maven + +```bash +python -m cli.scaffold_gha --name "Java Service" --languages java --custom-values maven-config.json +# Output: .github/workflows/java-service-complete.yml +``` + +### Multi-language microservices with Kubernetes + +```bash +python -m cli.scaffold_gha \ + --name "Microservices" \ + --languages python,javascript,go \ + --kubernetes --k8s-method kustomize +# Output: .github/workflows/microservices-complete.yml +``` + +### Matrix build (cross-platform) + +```bash +python -m cli.scaffold_gha --name "Node.js App" --languages javascript --matrix +# Output: .github/workflows/node-js-app-complete.yml +``` + +### Reusable workflow + +```bash +python -m cli.scaffold_gha --name "shared" --type reusable +# Output: .github/workflows/shared-reusable.yml +``` + +--- + +## Environment Variables + +All options can be set using environment variables prefixed with `DEVOPS_OS_GHA_`: + +```bash +export DEVOPS_OS_GHA_NAME="API Service" +export DEVOPS_OS_GHA_TYPE="complete" +export DEVOPS_OS_GHA_LANGUAGES="python,go" +export DEVOPS_OS_GHA_KUBERNETES="true" +export DEVOPS_OS_GHA_K8S_METHOD="kustomize" +export DEVOPS_OS_GHA_MATRIX="true" + +python -m cli.scaffold_gha +# Output: .github/workflows/api-service-complete.yml +``` + +--- + +## Kubernetes Deployment Methods + +| Method | What happens | +|--------|-------------| +| `kubectl` | Direct deployment using `kubectl set image` and rollout status | +| `kustomize` | `kustomize edit set image` + `kubectl apply -k` | +| `argocd` | `argocd app set` + sync + wait | +| `flux` | `flux reconcile` + kustomization reconcile | + +--- + +## Reusable Workflows + +Call the generated reusable workflow from another workflow: + +```yaml +jobs: + call-devops-os-workflow: + uses: ./.github/workflows/shared-reusable.yml + with: + languages: '{"python": true, "java": true}' + deploy_environment: 'production' +``` + +--- + +## Custom Values File + +```json +{ + "build": { + "cache": true, + "timeout_minutes": 30, + "artifact_paths": ["dist/**", "build/**"] + }, + "test": { + "coverage": true, + "junit_reports": true, + "parallel": 4 + }, + "deploy": { + "environments": ["dev", "staging", "prod"], + "approval_required": true, + "rollback_enabled": true + }, + "matrix": { + "os": ["ubuntu-latest", "windows-latest", "macos-latest"], + "architecture": ["x86_64", "arm64"] + } +} +``` + +```bash +python -m cli.scaffold_gha --custom-values advanced-config.json +``` + +--- + +## Generated Workflow Structure + +```yaml +name: My CI/CD +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + container: + image: ghcr.io/yourorg/devops-os:latest + steps: + - uses: actions/checkout@v3 + # Language-specific build steps... + + test: + needs: build + # ... + + deploy: + needs: test + if: github.event_name == 'push' + # ... +``` + +--- + +## Best Practices + +1. Start with `--type complete` and remove stages you don't need +2. Pin the `--image` to a specific tag in production +3. Use `--env-file` to align CI/CD with your local dev container +4. Use reusable workflows to standardize pipelines across multiple repos +5. Store secrets in GitHub Secrets, reference them with `${{ secrets.MY_SECRET }}` diff --git a/hugo-docs/content/docs/ci-cd/gitlab-ci.md b/hugo-docs/content/docs/ci-cd/gitlab-ci.md new file mode 100644 index 0000000..51ec524 --- /dev/null +++ b/hugo-docs/content/docs/ci-cd/gitlab-ci.md @@ -0,0 +1,168 @@ +--- +title: "GitLab CI" +weight: 22 +--- + +# GitLab CI Pipeline Generator + +Generate a complete `.gitlab-ci.yml` for your project in a single command. The generator creates multi-stage pipelines with language-specific test jobs, Docker build, and Kubernetes deployment. + +--- + +## Basic Usage + +```bash +python -m cli.scaffold_gitlab --name my-app --languages python --type complete +``` + +**Output:** `.gitlab-ci.yml` (default) +Change the output path with `--output `. + +--- + +## Options + +| Option | Default | Description | +|--------|---------|-------------| +| `--name NAME` | `my-app` | Application / pipeline name | +| `--type TYPE` | `complete` | `build` \| `test` \| `deploy` \| `complete` | +| `--languages LANGS` | `python` | Comma-separated: `python`, `java`, `javascript`, `go` | +| `--kubernetes` | off | Add a Kubernetes deploy stage | +| `--k8s-method METHOD` | `kubectl` | `kubectl` \| `kustomize` \| `argocd` \| `flux` | +| `--output FILE` | `.gitlab-ci.yml` | Output file path | +| `--image IMAGE` | `docker:24` | Default Docker image for pipeline jobs | +| `--branches BRANCHES` | `main` | Branches that trigger deploy jobs | +| `--kube-namespace NS` | _(empty)_ | Kubernetes namespace (empty = use `$KUBE_NAMESPACE` variable) | +| `--custom-values FILE` | _(none)_ | Path to custom values JSON file | + +All options can be set via environment variables prefixed `DEVOPS_OS_GITLAB_`. + +--- + +## Generated Pipeline Stages + +### `build` stage + +- Logs in to the GitLab Container Registry +- Runs language-specific compile / install steps (detected from project files) +- Builds and pushes a Docker image tagged with `$CI_COMMIT_SHORT_SHA` and `latest` + +### `test` stage + +| Language | Image | Test command | +|----------|-------|-------------| +| Python | `python:3.11-slim` | `pytest --cov` | +| Java | `maven:3.9-eclipse-temurin-17` | `mvn test` / `gradle test` | +| JavaScript | `node:20-slim` | `npm test` | +| Go | `golang:1.21` | `go test ./...` | + +Each test job uploads JUnit / coverage artifacts automatically. + +### `deploy` stage (requires `--kubernetes`) + +| Method | What happens | +|--------|-------------| +| `kubectl` | `kubectl set image` + rollout status check | +| `kustomize` | `kustomize edit set image` + `kubectl apply` | +| `argocd` | `argocd app set` + sync + wait | +| `flux` | `flux reconcile image repository` + kustomization | + +--- + +## Examples + +### Python + Docker + kubectl deploy + +```bash +python -m cli.scaffold_gitlab \ + --name flask-api \ + --languages python \ + --type complete \ + --kubernetes \ + --k8s-method kubectl \ + --branches main,production +# Output: .gitlab-ci.yml +``` + +Generated stages: + +```yaml +stages: + - build + - test + - deploy + +variables: + APP_NAME: flask-api + IMAGE_TAG: $CI_COMMIT_SHORT_SHA + +build: + stage: build + image: docker:24 + script: + - docker login $CI_REGISTRY ... + - docker build -t $CI_REGISTRY_IMAGE:$IMAGE_TAG . + - docker push $CI_REGISTRY_IMAGE:$IMAGE_TAG + +test:python: + stage: test + image: python:3.11-slim + script: + - pip install -r requirements.txt pytest pytest-cov + - pytest --cov=./ --cov-report=xml -v + +deploy:kubernetes: + stage: deploy + image: bitnami/kubectl:1.29 + script: + - kubectl set image deployment/flask-api app=$CI_REGISTRY_IMAGE:$IMAGE_TAG + rules: + - if: $CI_COMMIT_BRANCH == "main" +``` + +### Java build + test (no deploy) + +```bash +python -m cli.scaffold_gitlab --name java-api --languages java --type test +# Output: .gitlab-ci.yml +``` + +### Multi-language with ArgoCD deploy + +```bash +python -m cli.scaffold_gitlab \ + --name my-app \ + --languages python,go \ + --kubernetes --k8s-method argocd +# Output: .gitlab-ci.yml +``` + +--- + +## Environment Variables + +```bash +export DEVOPS_OS_GITLAB_NAME=my-app +export DEVOPS_OS_GITLAB_TYPE=complete +export DEVOPS_OS_GITLAB_LANGUAGES=python,javascript +export DEVOPS_OS_GITLAB_KUBERNETES=true +export DEVOPS_OS_GITLAB_K8S_METHOD=kustomize + +python -m cli.scaffold_gitlab +# Output: .gitlab-ci.yml +``` + +--- + +## Required GitLab CI/CD Variables + +Set these in **GitLab → Settings → CI/CD → Variables**: + +| Variable | Description | +|----------|-------------| +| `CI_REGISTRY_USER` | GitLab registry username (auto-set for GitLab) | +| `CI_REGISTRY_PASSWORD` | GitLab registry password (auto-set for GitLab) | +| `KUBE_CONTEXT` | kubectl context name | +| `KUBE_NAMESPACE` | Target Kubernetes namespace | +| `ARGOCD_SERVER` | ArgoCD server hostname (if using ArgoCD) | +| `ARGOCD_TOKEN` | ArgoCD API token (if using ArgoCD) | diff --git a/hugo-docs/content/docs/ci-cd/jenkins.md b/hugo-docs/content/docs/ci-cd/jenkins.md new file mode 100644 index 0000000..8230f12 --- /dev/null +++ b/hugo-docs/content/docs/ci-cd/jenkins.md @@ -0,0 +1,213 @@ +--- +title: "Jenkins" +weight: 23 +--- + +# Jenkins Pipeline Generator + +Generate `Jenkinsfile` scripts using the declarative pipeline syntax. Pipelines leverage the DevOps-OS container for a consistent build environment. + +--- + +## Basic Usage + +```bash +python -m cli.scaffold_jenkins --name "my-app" --type complete +``` + +**Output:** `Jenkinsfile` (default) +Change the output path with `--output `. + +--- + +## Options + +| Option | Default | Description | +|--------|---------|-------------| +| `--name NAME` | `DevOps-OS` | Pipeline name | +| `--type TYPE` | `complete` | `build` \| `test` \| `deploy` \| `complete` \| `parameterized` | +| `--languages LANGS` | `python,javascript` | Comma-separated: `python`, `java`, `javascript`, `go` | +| `--kubernetes` | off | Add Kubernetes deploy stage | +| `--registry URL` | `docker.io` | Container registry URL | +| `--k8s-method METHOD` | `kubectl` | `kubectl` \| `kustomize` \| `argocd` \| `flux` | +| `--output FILE` | `Jenkinsfile` | Output file path | +| `--custom-values FILE` | _(none)_ | Path to custom values JSON file | +| `--image IMAGE` | `docker.io/yourorg/devops-os:latest` | DevOps-OS container image | +| `--scm SCM` | `git` | Source control: `git` \| `svn` \| `none` | +| `--parameters` | off | Add runtime parameters (auto-enabled for `--type parameterized`) | +| `--env-file FILE` | _(cli dir)_ | Path to `devcontainer.env.json` | + +All options can be set via environment variables prefixed `DEVOPS_OS_JENKINS_`. + +--- + +## Pipeline Types + +| Type | Description | +|------|-------------| +| `build` | Focuses on building and packaging your application | +| `test` | Focuses on running tests | +| `deploy` | Focuses on deploying to the target environment | +| `complete` | Combines build, test, and deploy stages | +| `parameterized` | Adds runtime parameters for interactive runs | + +--- + +## Examples + +### Complete Java pipeline + +```bash +python -m cli.scaffold_jenkins --name my-app --languages java --type complete +# Output: Jenkinsfile +``` + +### Parameterized deployment pipeline + +```bash +python -m cli.scaffold_jenkins \ + --name "Deployment" \ + --languages go \ + --kubernetes --k8s-method argocd \ + --parameters +# Output: Jenkinsfile +``` + +### Custom output location + +```bash +python -m cli.scaffold_jenkins \ + --name my-app \ + --languages python \ + --output pipelines/Jenkinsfile +# Output: pipelines/Jenkinsfile +``` + +--- + +## Generated Pipeline Structure + +```groovy +pipeline { + agent { + docker { + image 'docker.io/yourorg/devops-os:latest' + args '-v /var/run/docker.sock:/var/run/docker.sock -u root' + } + } + parameters { + booleanParam(name: 'PYTHON_ENABLED', defaultValue: true, ...) + choice(name: 'ENVIRONMENT', choices: ['dev', 'test', 'staging', 'prod'], ...) + string(name: 'IMAGE_TAG', defaultValue: 'latest', ...) + } + environment { + REGISTRY = 'docker.io' + IMAGE_NAME = 'myorg/my-app' + } + stages { + stage('Build') { ... } + stage('Test') { ... } + stage('Deploy') { ... } + } + post { + always { cleanWs() } + failure { mail to: 'team@example.com', subject: 'Build failed' } + } +} +``` + +--- + +## Parameterized Pipelines + +Parameterized pipelines accept runtime inputs: + +```groovy +parameters { + booleanParam(name: 'PYTHON_ENABLED', defaultValue: true, + description: 'Enable Python tools') + choice(name: 'ENVIRONMENT', + choices: ['dev', 'test', 'staging', 'prod'], + defaultValue: 'dev') + string(name: 'IMAGE_TAG', defaultValue: 'latest', + description: 'Container image tag') +} +``` + +--- + +## Credentials Management + +The generated pipeline uses Jenkins credentials: + +```groovy +withCredentials([ + file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG') +]) { + sh 'kubectl apply -f ./k8s/deployment.yaml' +} + +withCredentials([ + usernamePassword( + credentialsId: 'registry-credentials', + usernameVariable: 'REGISTRY_USER', + passwordVariable: 'REGISTRY_PASSWORD' + ) +]) { + sh 'docker login -u $REGISTRY_USER -p $REGISTRY_PASSWORD' +} +``` + +--- + +## Environment Variables + +```bash +export DEVOPS_OS_JENKINS_NAME="API Service" +export DEVOPS_OS_JENKINS_TYPE="complete" +export DEVOPS_OS_JENKINS_LANGUAGES="python,go" +export DEVOPS_OS_JENKINS_KUBERNETES="true" +export DEVOPS_OS_JENKINS_K8S_METHOD="kustomize" +export DEVOPS_OS_JENKINS_PARAMETERS="true" + +python -m cli.scaffold_jenkins +# Output: Jenkinsfile +``` + +--- + +## Custom Values File + +```json +{ + "build": { + "timeout_minutes": 30, + "tool_options": { + "maven": { "goals": ["clean", "package"] } + } + }, + "credentials": { + "docker": "docker-registry-credentials", + "kubernetes": "kubeconfig", + "argocd": "argocd-credentials" + }, + "notifications": { + "slack": { "channel": "deployments", "failure": true } + } +} +``` + +```bash +python -m cli.scaffold_jenkins --custom-values advanced-config.json +# Output: Jenkinsfile +``` + +--- + +## Best Practices + +1. Use `--parameters` for environments that benefit from manual approval gates +2. Store credentials in Jenkins Credentials Store, never in the Jenkinsfile +3. Use `--env-file` to align with your dev container configuration +4. Start with `--type complete` and remove stages you don't need +5. Set appropriate timeouts in `--custom-values` for long builds diff --git a/hugo-docs/content/docs/dev-container/_index.md b/hugo-docs/content/docs/dev-container/_index.md new file mode 100644 index 0000000..4e1eb76 --- /dev/null +++ b/hugo-docs/content/docs/dev-container/_index.md @@ -0,0 +1,170 @@ +--- +title: "Dev Container" +weight: 60 +--- + +# Dev Container Setup + +DevOps-OS provides a pre-configured VS Code Dev Container that gives you a consistent, multi-language development environment with all CI/CD tools included. + +--- + +## Quick Start + +### Generate with the CLI (recommended) + +```bash +python -m cli.scaffold_devcontainer \ + --languages python,go \ + --cicd-tools docker,kubectl,helm \ + --kubernetes-tools k9s,argocd_cli,flux +# Output: .devcontainer/devcontainer.json +# .devcontainer/devcontainer.env.json +``` + +Then open VS Code and run **"Dev Containers: Reopen in Container"** from the Command Palette. + +### Or via the unified CLI + +```bash +python -m cli.devopsos scaffold devcontainer +``` + +--- + +## Options + +| Option | Default | Description | +|--------|---------|-------------| +| `--languages LANGS` | `python` | Comma-separated languages: `python`, `java`, `node`, `ruby`, `csharp`, `php`, `rust`, `typescript`, `kotlin`, `c`, `cpp`, `javascript`, `go` | +| `--cicd-tools TOOLS` | `docker,github_actions` | CI/CD tools: `docker`, `terraform`, `kubectl`, `helm`, `github_actions`, `jenkins` | +| `--kubernetes-tools TOOLS` | _(none)_ | K8s tools: `k9s`, `kustomize`, `argocd_cli`, `lens`, `kubeseal`, `flux`, `kind`, `minikube`, `openshift_cli` | +| `--build-tools TOOLS` | _(none)_ | Build tools: `gradle`, `maven`, `ant`, `make`, `cmake` | +| `--code-analysis TOOLS` | _(none)_ | Analysis tools: `sonarqube`, `checkstyle`, `pmd`, `eslint`, `pylint` | +| `--devops-tools TOOLS` | _(none)_ | DevOps tools: `nexus`, `prometheus`, `grafana`, `elk`, `jenkins` | +| `--python-version VER` | `3.11` | Python version | +| `--java-version VER` | `17` | Java JDK version | +| `--node-version VER` | `20` | Node.js version | +| `--go-version VER` | `1.21` | Go version | +| `--output-dir DIR` | `.` | Root directory; files written to `/.devcontainer/` | + +All options can be set via environment variables prefixed `DEVOPS_OS_DEVCONTAINER_`. + +--- + +## Generated Files + +``` +.devcontainer/ +├── devcontainer.json VS Code dev container configuration +└── devcontainer.env.json Tool / language selection & versions +``` + +**`devcontainer.json`** contains: build args, VS Code extensions, forwarded ports, and post-create commands. + +**`devcontainer.env.json`** controls which languages and tools are enabled — it drives the Dockerfile build args. + +--- + +## Full-stack Example + +```bash +python -m cli.scaffold_devcontainer \ + --languages python,java,javascript \ + --cicd-tools docker,terraform,kubectl,helm \ + --kubernetes-tools k9s,kustomize,argocd_cli,flux \ + --devops-tools prometheus,grafana \ + --python-version 3.12 +``` + +This generates a dev container with: +- Python 3.12, Java 17, Node.js 20 +- Docker, Terraform, kubectl, Helm +- K9s, Kustomize, ArgoCD CLI, Flux CD +- Prometheus (port 9090) and Grafana (port 3000) forwarded + +--- + +## Manual Configuration + +Edit `.devcontainer/devcontainer.env.json` directly: + +```json +{ + "languages": { + "python": true, + "java": true, + "javascript": true, + "go": false + }, + "cicd": { + "docker": true, + "terraform": true, + "kubectl": true, + "helm": true, + "github_actions": true + }, + "kubernetes": { + "k9s": true, + "kustomize": true, + "argocd_cli": true, + "flux": true + }, + "versions": { + "python": "3.11", + "java": "17", + "node": "20", + "go": "1.21" + } +} +``` + +After editing, rebuild the container: **"Dev Containers: Rebuild Container"**. + +--- + +## Using with Existing Projects + +Copy the generated `.devcontainer/` directory to your project: + +```bash +# In the devops_os repo: +python -m cli.scaffold_devcontainer \ + --languages python,go \ + --output-dir /path/to/my-project + +# Open your project in VS Code and reopen in container +cd /path/to/my-project +code . +``` + +--- + +## Included Tools (by category) + +| Category | Tools | +|----------|-------| +| **Languages** | Python · Java · Node.js · Go · Ruby · C/C++ · Rust | +| **Containers** | Docker CLI · Docker Compose | +| **IaC** | Terraform · AWS CLI · Azure CLI | +| **Kubernetes** | kubectl · Helm · K9s · Kustomize · KinD · Minikube | +| **GitOps** | ArgoCD CLI · Flux CD | +| **Observability** | Prometheus · Grafana · ELK Stack | +| **Build** | Maven · Gradle · npm · pip · Go tools | +| **Code Quality** | SonarQube · ESLint · Pylint · CheckStyle | + +--- + +## Troubleshooting + +### Container fails to build + +1. Check Docker is running +2. Ensure sufficient disk space (>5 GB free) +3. Review Docker build logs for specific errors + +### Tool not available after build + +1. Check the tool is `true` in `devcontainer.env.json` +2. Rebuild the container: **"Dev Containers: Rebuild Container"** +3. Run `python3 .devcontainer/configure.py` manually inside the container diff --git a/hugo-docs/content/docs/getting-started/_index.md b/hugo-docs/content/docs/getting-started/_index.md new file mode 100644 index 0000000..cde3dd1 --- /dev/null +++ b/hugo-docs/content/docs/getting-started/_index.md @@ -0,0 +1,176 @@ +--- +title: "Getting Started" +weight: 10 +bookCollapseSection: true +--- + +# Getting Started with DevOps-OS + +Welcome! This guide walks you through DevOps-OS from **zero to your first generated pipeline** in under five minutes. + +--- + +## What is DevOps-OS? + +DevOps-OS is a toolkit that generates production-ready CI/CD pipelines, Kubernetes manifests, and SRE monitoring configs — so you can stop writing boilerplate and start shipping. + +| Category | Tools | +|----------|-------| +| CI/CD | GitHub Actions, GitLab CI, Jenkins | +| GitOps / Deploy | ArgoCD, Flux CD, kubectl, Kustomize | +| Containers | Docker, Helm | +| SRE / Observability | Prometheus alert rules, Grafana dashboards, SLO configs | +| AI Integration | Claude (MCP Server), OpenAI (function calling) | + +--- + +## Prerequisites + +| Requirement | Why | +|------------|-----| +| Python 3.10+ | Runs the CLI generators | +| pip | Installs Python dependencies | +| Git | Clones the repo | +| Docker *(optional)* | Builds / runs the dev container | +| VS Code + Dev Containers extension *(optional)* | Opens the pre-configured dev environment | + +--- + +## 1 — Clone and install + +```bash +git clone https://github.com/cloudengine-labs/devops_os.git +cd devops_os +``` + +**Set up a virtual environment** (strongly recommended): + +```bash +python -m venv .venv + +# Activate +source .venv/bin/activate # macOS / Linux +# .venv\Scripts\activate # Windows (cmd) +# .venv\Scripts\Activate.ps1 # Windows (PowerShell) +``` + +**Install the CLI dependencies:** + +```bash +pip install -r cli/requirements.txt +``` + +> [!WARNING] +> Run `source .venv/bin/activate` in every new terminal session before using `python -m cli.*` commands. + +--- + +## 2 — Generate your first CI/CD pipeline + +### GitHub Actions + +```bash +python -m cli.scaffold_gha --name my-app --languages python,javascript --type complete +``` + +**Output:** `.github/workflows/my-app-complete.yml` + +### GitLab CI + +```bash +python -m cli.scaffold_gitlab --name my-app --languages python --type complete +``` + +**Output:** `.gitlab-ci.yml` + +### Jenkins + +```bash +python -m cli.scaffold_jenkins --name my-app --languages java --type complete +``` + +**Output:** `Jenkinsfile` + +--- + +## 3 — Generate Kubernetes / GitOps configs + +```bash +# ArgoCD Application CR + AppProject +python -m cli.scaffold_argocd --name my-app \ + --repo https://github.com/myorg/my-app.git \ + --namespace production +# Output: argocd/application.yaml + argocd/appproject.yaml + +# Flux CD configs +python -m cli.scaffold_argocd --name my-app --method flux \ + --repo https://github.com/myorg/my-app.git +# Output: flux/ directory +``` + +--- + +## 4 — Generate SRE configs + +```bash +python -m cli.scaffold_sre --name my-app --team platform +``` + +**Output:** `sre/` directory containing: +- `alert-rules.yaml` — Prometheus PrometheusRule CR +- `grafana-dashboard.json` — Grafana importable dashboard +- `slo.yaml` — Sloth-compatible SLO manifest +- `alertmanager-config.yaml` — Alertmanager routing stub + +--- + +## 5 — Interactive wizard (all-in-one) + +```bash +python -m cli.devopsos init # interactive project configurator +python -m cli.devopsos scaffold gha # scaffold GitHub Actions +python -m cli.devopsos scaffold gitlab # scaffold GitLab CI +python -m cli.devopsos scaffold jenkins # scaffold Jenkins +python -m cli.devopsos scaffold argocd # scaffold ArgoCD / Flux +python -m cli.devopsos scaffold sre # scaffold SRE configs +``` + +--- + +## 6 — Use with an AI assistant + +```bash +pip install -r mcp_server/requirements.txt +python mcp_server/server.py +``` + +Add to `claude_desktop_config.json`: + +```json +{ + "mcpServers": { + "devops-os": { + "command": "python", + "args": ["-m", "mcp_server.server"], + "cwd": "/path/to/devops_os" + } + } +} +``` + +Then ask Claude: *"Generate a complete GitLab CI pipeline for a Python Flask API with Docker build and ArgoCD deployment."* + +--- + +## Next steps + +| I want to… | Read | +|-----------|------| +| See every CLI option and output path | [CLI Reference]({{< relref "/docs/reference" >}}) | +| Deep-dive GitHub Actions | [GitHub Actions]({{< relref "/docs/ci-cd/github-actions" >}}) | +| Deep-dive GitLab CI | [GitLab CI]({{< relref "/docs/ci-cd/gitlab-ci" >}}) | +| Deep-dive Jenkins | [Jenkins]({{< relref "/docs/ci-cd/jenkins" >}}) | +| Learn ArgoCD integration | [GitOps & ArgoCD]({{< relref "/docs/gitops" >}}) | +| Set up SRE monitoring configs | [SRE Configuration]({{< relref "/docs/sre" >}}) | +| Set up the dev container | [Dev Container]({{< relref "/docs/dev-container" >}}) | +| Use with Claude / ChatGPT | [AI Integration]({{< relref "/docs/ai-integration" >}}) | diff --git a/hugo-docs/content/docs/getting-started/quickstart.md b/hugo-docs/content/docs/getting-started/quickstart.md new file mode 100644 index 0000000..3af29a5 --- /dev/null +++ b/hugo-docs/content/docs/getting-started/quickstart.md @@ -0,0 +1,201 @@ +--- +title: "Quick Start Reference" +weight: 11 +--- + +# Quick Start Reference + +All essential CLI commands in one place, with default output paths for every generator. + +> [!NOTE] +> 📖 **Full reference:** For the complete option tables, input files, and all output locations see the [CLI Commands Reference]({{< relref "/docs/reference" >}}). + +--- + +## Installation + +```bash +git clone https://github.com/cloudengine-labs/devops_os.git +cd devops_os +python -m venv .venv && source .venv/bin/activate +pip install -r cli/requirements.txt +``` + +--- + +## GitHub Actions Workflows + +```bash +# Complete CI/CD pipeline +# Output: .github/workflows/my-app-complete.yml +python -m cli.scaffold_gha --name my-app --languages python,javascript --type complete + +# Build-only workflow +# Output: .github/workflows/my-app-build.yml +python -m cli.scaffold_gha --name my-app --type build + +# With Kubernetes deployment via ArgoCD +# Output: .github/workflows/my-app-complete.yml +python -m cli.scaffold_gha --name my-app --kubernetes --k8s-method argocd + +# Reusable workflow +# Output: .github/workflows/shared-reusable.yml +python -m cli.scaffold_gha --name shared --type reusable + +# Matrix build +python -m cli.scaffold_gha --name my-app --matrix +``` + +### Environment Variables + +```bash +export DEVOPS_OS_GHA_NAME=my-app +export DEVOPS_OS_GHA_TYPE=complete +export DEVOPS_OS_GHA_LANGUAGES=python,javascript +export DEVOPS_OS_GHA_KUBERNETES=true +export DEVOPS_OS_GHA_K8S_METHOD=kustomize +python -m cli.scaffold_gha +# Output: .github/workflows/my-app-complete.yml +``` + +--- + +## GitLab CI Pipelines + +```bash +# Complete pipeline +# Output: .gitlab-ci.yml +python -m cli.scaffold_gitlab --name my-app --languages python --type complete + +# With Kubernetes deployment via ArgoCD +# Output: .gitlab-ci.yml +python -m cli.scaffold_gitlab --name my-app --languages python,go \ + --kubernetes --k8s-method argocd + +# Custom output path +# Output: ci/my-pipeline.yml +python -m cli.scaffold_gitlab --name my-app --output ci/my-pipeline.yml +``` + +--- + +## Jenkins Pipelines + +```bash +# Complete pipeline +# Output: Jenkinsfile +python -m cli.scaffold_jenkins --name my-app --languages java --type complete + +# Parameterized pipeline +# Output: Jenkinsfile +python -m cli.scaffold_jenkins --name my-app --languages python --type parameterized + +# Custom output path +# Output: pipelines/Jenkinsfile +python -m cli.scaffold_jenkins --name my-app --output pipelines/Jenkinsfile +``` + +--- + +## GitOps — ArgoCD & Flux CD + +```bash +# ArgoCD Application + AppProject +# Output: argocd/application.yaml + argocd/appproject.yaml +python -m cli.scaffold_argocd --name my-app \ + --repo https://github.com/myorg/my-app.git \ + --namespace production + +# ArgoCD with automated sync + canary rollout +# Output: argocd/application.yaml + argocd/appproject.yaml + argocd/rollout.yaml +python -m cli.scaffold_argocd --name my-app \ + --repo https://github.com/myorg/my-app.git \ + --auto-sync --rollouts + +# Flux CD (GitRepository + Kustomization + Image Automation) +# Output: flux/git-repository.yaml + flux/kustomization.yaml + flux/image-update-automation.yaml +python -m cli.scaffold_argocd --name my-app --method flux \ + --repo https://github.com/myorg/my-app.git \ + --image ghcr.io/myorg/my-app +``` + +--- + +## SRE Configuration + +```bash +# All SRE configs +# Output: sre/alert-rules.yaml + sre/grafana-dashboard.json + sre/slo.yaml + sre/alertmanager-config.yaml +python -m cli.scaffold_sre --name my-app --team platform + +# Availability-only SLO +python -m cli.scaffold_sre --name my-app --slo-type availability --slo-target 99.9 + +# Latency SLO with 200ms threshold + PagerDuty alerting +python -m cli.scaffold_sre --name my-app --slo-type latency \ + --latency-threshold 0.2 \ + --pagerduty-key YOUR_PD_KEY \ + --slack-channel "#platform-alerts" + +# Custom output directory +# Output: monitoring/alert-rules.yaml (etc.) +python -m cli.scaffold_sre --name my-app --output-dir monitoring +``` + +--- + +## Dev Container Configuration + +```bash +# Python + Go dev container +# Output: .devcontainer/devcontainer.json + .devcontainer/devcontainer.env.json +python -m cli.scaffold_devcontainer \ + --languages python,go \ + --cicd-tools docker,kubectl,helm + +# Full-stack with Kubernetes tools +python -m cli.scaffold_devcontainer \ + --languages python,java,javascript \ + --cicd-tools docker,terraform,kubectl,helm \ + --kubernetes-tools k9s,kustomize,argocd_cli,flux \ + --devops-tools prometheus,grafana \ + --python-version 3.12 +``` + +--- + +## Common Options Quick Reference + +| Option | `scaffold_gha` | `scaffold_gitlab` | `scaffold_jenkins` | `scaffold_argocd` | `scaffold_sre` | +|--------|:-:|:-:|:-:|:-:|:-:| +| `--name` | ✓ | ✓ | ✓ | ✓ | ✓ | +| `--type` | ✓ | ✓ | ✓ | — | — | +| `--languages` | ✓ | ✓ | ✓ | — | — | +| `--kubernetes` | ✓ | ✓ | ✓ | — | — | +| `--k8s-method` | ✓ | ✓ | ✓ | — | — | +| `--output` | ✓ | ✓ | ✓ | — | — | +| `--output-dir` | — | — | — | ✓ | ✓ | +| `--custom-values` | ✓ | ✓ | ✓ | — | — | + +--- + +## Troubleshooting + +```bash +# Show help for each generator +python -m cli.scaffold_gha --help +python -m cli.scaffold_gitlab --help +python -m cli.scaffold_jenkins --help +python -m cli.scaffold_argocd --help +python -m cli.scaffold_sre --help +python -m cli.scaffold_devcontainer --help + +# Verify generated output +ls -la .github/workflows/ # GitHub Actions +cat .gitlab-ci.yml # GitLab CI +cat Jenkinsfile # Jenkins +ls -la argocd/ # ArgoCD +ls -la flux/ # Flux CD +ls -la sre/ # SRE configs +ls -la .devcontainer/ # Dev container +``` diff --git a/hugo-docs/content/docs/gitops/_index.md b/hugo-docs/content/docs/gitops/_index.md new file mode 100644 index 0000000..7ee75a8 --- /dev/null +++ b/hugo-docs/content/docs/gitops/_index.md @@ -0,0 +1,184 @@ +--- +title: "GitOps — ArgoCD & Flux CD" +weight: 30 +--- + +# GitOps Configuration Generator + +DevOps-OS generates production-ready GitOps configuration files for both **ArgoCD** and **Flux CD** in a single command. + +--- + +## Quick Start + +```bash +# ArgoCD Application + AppProject +python -m cli.scaffold_argocd \ + --name my-app \ + --repo https://github.com/myorg/my-app.git \ + --namespace production +# Output: argocd/application.yaml + argocd/appproject.yaml + +# Flux CD (GitRepository + Kustomization + Image Automation) +python -m cli.scaffold_argocd \ + --name my-app --method flux \ + --repo https://github.com/myorg/my-app.git \ + --image ghcr.io/myorg/my-app +# Output: flux/git-repository.yaml + flux/kustomization.yaml + flux/image-update-automation.yaml +``` + +--- + +## Options + +| Option | Default | Description | +|--------|---------|-------------| +| `--name NAME` | `my-app` | Application name | +| `--method METHOD` | `argocd` | GitOps tool: `argocd` \| `flux` | +| `--repo URL` | `https://github.com/myorg/my-app.git` | Git repository URL | +| `--revision REV` | `HEAD` | Branch / tag / commit to sync | +| `--path PATH` | `k8s` | Path inside the repository to manifests | +| `--namespace NS` | `default` | Target Kubernetes namespace | +| `--project PROJECT` | `default` | ArgoCD project name | +| `--server URL` | `https://kubernetes.default.svc` | Destination Kubernetes API server | +| `--auto-sync` | off | Enable automated sync (prune + self-heal) | +| `--rollouts` | off | Add an Argo Rollouts canary strategy resource | +| `--image IMAGE` | `ghcr.io/myorg/my-app` | Container image (for Rollouts / Flux image automation) | +| `--output-dir DIR` | `.` | Root directory for all output files | +| `--allow-any-source-repo` | off | Allow `*` in AppProject sourceRepos (use with caution) | + +All options can be set via environment variables prefixed `DEVOPS_OS_ARGOCD_`. + +--- + +## ArgoCD Output Files + +``` +argocd/ +├── application.yaml ArgoCD Application CR +├── appproject.yaml ArgoCD AppProject CR +└── rollout.yaml Argo Rollouts Rollout (only with --rollouts) +``` + +### application.yaml + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: my-app + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/myorg/my-app.git + targetRevision: HEAD + path: k8s + destination: + server: https://kubernetes.default.svc + namespace: production + syncPolicy: + syncOptions: + - CreateNamespace=true +``` + +### appproject.yaml + +Restricts which repositories and namespaces the application can target (principle of least privilege). + +### rollout.yaml (`--rollouts`) + +Generates an Argo Rollouts **canary strategy** that gradually shifts traffic: +``` +10% → 1 min wait → 30% → 2 min wait → 60% → 2 min wait → 100% +``` + +--- + +## Flux CD Output Files + +``` +flux/ +├── git-repository.yaml Flux GitRepository source +├── kustomization.yaml Flux Kustomization +└── image-update-automation.yaml ImageRepository + ImagePolicy + ImageUpdateAutomation +``` + +The image update automation configures Flux to watch the container registry and automatically open a commit/PR when a new semver-compatible image is pushed. + +--- + +## Examples + +### ArgoCD with automated sync and canary rollout + +```bash +python -m cli.scaffold_argocd \ + --name my-app \ + --repo https://github.com/myorg/my-app.git \ + --auto-sync --rollouts +# Output: argocd/application.yaml +# argocd/appproject.yaml +# argocd/rollout.yaml +``` + +### ArgoCD in a custom output directory + +```bash +python -m cli.scaffold_argocd \ + --name my-app \ + --repo https://github.com/myorg/my-app.git \ + --namespace production \ + --output-dir gitops +# Output: gitops/argocd/application.yaml +# gitops/argocd/appproject.yaml +``` + +### Flux CD with image automation + +```bash +python -m cli.scaffold_argocd \ + --name my-app --method flux \ + --repo https://github.com/myorg/my-app.git \ + --image ghcr.io/myorg/my-app \ + --output-dir gitops +# Output: gitops/flux/git-repository.yaml +# gitops/flux/kustomization.yaml +# gitops/flux/image-update-automation.yaml +``` + +--- + +## Applying the Generated Configs + +### ArgoCD + +```bash +# Apply AppProject first (it constrains the Application) +kubectl apply -f argocd/appproject.yaml +kubectl apply -f argocd/application.yaml + +# Watch sync status +argocd app get my-app +argocd app sync my-app +``` + +### Flux CD + +```bash +# Bootstrap Flux (first time only) +flux bootstrap github \ + --owner=myorg \ + --repository=my-app \ + --branch=main \ + --path=flux + +# Apply generated resources +kubectl apply -f flux/git-repository.yaml +kubectl apply -f flux/kustomization.yaml +kubectl apply -f flux/image-update-automation.yaml + +# Watch reconciliation +flux get kustomizations +flux logs --follow +``` diff --git a/hugo-docs/content/docs/kubernetes/_index.md b/hugo-docs/content/docs/kubernetes/_index.md new file mode 100644 index 0000000..105bd1f --- /dev/null +++ b/hugo-docs/content/docs/kubernetes/_index.md @@ -0,0 +1,142 @@ +--- +title: "Kubernetes" +weight: 50 +--- + +# Kubernetes Deployment Configuration + +DevOps-OS provides a Kubernetes configuration generator (`k8s-config-generator.py`) that creates YAML manifest files for deploying your applications to Kubernetes clusters. + +--- + +## Basic Usage + +```bash +python kubernetes/k8s-config-generator.py --name my-app --image ghcr.io/myorg/my-app:v1 +``` + +Generated files are written to the `./k8s` directory by default. + +--- + +## Options + +| Option | Default | Description | +|--------|---------|-------------| +| `--name NAME` | `app` | Application name | +| `--namespace NS` | `default` | Kubernetes namespace | +| `--image IMAGE` | `docker.io/yourorg/app:latest` | Container image | +| `--port PORT` | `8080` | Port your application listens on | +| `--replicas N` | `1` | Number of pod replicas | +| `--output DIR` | `./k8s` | Output directory for generated files | +| `--containers NAMES` | _(single)_ | Comma-separated container names for multi-container pods | +| `--ingress` | off | Add an Ingress resource | +| `--storage` | off | Add a PersistentVolumeClaim | +| `--kustomize` | off | Add Kustomize base + overlays structure | +| `--environments ENVS` | _(single)_ | Comma-separated environments (e.g. `dev,staging,prod`) | + +--- + +## Examples + +### Single container application + +```bash +python kubernetes/k8s-config-generator.py \ + --name backend-api \ + --namespace backend \ + --image docker.io/myorg/api:v1.0 \ + --port 3000 \ + --replicas 3 +# Output: k8s/deployment.yaml + k8s/service.yaml +``` + +### With Ingress and persistent storage + +```bash +python kubernetes/k8s-config-generator.py \ + --name web-app \ + --ingress \ + --storage +# Output: k8s/deployment.yaml + k8s/service.yaml + k8s/ingress.yaml + k8s/pvc.yaml +``` + +### Multi-environment with Kustomize + +```bash +python kubernetes/k8s-config-generator.py \ + --name my-app \ + --kustomize \ + --environments dev,staging,prod +# Output: k8s/base/ + k8s/overlays/dev/ + k8s/overlays/staging/ + k8s/overlays/prod/ +``` + +--- + +## Applying Configurations + +```bash +# Apply with kubectl +kubectl apply -f k8s/deployment.yaml +kubectl apply -f k8s/service.yaml + +# Apply all files +kubectl apply -f k8s/ + +# Apply with Kustomize overlay +kubectl apply -k k8s/overlays/dev + +# Deploy with ArgoCD (after generating ArgoCD configs) +python -m cli.scaffold_argocd --name my-app \ + --repo https://github.com/myorg/my-app.git \ + --path k8s --namespace production +kubectl apply -f argocd/appproject.yaml +kubectl apply -f argocd/application.yaml +``` + +--- + +## Integration with CI/CD Generators + +Use the Kubernetes generator alongside the CI/CD generators for a complete pipeline: + +```bash +# Generate the K8s manifests +python kubernetes/k8s-config-generator.py --name my-api --replicas 3 + +# Generate a GitHub Actions workflow that deploys to K8s +python -m cli.scaffold_gha \ + --name my-api \ + --kubernetes \ + --k8s-method kustomize + +# Or generate an ArgoCD Application pointing to the k8s/ directory +python -m cli.scaffold_argocd \ + --name my-api \ + --repo https://github.com/myorg/my-api.git \ + --path k8s \ + --auto-sync +``` + +--- + +## GitOps Deployment + +For GitOps-based deployment, generate ArgoCD or Flux configs that watch the `k8s/` directory in your repository: + +```bash +# ArgoCD watches k8s/ directory +python -m cli.scaffold_argocd \ + --name my-app \ + --path k8s \ + --repo https://github.com/myorg/my-app.git + +# Flux CD watches k8s/ directory +python -m cli.scaffold_argocd \ + --name my-app \ + --method flux \ + --path k8s \ + --repo https://github.com/myorg/my-app.git +``` + +Commit the generated `k8s/` directory to your repo — ArgoCD/Flux will automatically apply changes on every push. diff --git a/hugo-docs/content/docs/reference/_index.md b/hugo-docs/content/docs/reference/_index.md new file mode 100644 index 0000000..f5e394d --- /dev/null +++ b/hugo-docs/content/docs/reference/_index.md @@ -0,0 +1,224 @@ +--- +title: "CLI Reference" +weight: 80 +bookFlatSection: false +--- + +# CLI Commands Reference + +Complete reference for every DevOps-OS CLI command: options, default values, environment variable equivalents, input files, and exact output paths. + +--- + +## Command Overview + +| Command | Invocation | Default output | +|---------|-----------|----------------| +| GitHub Actions | `python -m cli.scaffold_gha` | `.github/workflows/-.yml` | +| GitLab CI | `python -m cli.scaffold_gitlab` | `.gitlab-ci.yml` | +| Jenkins | `python -m cli.scaffold_jenkins` | `Jenkinsfile` | +| ArgoCD | `python -m cli.scaffold_argocd` | `argocd/` directory | +| Flux CD | `python -m cli.scaffold_argocd --method flux` | `flux/` directory | +| SRE configs | `python -m cli.scaffold_sre` | `sre/` directory | +| Dev Container | `python -m cli.scaffold_devcontainer` | `.devcontainer/` directory | +| Interactive wizard | `python -m cli.devopsos` | varies | + +--- + +## scaffold_gha — GitHub Actions + +```bash +python -m cli.scaffold_gha [options] +``` + +| Option | Env var | Default | Description | +|--------|---------|---------|-------------| +| `--name NAME` | `DEVOPS_OS_GHA_NAME` | `DevOps-OS` | Workflow name | +| `--type TYPE` | `DEVOPS_OS_GHA_TYPE` | `complete` | `build` \| `test` \| `deploy` \| `complete` \| `reusable` | +| `--languages LANGS` | `DEVOPS_OS_GHA_LANGUAGES` | `python,javascript` | Comma-separated languages | +| `--kubernetes` | `DEVOPS_OS_GHA_KUBERNETES` | `false` | Include Kubernetes steps | +| `--registry URL` | `DEVOPS_OS_GHA_REGISTRY` | `ghcr.io` | Container registry | +| `--k8s-method METHOD` | `DEVOPS_OS_GHA_K8S_METHOD` | `kubectl` | `kubectl` \| `kustomize` \| `argocd` \| `flux` | +| `--output DIR` | `DEVOPS_OS_GHA_OUTPUT` | `.github/workflows` | Output directory | +| `--custom-values FILE` | `DEVOPS_OS_GHA_CUSTOM_VALUES` | _(none)_ | Custom values JSON | +| `--image IMAGE` | `DEVOPS_OS_GHA_IMAGE` | `ghcr.io/yourorg/devops-os:latest` | DevOps-OS image | +| `--branches BRANCHES` | `DEVOPS_OS_GHA_BRANCHES` | `main` | Trigger branches | +| `--matrix` | `DEVOPS_OS_GHA_MATRIX` | `false` | Enable matrix builds | +| `--env-file FILE` | `DEVOPS_OS_GHA_ENV_FILE` | _(cli dir)_ | `devcontainer.env.json` path | +| `--reusable` | `DEVOPS_OS_GHA_REUSABLE` | `false` | Generate reusable workflow | + +**Output:** `/-.yml` + +--- + +## scaffold_gitlab — GitLab CI + +```bash +python -m cli.scaffold_gitlab [options] +``` + +| Option | Env var | Default | Description | +|--------|---------|---------|-------------| +| `--name NAME` | `DEVOPS_OS_GITLAB_NAME` | `my-app` | Pipeline name | +| `--type TYPE` | `DEVOPS_OS_GITLAB_TYPE` | `complete` | `build` \| `test` \| `deploy` \| `complete` | +| `--languages LANGS` | `DEVOPS_OS_GITLAB_LANGUAGES` | `python` | Comma-separated languages | +| `--kubernetes` | `DEVOPS_OS_GITLAB_KUBERNETES` | `false` | Add Kubernetes deploy stage | +| `--k8s-method METHOD` | `DEVOPS_OS_GITLAB_K8S_METHOD` | `kubectl` | Deploy method | +| `--output FILE` | `DEVOPS_OS_GITLAB_OUTPUT` | `.gitlab-ci.yml` | Output file path | +| `--image IMAGE` | `DEVOPS_OS_GITLAB_IMAGE` | `docker:24` | Default pipeline image | +| `--branches BRANCHES` | `DEVOPS_OS_GITLAB_BRANCHES` | `main` | Protected branches | +| `--custom-values FILE` | _(not in env)_ | _(none)_ | Custom values JSON | + +**Output:** `` (default: `.gitlab-ci.yml`) + +--- + +## scaffold_jenkins — Jenkins + +```bash +python -m cli.scaffold_jenkins [options] +``` + +| Option | Env var | Default | Description | +|--------|---------|---------|-------------| +| `--name NAME` | `DEVOPS_OS_JENKINS_NAME` | `DevOps-OS` | Pipeline name | +| `--type TYPE` | `DEVOPS_OS_JENKINS_TYPE` | `complete` | `build` \| `test` \| `deploy` \| `complete` \| `parameterized` | +| `--languages LANGS` | `DEVOPS_OS_JENKINS_LANGUAGES` | `python,javascript` | Comma-separated languages | +| `--kubernetes` | `DEVOPS_OS_JENKINS_KUBERNETES` | `false` | Add Kubernetes stage | +| `--registry URL` | `DEVOPS_OS_JENKINS_REGISTRY` | `docker.io` | Registry URL | +| `--k8s-method METHOD` | `DEVOPS_OS_JENKINS_K8S_METHOD` | `kubectl` | Deploy method | +| `--output FILE` | `DEVOPS_OS_JENKINS_OUTPUT` | `Jenkinsfile` | Output file path | +| `--custom-values FILE` | `DEVOPS_OS_JENKINS_CUSTOM_VALUES` | _(none)_ | Custom values JSON | +| `--image IMAGE` | `DEVOPS_OS_JENKINS_IMAGE` | `docker.io/yourorg/devops-os:latest` | DevOps-OS image | +| `--scm SCM` | `DEVOPS_OS_JENKINS_SCM` | `git` | `git` \| `svn` \| `none` | +| `--parameters` | `DEVOPS_OS_JENKINS_PARAMETERS` | `false` | Add runtime parameters | +| `--env-file FILE` | `DEVOPS_OS_JENKINS_ENV_FILE` | _(cli dir)_ | `devcontainer.env.json` path | + +**Output:** `` (default: `Jenkinsfile`) + +--- + +## scaffold_argocd — ArgoCD / Flux CD + +```bash +python -m cli.scaffold_argocd [options] +``` + +| Option | Env var | Default | Description | +|--------|---------|---------|-------------| +| `--name NAME` | `DEVOPS_OS_ARGOCD_NAME` | `my-app` | Application name | +| `--method METHOD` | `DEVOPS_OS_ARGOCD_METHOD` | `argocd` | `argocd` \| `flux` | +| `--repo URL` | `DEVOPS_OS_ARGOCD_REPO` | `https://github.com/myorg/my-app.git` | Git repo URL | +| `--revision REV` | `DEVOPS_OS_ARGOCD_REVISION` | `HEAD` | Branch / tag / SHA | +| `--path PATH` | `DEVOPS_OS_ARGOCD_PATH` | `k8s` | Manifest path in repo | +| `--namespace NS` | `DEVOPS_OS_ARGOCD_NAMESPACE` | `default` | Target namespace | +| `--project PROJECT` | `DEVOPS_OS_ARGOCD_PROJECT` | `default` | ArgoCD project | +| `--server URL` | `DEVOPS_OS_ARGOCD_SERVER` | `https://kubernetes.default.svc` | Destination API server | +| `--auto-sync` | `DEVOPS_OS_ARGOCD_AUTO_SYNC` | `false` | Enable automated sync | +| `--rollouts` | `DEVOPS_OS_ARGOCD_ROLLOUTS` | `false` | Add Argo Rollouts canary | +| `--image IMAGE` | `DEVOPS_OS_ARGOCD_IMAGE` | `ghcr.io/myorg/my-app` | Container image | +| `--output-dir DIR` | `DEVOPS_OS_ARGOCD_OUTPUT_DIR` | `.` | Root output directory | +| `--allow-any-source-repo` | `DEVOPS_OS_ARGOCD_ALLOW_ANY_SOURCE_REPO` | `false` | Allow `*` in AppProject sourceRepos | + +**Output (ArgoCD):** `/argocd/application.yaml` + `appproject.yaml` + optional `rollout.yaml` +**Output (Flux):** `/flux/git-repository.yaml` + `kustomization.yaml` + `image-update-automation.yaml` + +--- + +## scaffold_sre — SRE Configuration + +```bash +python -m cli.scaffold_sre [options] +``` + +| Option | Env var | Default | Description | +|--------|---------|---------|-------------| +| `--name NAME` | `DEVOPS_OS_SRE_NAME` | `my-app` | Service name | +| `--team TEAM` | `DEVOPS_OS_SRE_TEAM` | `platform` | Owning team | +| `--namespace NS` | `DEVOPS_OS_SRE_NAMESPACE` | `default` | Kubernetes namespace | +| `--slo-type TYPE` | `DEVOPS_OS_SRE_SLO_TYPE` | `all` | `availability` \| `latency` \| `error_rate` \| `all` | +| `--slo-target PCT` | `DEVOPS_OS_SRE_SLO_TARGET` | `99.9` | SLO target percentage | +| `--latency-threshold SEC` | `DEVOPS_OS_SRE_LATENCY_THRESHOLD` | `0.5` | Latency SLI threshold (seconds) | +| `--pagerduty-key KEY` | `DEVOPS_OS_SRE_PAGERDUTY_KEY` | _(empty)_ | PagerDuty integration key | +| `--slack-channel CHANNEL` | `DEVOPS_OS_SRE_SLACK_CHANNEL` | `#alerts` | Slack channel | +| `--output-dir DIR` | `DEVOPS_OS_SRE_OUTPUT_DIR` | `sre` | Output directory | + +**Output:** `/alert-rules.yaml` + `grafana-dashboard.json` + `slo.yaml` + `alertmanager-config.yaml` + +--- + +## scaffold_devcontainer — Dev Container + +```bash +python -m cli.scaffold_devcontainer [options] +``` + +| Option | Env var | Default | Description | +|--------|---------|---------|-------------| +| `--languages LANGS` | `DEVOPS_OS_DEVCONTAINER_LANGUAGES` | `python` | Languages to enable | +| `--cicd-tools TOOLS` | `DEVOPS_OS_DEVCONTAINER_CICD_TOOLS` | `docker,github_actions` | CI/CD tools | +| `--kubernetes-tools TOOLS` | `DEVOPS_OS_DEVCONTAINER_KUBERNETES_TOOLS` | _(none)_ | Kubernetes tools | +| `--build-tools TOOLS` | `DEVOPS_OS_DEVCONTAINER_BUILD_TOOLS` | _(none)_ | Build tools | +| `--code-analysis TOOLS` | `DEVOPS_OS_DEVCONTAINER_CODE_ANALYSIS` | _(none)_ | Code analysis tools | +| `--devops-tools TOOLS` | `DEVOPS_OS_DEVCONTAINER_DEVOPS_TOOLS` | _(none)_ | DevOps tools | +| `--python-version VER` | `DEVOPS_OS_DEVCONTAINER_PYTHON_VERSION` | `3.11` | Python version | +| `--java-version VER` | `DEVOPS_OS_DEVCONTAINER_JAVA_VERSION` | `17` | Java JDK version | +| `--node-version VER` | `DEVOPS_OS_DEVCONTAINER_NODE_VERSION` | `20` | Node.js version | +| `--go-version VER` | `DEVOPS_OS_DEVCONTAINER_GO_VERSION` | `1.21` | Go version | +| `--output-dir DIR` | `DEVOPS_OS_DEVCONTAINER_OUTPUT_DIR` | `.` | Root directory | + +**Output:** `/.devcontainer/devcontainer.json` + `devcontainer.env.json` + +--- + +## devopsos — Unified CLI + +```bash +python -m cli.devopsos COMMAND [options] +``` + +| Command | Description | +|---------|-------------| +| `init` | Interactive wizard (select languages, tools, generate dev container) | +| `scaffold TARGET` | Non-interactive generator (delegates to `scaffold_*` modules) | + +### Scaffold targets + +```bash +python -m cli.devopsos scaffold gha # GitHub Actions +python -m cli.devopsos scaffold gitlab # GitLab CI +python -m cli.devopsos scaffold jenkins # Jenkins +python -m cli.devopsos scaffold argocd # ArgoCD / Flux +python -m cli.devopsos scaffold sre # SRE configs +python -m cli.devopsos scaffold devcontainer # Dev container +``` + +--- + +## Input File Formats + +### devcontainer.env.json (used by `--env-file`) + +```json +{ + "languages": { "python": true, "java": false, "javascript": true, "go": false }, + "cicd": { "docker": true, "terraform": false, "kubectl": true, "helm": true }, + "kubernetes": { "k9s": false, "kustomize": true, "argocd_cli": false, "flux": false }, + "build_tools": { "gradle": false, "maven": true, "ant": false, "make": true }, + "code_analysis": { "sonarqube": false, "eslint": true, "pylint": true }, + "devops_tools": { "nexus": false, "prometheus": false, "grafana": false }, + "versions": { "python": "3.11", "java": "17", "node": "20", "go": "1.21" } +} +``` + +### custom-values.json (used by `--custom-values`) + +```json +{ + "build": { "cache": true, "timeout_minutes": 30 }, + "test": { "coverage": true, "junit_reports": true, "parallel": 4 }, + "deploy": { "environments": ["dev", "staging", "prod"], "approval_required": true }, + "credentials": { "docker": "docker-registry-credentials", "kubernetes": "kubeconfig" }, + "matrix": { "os": ["ubuntu-latest", "windows-latest"], "architecture": ["x86_64", "arm64"] } +} +``` diff --git a/hugo-docs/content/docs/sre/_index.md b/hugo-docs/content/docs/sre/_index.md new file mode 100644 index 0000000..ce85f2b --- /dev/null +++ b/hugo-docs/content/docs/sre/_index.md @@ -0,0 +1,155 @@ +--- +title: "SRE Configuration" +weight: 40 +--- + +# SRE Configuration Generator + +DevOps-OS generates production-grade SRE configuration files for any service with a single command — covering alerting, dashboards, SLOs, and routing. + +--- + +## Quick Start + +```bash +# Generate all SRE configs +python -m cli.scaffold_sre --name my-app --team platform +# Output: sre/alert-rules.yaml +# sre/grafana-dashboard.json +# sre/slo.yaml +# sre/alertmanager-config.yaml + +# Availability-only SLO with 99.9% target +python -m cli.scaffold_sre --name my-app --slo-type availability --slo-target 99.9 + +# Latency SLO with 200ms threshold +python -m cli.scaffold_sre --name my-app --slo-type latency --latency-threshold 0.2 + +# Send critical alerts to PagerDuty +python -m cli.scaffold_sre --name my-app \ + --pagerduty-key YOUR_PD_KEY \ + --slack-channel "#platform-alerts" +``` + +--- + +## Options + +| Option | Default | Description | +|--------|---------|-------------| +| `--name NAME` | `my-app` | Application / service name | +| `--team TEAM` | `platform` | Owning team (used in labels and alert routing) | +| `--namespace NS` | `default` | Kubernetes namespace where the app runs | +| `--slo-type TYPE` | `all` | `availability` \| `latency` \| `error_rate` \| `all` | +| `--slo-target PCT` | `99.9` | SLO target as a percentage (e.g. `99.5`) | +| `--latency-threshold SEC` | `0.5` | Latency SLI threshold in seconds | +| `--pagerduty-key KEY` | _(empty)_ | PagerDuty integration key; omit to skip PD routing | +| `--slack-channel CHANNEL` | `#alerts` | Slack channel for alert routing | +| `--output-dir DIR` | `sre` | Directory where all output files are written | + +All options can be set via environment variables prefixed `DEVOPS_OS_SRE_`. + +--- + +## Generated Files + +``` +sre/ +├── alert-rules.yaml Prometheus PrometheusRule CR +├── grafana-dashboard.json Grafana dashboard (importable via API or UI) +├── slo.yaml Sloth-compatible SLO manifest +└── alertmanager-config.yaml Alertmanager routing config stub +``` + +--- + +## alert-rules.yaml + +A `PrometheusRule` Custom Resource compatible with **kube-prometheus-stack**. + +```bash +kubectl apply -f sre/alert-rules.yaml +``` + +### Included alert groups + +| Group | Alerts | +|-------|--------| +| `.availability` | `HighErrorRate`, `SLOBurnRate` | +| `.latency` | `HighLatency`, `LatencyBudgetBurn` | +| `.infrastructure` | `PodRestartingFrequently`, `DeploymentReplicasMismatch` | + +**SLO burn-rate alerts** fire before you exhaust your error budget. + +--- + +## grafana-dashboard.json + +A ready-to-import Grafana dashboard with six panels: + +| Panel | Metric | +|-------|--------| +| Request Rate (RPS) | `http_requests_total` | +| Error Rate | 5xx ratio | +| p99 Latency | histogram quantile | +| Pod Restarts | `kube_pod_container_status_restarts_total` | +| CPU Usage | `container_cpu_usage_seconds_total` | +| Memory Usage | `container_memory_working_set_bytes` | + +### Import the dashboard + +```bash +# Via Grafana HTTP API +curl -X POST http://localhost:3000/api/dashboards/import \ + -H "Content-Type: application/json" \ + -d "{\"dashboard\": $(cat sre/grafana-dashboard.json), \"overwrite\": true}" +``` + +Or use **Grafana → Dashboards → Import → Upload JSON file**. + +--- + +## slo.yaml + +A [Sloth](https://sloth.dev)-compatible SLO manifest that Sloth converts into multi-window, multi-burn-rate Prometheus recording rules and alerts: + +```bash +sloth generate -i sre/slo.yaml -o sre/slo-rules.yaml +kubectl apply -f sre/slo-rules.yaml +``` + +--- + +## alertmanager-config.yaml + +An Alertmanager routing configuration that: + +1. Routes **all alerts** to your Slack channel +2. Routes **critical alerts** to PagerDuty *(if `--pagerduty-key` is set)* +3. Inhibits duplicate `warning` alerts when a `critical` alert is firing for the same service + +### Apply to the cluster (kube-prometheus-stack) + +```bash +kubectl create secret generic alertmanager-kube-prometheus-stack-alertmanager \ + --from-file=alertmanager.yaml=sre/alertmanager-config.yaml \ + --namespace monitoring --dry-run=client -o yaml | kubectl apply -f - +``` + +--- + +## Prerequisites + +Your application must expose Prometheus-compatible metrics: + +| Metric | Description | +|--------|-------------| +| `http_requests_total{status}` | Request count, labelled by HTTP status | +| `http_request_duration_seconds_bucket{le}` | Latency histogram | + +Auto-instrumentation libraries: + +- **Python**: `prometheus-flask-exporter`, `starlette-prometheus` +- **Java**: `micrometer-registry-prometheus` +- **Go**: `prometheus/client_golang` +- **Node.js**: `prom-client` diff --git a/hugo-docs/go.mod b/hugo-docs/go.mod new file mode 100644 index 0000000..c363f3e --- /dev/null +++ b/hugo-docs/go.mod @@ -0,0 +1,5 @@ +module github.com/cloudengine-labs/devops_os/hugo-docs + +go 1.21.0 + +require github.com/alex-shpak/hugo-book v0.0.0-20260108111751-81a841c92d62 // indirect diff --git a/hugo-docs/go.sum b/hugo-docs/go.sum new file mode 100644 index 0000000..2510024 --- /dev/null +++ b/hugo-docs/go.sum @@ -0,0 +1,2 @@ +github.com/alex-shpak/hugo-book v0.0.0-20260108111751-81a841c92d62 h1:LQYO67RaSBiWnW3ZPyVPZCVTOFaNK/dc/ZRbh8Wu8Wo= +github.com/alex-shpak/hugo-book v0.0.0-20260108111751-81a841c92d62/go.mod h1:L4NMyzbn15fpLIpmmtDg9ZFFyTZzw87/lk7M2bMQ7ds= diff --git a/hugo-docs/hugo.toml b/hugo-docs/hugo.toml new file mode 100644 index 0000000..1c97f20 --- /dev/null +++ b/hugo-docs/hugo.toml @@ -0,0 +1,50 @@ +baseURL = 'https://cloudengine-labs.github.io/devops_os/' +languageCode = 'en-us' +title = 'DevOps-OS' + +# Enable Git info for Last Modified dates +enableGitInfo = true + +# Handle HTML in Markdown +[markup.goldmark.renderer] + unsafe = true + +[markup.highlight] + style = 'github' + lineNos = false + codeFences = true + noClasses = false + +# Hugo Book Theme Parameters +[params] + # light, dark, or auto + BookTheme = 'auto' + + # Show table-of-contents on each page + BookToC = true + + # Path to favicon (relative to static/) + BookFavicon = '/favicon.png' + + # GitHub repo for "Edit this page" links + BookRepo = 'https://github.com/cloudengine-labs/devops_os' + + # Path segment after BookRepo for "Edit this page" links + BookEditPath = 'edit/main/hugo-docs/content' + + # Enable in-page search (powered by lunr.js) + BookSearch = true + + # Disable comments + BookComments = false + + # Enable portable links (useful for offline builds) + BookPortableLinks = false + + # Enable service worker for offline support + BookServiceWorker = false + +# Hugo modules: pull in the Book theme +[module] + [[module.imports]] + path = 'github.com/alex-shpak/hugo-book'