diff --git a/.env.example b/.env.example index 10b1c64..dd6da70 100644 --- a/.env.example +++ b/.env.example @@ -27,6 +27,17 @@ SNOWFLAKE_DATABASE= SNOWFLAKE_SCHEMA= SNOWFLAKE_WAREHOUSE= +# --------------------------------------------------------------------------- +# AWS telemetry mirror (optional — disabled for local development) +# --------------------------------------------------------------------------- +# When enabled, accepted inference logs are queued and mirrored to Amazon Data +# Firehose. Use the AWS SDK default credential chain; prefer workload roles/IRSA +# in AWS instead of committing long-lived access keys. +FIREHOSE_ENABLED=false +FIREHOSE_DELIVERY_STREAM=sentinelai-telemetry +FIREHOSE_QUEUE_SIZE=1000 +AWS_REGION=us-east-1 + # --------------------------------------------------------------------------- # LLM / Ollama (optional — llm-guard falls back to stub if unreachable) # --------------------------------------------------------------------------- diff --git a/.github/workflows/aws-telemetry.yml b/.github/workflows/aws-telemetry.yml new file mode 100644 index 0000000..f9435e5 --- /dev/null +++ b/.github/workflows/aws-telemetry.yml @@ -0,0 +1,75 @@ +name: AWS Telemetry Validation + +on: + pull_request: + branches: [main] + paths: + - "ingestion-service/**" + - "terraform/**" + - ".github/workflows/aws-telemetry.yml" + push: + branches: [main] + paths: + - "ingestion-service/**" + - "terraform/**" + - ".github/workflows/aws-telemetry.yml" + +permissions: + contents: read + +jobs: + firehose-producer: + name: Firehose producer unit tests + runs-on: ubuntu-latest + timeout-minutes: 10 + + defaults: + run: + working-directory: ingestion-service + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.24.x" + cache-dependency-path: ingestion-service/go.sum + + - name: Verify module files are tidy + run: | + go mod tidy + git diff --exit-code -- go.mod go.sum + + - name: Run ingestion tests + run: go test ./... + + terraform: + name: Terraform fmt and validate + runs-on: ubuntu-latest + timeout-minutes: 10 + + defaults: + run: + working-directory: terraform + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.13.3" + + - name: Verify Terraform formatting + run: | + terraform fmt -recursive + git diff --exit-code -- . + + - name: Initialize providers without backend + run: terraform init -backend=false -input=false + + - name: Validate configuration + run: terraform validate diff --git a/README.md b/README.md index 5509e42..2991ac1 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ # SentinelAI — Reproducible Drift-Monitoring Reference System [![CI](https://github.com/CoreyLeath-code/SentinelAI/actions/workflows/ci-cd.yml/badge.svg)](https://github.com/CoreyLeath-code/SentinelAI/actions/workflows/ci-cd.yml) +[![AWS telemetry](https://github.com/CoreyLeath-code/SentinelAI/actions/workflows/aws-telemetry.yml/badge.svg)](https://github.com/CoreyLeath-code/SentinelAI/actions/workflows/aws-telemetry.yml) [![Research benchmark](https://github.com/CoreyLeath-code/SentinelAI/actions/workflows/benchmarks.yml/badge.svg)](https://github.com/CoreyLeath-code/SentinelAI/actions/workflows/benchmarks.yml) [![Security](https://github.com/CoreyLeath-code/SentinelAI/actions/workflows/security.yml/badge.svg)](https://github.com/CoreyLeath-code/SentinelAI/actions/workflows/security.yml) [![SAST](https://github.com/CoreyLeath-code/SentinelAI/actions/workflows/sast.yml/badge.svg)](https://github.com/CoreyLeath-code/SentinelAI/actions/workflows/sast.yml) @@ -17,7 +18,9 @@ SentinelAI is a multi-service observability prototype for AI systems. Its directly implemented statistical component compares an expected and an observed histogram with Population Stability Index (PSI) and a Kolmogorov–Smirnov (KS) CDF distance, then raises a drift flag when either configured threshold is crossed. -The versioned evidence measures a portable Python reference of that decision rule on seeded 32-bin synthetic histograms—not native C++ execution, HTTP latency, concurrent service load, or production drift-detection accuracy. The benchmark is a repeatable regression signal, not a claim of real-world model quality. +The Go ingestion service can optionally mirror accepted inference telemetry to Amazon Data Firehose, which Terraform configures to deliver GZIP-compressed NDJSON objects into a private, versioned, encrypted Amazon S3 telemetry bucket. This AWS path is disabled by default for local development and is a best-effort observability mirror, not a transactional dual-write guarantee. + +The versioned evidence measures a portable Python reference of the drift decision rule on seeded 32-bin synthetic histograms—not native C++ execution, HTTP latency, concurrent service load, Firehose/S3 throughput, or production drift-detection accuracy. The benchmark is a repeatable regression signal, not a claim of real-world model quality. ## Formal decision rule @@ -112,23 +115,25 @@ The intended use, excluded use, data/credential handling, model or algorithm lim **What should be completed next?** Use the linked production-readiness issue for this repository as the checklist. Resolve missing tests, deployment instructions, observability, supply-chain controls, and release evidence before attaching a production claim. - ## 🏛️ Advanced Platform Architecture & Telemetry Decoupling SentinelAI separates primary inference paths from telemetry and evaluation layers in its local architecture. + +```text [ Incoming User Query ] ───► [ Async Proxy Gateway ] ───► [ Downstream Application ] -│ -(Non-Blocking Telemetry Mirror) -▼ -┌──────────────────────────────────────┐ -│ SentinelAI Asynchronous Engine │ -├──────────────────────────────────────┤ -│ • Parallelized Guardrail Evaluation │ -│ • GPT-4 Intelligent SRE Diagnostics │ -│ • Token Cost & Allocation Trackers │ -└──────────────────┬───────────────────┘ -▼ -[ Streamlit Observability Control Plane ] + │ + (Non-Blocking Telemetry Mirror) + ▼ + ┌──────────────────────────────────────┐ + │ SentinelAI Asynchronous Engine │ + ├──────────────────────────────────────┤ + │ • Parallelized Guardrail Evaluation │ + │ • LLM SRE Diagnostics │ + │ • Token Cost & Allocation Trackers │ + └──────────────────┬───────────────────┘ + ▼ + [ Streamlit Observability Control Plane ] +``` ## 🚀 Quickstart — Docker Compose (recommended) @@ -181,7 +186,7 @@ curl -X POST http://localhost:8000/summarize \ ## ⚙️ Configuration -All configuration is via environment variables. Copy `.env.example` to `.env` and adjust. +All configuration is via environment variables. Copy `.env.example` to `.env` and adjust. | Variable | Default | Description | |----------|---------|-------------| @@ -190,6 +195,10 @@ All configuration is via environment variables. Copy `.env.example` to `.env` a | `POSTGRES_USER` | `sentinel` | Postgres user | | `POSTGRES_PASSWORD` | `sentinel` | Postgres password | | `POSTGRES_DB` | `sentinel` | Postgres database | +| `FIREHOSE_ENABLED` | `false` | Enables the optional non-blocking Amazon Data Firehose telemetry mirror | +| `FIREHOSE_DELIVERY_STREAM` | `sentinelai-telemetry` | Firehose delivery stream name | +| `FIREHOSE_QUEUE_SIZE` | `1000` | Maximum in-memory records waiting for Firehose delivery | +| `AWS_REGION` | `us-east-1` | AWS region used by the Firehose client | | `OLLAMA_HOST` | `http://ollama:11434` | Ollama endpoint (optional) | | `LLM_MODEL` | `llama2` | LLM model name | | `API_BEARER_TOKEN` | *(unset)* | Required shared bearer token for `POST /infer`; the endpoint returns 503 until configured | @@ -202,10 +211,16 @@ All configuration is via environment variables. Copy `.env.example` to `.env` a The host-facing ingestion endpoint is an NGINX gateway at port `8080`; Go ingestion replicas are internal-only and can be started with `docker compose up --build --scale ingestion-service=3`. `/health` is liveness and `/ready` includes the Postgres dependency; the CI smoke test checks gateway readiness, multi-replica routing, and continued readiness after one replica stops. The optional `EXPOSE_INSTANCE_ID=true` setting exists only for that test and is disabled by default. +When `FIREHOSE_ENABLED=true`, a bounded worker mirrors accepted inference events to Amazon Data Firehose after the primary warehouse path succeeds. Firehose queue pressure or AWS delivery errors do not change `/ready` and do not fail an otherwise accepted primary ingestion request; they are surfaced through Prometheus metrics and logs. + ## 🏗️ Architecture -``` +```text User → NGINX Ingestion Gateway (8080) → Go Ingestion Replicas → Postgres (local) / Snowflake (optional) + │ + ├──► bounded Firehose queue ──► Amazon Data Firehose ──► Amazon S3 + │ │ + │ └──► CloudWatch delivery logs ↓ Drift Engine C++ (7070) ↓ @@ -220,13 +235,66 @@ User → NGINX Ingestion Gateway (8080) → Go Ingestion Replicas → Postgres ( | Service | Language | Port | Description | |---------|----------|------|-------------| -| `ingestion-service` | Go | 8080 | Receives inference logs, writes to warehouse | +| `ingestion-service` | Go | 8080 | Receives inference logs, writes to warehouse, optionally mirrors to Firehose | | `drift-engine` | C++ + Python | 7070 | PSI/KS drift detection | | `llm-guard` | Python | 8000 | LLM-powered incident summarization | | `streamlit-dashboard` | Python | 8501 | Control plane UI | | `postgres` | — | 5432 | Local warehouse (default) | | `prometheus` | — | 9090 | Metrics scraping | | `grafana` | — | 3000 | Dashboards | +| Amazon Data Firehose | AWS managed | — | Optional telemetry buffering and delivery | +| Amazon S3 | AWS managed | — | Optional compressed telemetry lake | + +--- + +## ☁️ AWS telemetry mirror — Amazon Data Firehose + S3 + +The `terraform/` configuration defines an AWS telemetry path that can be provisioned independently of local Docker Compose: + +```mermaid +flowchart LR + App[Model / application] --> Gateway[NGINX] + Gateway --> Go[Go ingestion replicas] + Go --> Warehouse[(Postgres / Snowflake path)] + Go -. bounded fail-open mirror .-> Queue[In-memory queue] + Queue --> Firehose[Amazon Data Firehose] + Firehose --> S3[(Private S3 telemetry bucket)] + Firehose --> CW[CloudWatch delivery logs] +``` + +Terraform defines a private S3 bucket with public-access blocking, versioning, SSE-S3 encryption and a configurable retention policy; a Firehose stream with 60-second / 5-MiB buffering, GZIP compression and time-partitioned S3 prefixes; CloudWatch delivery logging; a Firehose service role; and a separate least-privilege writer policy for the SentinelAI workload. + +```bash +cd terraform +terraform init +terraform fmt -check +terraform validate +terraform plan +``` + +Do not commit AWS access keys. The Go producer uses the AWS SDK default credential chain; for an EKS deployment, attach the Terraform `firehose_writer_policy_arn` output to the ingestion workload identity rather than injecting long-lived credentials. + +Producer-side observability is exposed through: + +```text +ingestion_firehose_records_total{status="queued"} +ingestion_firehose_records_total{status="delivered"} +ingestion_firehose_records_total{status="error"} +ingestion_firehose_records_total{status="dropped"} +``` + +### AWS validation boundary + +The repository CI validates the AWS integration code without requiring AWS credentials: + +- `go mod tidy` must produce no module-file drift. +- `go test ./...` validates Firehose configuration and NDJSON `PutRecord` behavior against a fake client. +- `terraform fmt`, `terraform init -backend=false`, and `terraform validate` verify the IaC syntax and provider graph. +- The normal Docker Compose CI continues to exercise the local multi-replica ingestion path with Firehose disabled by default. + +CI does **not** run `terraform apply`, create billable AWS resources, or prove live Firehose-to-S3 delivery. Live AWS throughput, durability, retry behavior, IAM attachment, and end-to-end delivery latency remain deployment-level validation work. + +This path is implemented as a best-effort telemetry mirror. The queue is bounded and in-memory, process termination can lose queued mirror records, and SDK retries may create duplicates. See [docs/aws-firehose-s3.md](docs/aws-firehose-s3.md) for deployment, IAM, failure semantics, observability, and cost controls. --- @@ -257,138 +325,36 @@ python benchmarks/run_benchmark.py --output benchmarks/latest.json CI reruns the benchmark on every pull request, validates its schema and F1 regression floor, and uploads raw evidence for 30 days. For comparable hosts, median or P95 increases above 15% require investigation and a documented baseline update. -The perfect synthetic classification result is a regression signal for deliberately separated perturbation classes; it is **not** a production accuracy claim. Native C++, service concurrency, network, warehouse, GPU, and real-world labeled drift benchmarks remain future evaluation layers. +The perfect synthetic classification result is a regression signal for deliberately separated perturbation classes; it is **not** a production accuracy claim. Native C++, service concurrency, network, warehouse, GPU, Firehose/S3 throughput, and real-world labeled drift benchmarks remain future evaluation layers. ### Test and evidence status -| Evidence | Current state | Source | -|---|---:|---| -| Benchmark raw data | Versioned JSON | `benchmarks/latest.json` | -| Benchmark methodology | Versioned report | `benchmarks/benchmark_report.md` | -| Benchmark CI | Required execution + artifact | `.github/workflows/benchmarks.yml` | -| Drift thresholds | PSI 0.20 · KS 0.10 | `g++ -std=c++17 drift-engine/drift_engine.cpp -o drift-engine/drift_engine`; source: `drift-engine/drift_engine.cpp` | +| Evidence | Current repository contract | Source | +|---|---|---| +| Drift benchmark raw data | Versioned JSON | `benchmarks/latest.json` | +| Drift benchmark methodology | Versioned report | `benchmarks/benchmark_report.md` | +| Firehose producer validation | Unit-tested configuration + serialized `PutRecord` payloads | `ingestion-service/firehose_test.go` | +| AWS IaC validation | Formatting, provider initialization without backend, Terraform validation | `.github/workflows/aws-telemetry.yml` | +| Local ingestion resilience | Multi-replica NGINX routing and one-replica-loss readiness smoke test | `.github/workflows/ci-cd.yml` | -### Observability Metrics +### Observability metrics -| Metric Name | Type | Emitted By | Purpose | +| Metric | Type | Emitted by | Purpose | |---|---|---|---| -| `sentinel_requests_total` | Counter | `monitoring/metrics.py` | API request volume | -| `sentinel_request_latency_seconds` | Histogram | `monitoring/metrics.py` | API request latency | -| `inference_requests_total` | Counter | `monitoring/prometheus.py` | Inference request volume | -| `ingestion_logs_total{status}` | Counter | `ingestion-service/main.go` | Ingestion outcome counts | -| `ingestion_handler_seconds` | Histogram | `ingestion-service/main.go` | Go ingestion handler latency | +| `ingestion_logs_total{status}` | Counter | `ingestion-service/main.go` | Primary ingestion outcomes | +| `ingestion_handler_seconds` | Histogram | `ingestion-service/main.go` | Ingestion handler latency | +| `ingestion_firehose_records_total{status}` | Counter | `ingestion-service/firehose.go` | Firehose queued, delivered, error, and dropped records | | `drift_detected_total` | Counter | `drift-engine/server.py` | Drift event count | | `drift_compute_seconds` | Histogram | `drift-engine/server.py` | Drift calculation latency | | `llm_guard_summaries_total{method}` | Counter | `llm-guard/app.py` | Ollama vs fallback summary count | | `llm_guard_summary_seconds` | Histogram | `llm-guard/app.py` | Summary generation latency | -| `requests_total` | Counter | `backend/app/main.py` | Backend request volume | - - - -## Design Targets (Not Measured) - -The following historical values are retained for planning and comparison, but no committed generator or CI artifact establishes them as current measurements at this commit. They must not be treated as benchmark results or release evidence. - -### Historical validation claims - -| Evidence | Current state | Evidence status | -|---|---:|---| -| Focused API tests | 4 passed | No committed command/output establishes this historical audit value | -| Focused API coverage | 24% | No committed command/output establishes this historical audit value; the former static badge was removed | - -### Historical project inventory - -| Area | Metric | Current Value | Source | -|---|---:|---:|---| -| Codebase | Tracked files | 98 | `git ls-files` | -| Codebase | Python files | 32 | `*.py` files | -| Codebase | Go files | 1 | `ingestion-service/main.go` | -| Codebase | C++ files | 4 | Drift and ingestion engine sources | -| Codebase | TypeScript files | 7 | `frontend/` | -| Codebase | Source NCLOC | 1,201 | Non-empty, non-comment Python/Go/C++/TS lines | -| Tests | Python test files | 6 | `tests/` | -| Tests | Test declarations | 5 | `def test_*` scan | -| Tests | Focused API validation | 4 passed | `pytest tests/test_*.py` focused API scope | -| Tests | Focused `api` coverage | 24% | Local coverage run | -| CI/CD | GitHub Actions workflows | 7 | `.github/workflows/*.yml` | -| Dependencies | Python runtime dependencies | 11 | `requirements.txt` | -| Delivery | Dockerfiles | 5 | Root/services/dashboard Docker assets | -| Delivery | Kubernetes manifests | 9 | `k8s/*.yaml` | -| Delivery | Helm chart files | 1 | `helm/sentinel/templates/deployment.yaml` | -| Infrastructure | Terraform files | 1 | `terraform/main,TF` | -| Monitoring | Monitoring config files | 5 | `monitoring/` | -| Services | Docker Compose service URLs | 6 | Dashboard, Prometheus, Grafana, ingestion, drift, LLM guard | -| Validation limits | Native Go/C++ compile checks | Not run locally | Go/g++/MSVC unavailable in workspace | --- -## 🧠 Extended Q&A - -### Why use C++ for drift detection? -To achieve sub-millisecond statistical scoring at scale. - -### Why Go for ingestion? -Go provides efficient concurrency and low-latency HTTP handling. - -### Why Postgres locally (not Snowflake)? -Postgres is free, runs in Docker, and supports the same SQL schema. Switch to `WAREHOUSE_MODE=snowflake` when you're ready to push to production. - -### Why MLflow? -Experiment tracking, reproducibility, and version control. - -### Why LangChain + Ollama? -LLM-powered root cause summarization and RAG over historical incidents. - -### Why Kubernetes? -Horizontal scaling and production-grade orchestration. - -### Why Terraform? -Reproducible infrastructure as code. - ---- - -## 🏢 Enterprise Value - -SentinelAI demonstrates: - -- AI system lifecycle management -- Drift monitoring -- MLOps integration -- Distributed systems engineering -- Cloud-native architecture -- LLM augmentation -- Observability & metrics-driven design - ---- - -## 🔧 Recent Code Improvements - -See [CHANGELOG.md](CHANGELOG.md) for the dated fix history. - -### Running tests locally - -```bash -pip install -r requirements.txt -pytest tests/ -v -``` - -### Environment variables added - -| Variable | Default | Description | -|----------|---------|-------------| -| `API_USERNAME` | `admin` | Login username for the API auth endpoint | -| `API_PASSWORD` | *(unset — auth disabled until set)* | Login password; must be set to enable auth | -| `LLM_MODEL_NAME` | `meta-llama/Meta-Llama-3-8B` | HuggingFace model used by the inference route | - ---- - - - -- Add automated retraining pipeline -- Add Shadow Model Deployment -- Add Cost Optimization Engine -- Add Hallucination Classifier Model - - - +## Engineering roadmap +- Add an ephemeral live-AWS integration test that proves Firehose delivery into a disposable S3 prefix without turning normal PR CI into a billable deployment. +- Add `PutRecordBatch` batching and measured queue/backpressure benchmarks before claiming higher producer throughput. +- Add a durable local spool or explicit replay mechanism for mirror records that cannot be lost on process termination. +- Attach the writer policy through an EKS workload identity path and validate least-privilege IAM end to end. +- Add Athena/Glue-compatible telemetry schemas only after a concrete query workload and reproducible evidence exist. diff --git a/docker-compose.yml b/docker-compose.yml index 4aa6554..f02488a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,10 @@ services: WAREHOUSE_MODE: ${WAREHOUSE_MODE:-postgres} DATABASE_URL: ${DATABASE_URL:-postgres://sentinel:sentinel@postgres:5432/sentinel?sslmode=disable} PORT: "8080" + FIREHOSE_ENABLED: ${FIREHOSE_ENABLED:-false} + FIREHOSE_DELIVERY_STREAM: ${FIREHOSE_DELIVERY_STREAM:-sentinelai-telemetry} + FIREHOSE_QUEUE_SIZE: ${FIREHOSE_QUEUE_SIZE:-1000} + AWS_REGION: ${AWS_REGION:-us-east-1} # Disabled by default; CI enables this to verify multiple replicas route through NGINX. EXPOSE_INSTANCE_ID: ${EXPOSE_INSTANCE_ID:-false} expose: diff --git a/docs/aws-firehose-s3.md b/docs/aws-firehose-s3.md new file mode 100644 index 0000000..903b593 --- /dev/null +++ b/docs/aws-firehose-s3.md @@ -0,0 +1,97 @@ +# Amazon Data Firehose + S3 telemetry path + +SentinelAI can optionally mirror accepted inference telemetry from the Go ingestion service to Amazon Data Firehose. Firehose buffers the records and delivers GZIP-compressed newline-delimited JSON (NDJSON) objects to a private S3 telemetry bucket. + +This AWS path is **optional**. Local Docker Compose keeps `FIREHOSE_ENABLED=false`, so PostgreSQL remains the default local persistence path and no AWS account is required for development. + +## Architecture + +```mermaid +flowchart LR + Client[Model / application] --> Gateway[NGINX ingestion gateway] + Gateway --> Go[Go ingestion replicas] + Go --> DB[(PostgreSQL / Snowflake path)] + Go -. bounded fail-open mirror .-> Queue[In-memory Firehose queue] + Queue --> Firehose[Amazon Data Firehose] + Firehose --> S3[(Amazon S3 telemetry lake)] + Firehose --> CW[CloudWatch delivery logs] +``` + +The Firehose mirror intentionally does not participate in `/ready`. A temporary AWS failure increments `ingestion_firehose_records_total{status="error"}` and is logged, while the primary ingestion response continues to reflect the primary warehouse write. If the bounded queue fills, records are dropped from the mirror and counted with `status="dropped"` rather than allowing telemetry backpressure to take down ingestion. + +## Provision the AWS resources + +Prerequisites: + +- Terraform 1.6+ +- AWS credentials available through the standard AWS credential chain +- Permission to create S3, Firehose, CloudWatch Logs, IAM policy/role, and ECR resources + +```bash +cd terraform +terraform init +terraform fmt -check +terraform validate +terraform plan +terraform apply +``` + +The default configuration creates: + +- a private, versioned S3 bucket with SSE-S3 encryption; +- a 30-day telemetry lifecycle policy; +- an Amazon Data Firehose delivery stream named `sentinelai-telemetry`; +- 60-second / 5-MiB buffering with GZIP compression; +- time-partitioned S3 keys under `inference/year=.../month=.../day=.../hour=.../`; +- a CloudWatch log group for Firehose delivery errors; +- a Firehose service role with only the S3 and CloudWatch permissions it needs; +- a separate `sentinelai-firehose-writer` IAM policy granting `PutRecord` and `PutRecordBatch` to the SentinelAI workload; +- the existing SentinelAI ECR repository, now defined in a valid Terraform `.tf` file. + +Use `terraform output` after apply to retrieve the generated S3 bucket name, stream ARN, stream name, and writer-policy ARN. + +## Give the ingestion workload permission + +Do not put long-lived AWS access keys in the repository. Attach the Terraform output `firehose_writer_policy_arn` to the workload identity used by SentinelAI. On EKS, the intended production pattern is an IAM role associated with the ingestion service account (IRSA / EKS workload identity). + +For local development against a real AWS account, the AWS SDK for Go v2 uses its normal credential provider chain. Keep credentials outside the repo. + +## Enable the mirror + +```bash +export FIREHOSE_ENABLED=true +export FIREHOSE_DELIVERY_STREAM=sentinelai-telemetry +export FIREHOSE_QUEUE_SIZE=1000 +export AWS_REGION=us-east-1 +``` + +Then start SentinelAI and send a normal inference log: + +```bash +curl -X POST http://localhost:8080/log \ + -H "Content-Type: application/json" \ + -d '{"model_id":"demo","model_version":"v1","latency_ms":120,"tokens_in":32,"tokens_out":64,"status":"ok"}' +``` + +The producer appends a newline to every JSON record before `PutRecord`. That keeps individual events parseable after Firehose concatenates buffered records into S3 objects. + +## Observability + +The ingestion service exports: + +```text +ingestion_firehose_records_total{status="queued"} +ingestion_firehose_records_total{status="delivered"} +ingestion_firehose_records_total{status="error"} +ingestion_firehose_records_total{status="dropped"} +``` + +CloudWatch delivery logs cover the managed Firehose-to-S3 leg. Application metrics cover the producer-side queue and `PutRecord` result. + +## Failure semantics + +This feature is a telemetry mirror, not a transactional dual-write guarantee. The in-memory queue is intentionally bounded and is not persisted across process termination. AWS SDK retries may also produce duplicate Firehose records in some failure scenarios. Consumers should therefore treat the S3 telemetry dataset as at-least-once/best-effort observability data and use stable event identifiers if strict de-duplication is later required. + +## Cost control + +Firehose, S3, CloudWatch Logs, and related data transfer can incur AWS charges. The default 30-day S3 expiration and 14-day CloudWatch log retention are intended to keep a portfolio/dev deployment bounded. Review the Terraform plan and AWS pricing before leaving the stack running. diff --git a/ingestion-service/Dockerfile b/ingestion-service/Dockerfile index 5369d1b..6ed4f55 100644 --- a/ingestion-service/Dockerfile +++ b/ingestion-service/Dockerfile @@ -1,5 +1,5 @@ # ingestion-service — multi-stage Go build -FROM golang:1.21-alpine AS builder +FROM golang:1.24-alpine AS builder WORKDIR /app COPY go.mod go.sum* ./ RUN go mod download diff --git a/ingestion-service/firehose.go b/ingestion-service/firehose.go new file mode 100644 index 0000000..c28b899 --- /dev/null +++ b/ingestion-service/firehose.go @@ -0,0 +1,144 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "os" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/firehose" + "github.com/aws/aws-sdk-go-v2/service/firehose/types" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + defaultFirehoseQueueSize = 1000 + firehosePublishTimeout = 5 * time.Second +) + +var firehoseRecordsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ingestion_firehose_records_total", + Help: "Inference telemetry records handled by the optional Amazon Data Firehose mirror.", + }, + []string{"status"}, +) + +func init() { + prometheus.MustRegister(firehoseRecordsTotal) +} + +type firehoseAPI interface { + PutRecord(context.Context, *firehose.PutRecordInput, ...func(*firehose.Options)) (*firehose.PutRecordOutput, error) +} + +type firehosePublisher struct { + client firehoseAPI + streamName string +} + +type firehoseDispatcher struct { + publisher *firehosePublisher + queue chan InferenceLog +} + +func newFirehoseDispatcher() (*firehoseDispatcher, error) { + if !strings.EqualFold(strings.TrimSpace(os.Getenv("FIREHOSE_ENABLED")), "true") { + return nil, nil + } + + streamName := strings.TrimSpace(os.Getenv("FIREHOSE_DELIVERY_STREAM")) + if streamName == "" { + return nil, errors.New("FIREHOSE_DELIVERY_STREAM is required when FIREHOSE_ENABLED=true") + } + + region := strings.TrimSpace(os.Getenv("AWS_REGION")) + if region == "" { + region = "us-east-1" + } + + queueSize := defaultFirehoseQueueSize + if raw := strings.TrimSpace(os.Getenv("FIREHOSE_QUEUE_SIZE")); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil || parsed < 1 { + return nil, fmt.Errorf("FIREHOSE_QUEUE_SIZE must be a positive integer: %q", raw) + } + queueSize = parsed + } + + cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(region)) + if err != nil { + return nil, fmt.Errorf("load AWS configuration: %w", err) + } + + dispatcher := &firehoseDispatcher{ + publisher: &firehosePublisher{ + client: firehose.NewFromConfig(cfg), + streamName: streamName, + }, + queue: make(chan InferenceLog, queueSize), + } + go dispatcher.run() + + return dispatcher, nil +} + +func (d *firehoseDispatcher) enqueue(entry InferenceLog) { + select { + case d.queue <- entry: + firehoseRecordsTotal.WithLabelValues("queued").Inc() + default: + firehoseRecordsTotal.WithLabelValues("dropped").Inc() + log.Printf("firehose mirror queue full; dropping telemetry record for model=%s", entry.ModelID) + } +} + +func (d *firehoseDispatcher) run() { + for entry := range d.queue { + ctx, cancel := context.WithTimeout(context.Background(), firehosePublishTimeout) + err := d.publisher.publish(ctx, entry) + cancel() + + if err != nil { + firehoseRecordsTotal.WithLabelValues("error").Inc() + log.Printf("firehose PutRecord failed for model=%s: %v", entry.ModelID, err) + continue + } + firehoseRecordsTotal.WithLabelValues("delivered").Inc() + } +} + +func (p *firehosePublisher) publish(ctx context.Context, entry InferenceLog) error { + payload, err := encodeFirehoseRecord(entry) + if err != nil { + return err + } + + _, err = p.client.PutRecord(ctx, &firehose.PutRecordInput{ + DeliveryStreamName: &p.streamName, + Record: &types.Record{ + Data: payload, + }, + }) + if err != nil { + return fmt.Errorf("put record: %w", err) + } + return nil +} + +func encodeFirehoseRecord(entry InferenceLog) ([]byte, error) { + payload, err := json.Marshal(entry) + if err != nil { + return nil, fmt.Errorf("marshal inference log: %w", err) + } + + // Firehose concatenates records inside delivered objects. NDJSON keeps each + // inference event independently parseable after buffering and compression. + return append(payload, '\n'), nil +} diff --git a/ingestion-service/firehose_test.go b/ingestion-service/firehose_test.go new file mode 100644 index 0000000..eb14b3c --- /dev/null +++ b/ingestion-service/firehose_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/firehose" +) + +type stubFirehoseClient struct { + input *firehose.PutRecordInput +} + +func (s *stubFirehoseClient) PutRecord(_ context.Context, input *firehose.PutRecordInput, _ ...func(*firehose.Options)) (*firehose.PutRecordOutput, error) { + s.input = input + recordID := "test-record" + return &firehose.PutRecordOutput{RecordId: &recordID}, nil +} + +func TestNewFirehoseDispatcherDisabledByDefault(t *testing.T) { + t.Setenv("FIREHOSE_ENABLED", "false") + + dispatcher, err := newFirehoseDispatcher() + if err != nil { + t.Fatalf("newFirehoseDispatcher returned error: %v", err) + } + if dispatcher != nil { + t.Fatal("dispatcher should be nil when Firehose is disabled") + } +} + +func TestNewFirehoseDispatcherRequiresStream(t *testing.T) { + t.Setenv("FIREHOSE_ENABLED", "true") + t.Setenv("FIREHOSE_DELIVERY_STREAM", "") + + if _, err := newFirehoseDispatcher(); err == nil { + t.Fatal("expected missing stream configuration to return an error") + } +} + +func TestFirehosePublisherUsesNDJSON(t *testing.T) { + client := &stubFirehoseClient{} + publisher := &firehosePublisher{client: client, streamName: "sentinelai-telemetry"} + entry := InferenceLog{ + ModelID: "demo", + ModelVersion: "v1", + LatencyMs: 42, + Status: "ok", + Timestamp: time.Date(2026, 8, 30, 18, 0, 0, 0, time.UTC), + } + + if err := publisher.publish(context.Background(), entry); err != nil { + t.Fatalf("publish returned error: %v", err) + } + if client.input == nil || client.input.Record == nil { + t.Fatal("PutRecord input was not captured") + } + if got := *client.input.DeliveryStreamName; got != "sentinelai-telemetry" { + t.Fatalf("stream name = %q, want sentinelai-telemetry", got) + } + + payload := client.input.Record.Data + if len(payload) == 0 || payload[len(payload)-1] != '\n' { + t.Fatalf("payload must be newline-delimited JSON: %q", payload) + } + + var decoded InferenceLog + if err := json.Unmarshal(payload[:len(payload)-1], &decoded); err != nil { + t.Fatalf("payload is not valid JSON: %v", err) + } + if decoded.ModelID != entry.ModelID || decoded.LatencyMs != entry.LatencyMs { + t.Fatalf("decoded payload = %#v, want model=%q latency=%d", decoded, entry.ModelID, entry.LatencyMs) + } +} diff --git a/ingestion-service/go.mod b/ingestion-service/go.mod index 511e7f6..3620a04 100644 --- a/ingestion-service/go.mod +++ b/ingestion-service/go.mod @@ -1,13 +1,28 @@ module sentinel/ingestion-service -go 1.21 +go 1.24 require ( + github.com/aws/aws-sdk-go-v2/config v1.32.36 + github.com/aws/aws-sdk-go-v2/service/firehose v1.48.1 github.com/lib/pq v1.10.9 github.com/prometheus/client_golang v1.19.1 ) require ( + github.com/aws/aws-sdk-go-v2 v1.45.1 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.35 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.37 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.36 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.5.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.33.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.45.5 // indirect + github.com/aws/smithy-go v1.28.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect diff --git a/ingestion-service/go.sum b/ingestion-service/go.sum index 864f120..763d8b5 100644 --- a/ingestion-service/go.sum +++ b/ingestion-service/go.sum @@ -1,3 +1,33 @@ +github.com/aws/aws-sdk-go-v2 v1.45.1 h1:iIoG3NaLhV6UZpPXyPXlDj2I9oS8tV/nMcMnITCC6Ks= +github.com/aws/aws-sdk-go-v2 v1.45.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2/config v1.32.36 h1:mX6ietU7UlB4w/2IUaexJdsyUDvhTd+jYPjVePiyi6s= +github.com/aws/aws-sdk-go-v2/config v1.32.36/go.mod h1:rMpV4xk7ZK59edraSaHP0jsWrztWTT5tbCwWY495hug= +github.com/aws/aws-sdk-go-v2/credentials v1.19.35 h1:Cxua2RVdRwL0sfjHM/SnQoOnQ7xKng9m5EQBO8BnZlg= +github.com/aws/aws-sdk-go-v2/credentials v1.19.35/go.mod h1:9XQ+RSIGPkycr+oCJYnB1uTv5kMVVR+rd2vYK0Hxj2w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.36 h1:gucL1KH/PAYbpTpBg09CiVpBdTu4qkCl8C7xOTBixUg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.36/go.mod h1:usTB+PHhNMhrx2dxUeHcM7OrT5pySvmjYI++IsefPN0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 h1:pc138gM1CW+XPc60rEwUlwwuwWFQK16CI1T7v1F9Oec= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1/go.mod h1:1+koxpPIbfBdfzP6vojm5/zTpTQ/micYwlxIiNB3TxI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1 h1:K0JsbZQj+1h208Ro1zHeA4l7bMp0NvRffHQ91q8Ol1s= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1/go.mod h1:W3/vL6EtCIatICGy9ab29QhMuae+cOKPWcMxv02CO+Q= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.37 h1:oyd3ke4V9AhKcRR7rRgxk1VyI+DjK2CBQtbxh3OkdaA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.37/go.mod h1:aA9D7SqfG9IC1b7FLD7Iyc8Q4JN0a8gHhNjN4zPlIaI= +github.com/aws/aws-sdk-go-v2/service/firehose v1.48.1 h1:KtCWzKQiEQJdbdZHo0ncT0FC1uM34/yv5SLegbVNKBk= +github.com/aws/aws-sdk-go-v2/service/firehose v1.48.1/go.mod h1:Ze6lqKG4a9IO6qKODPBba8QlJyZdp9Hahf942qtmez8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.16 h1:iE4NGbvqUZnHDqddQAauZzCILYtFjOHwRM5MOOKLB5A= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.16/go.mod h1:VsjEgrP+ibcou8TlWA4tYaB+0OojuhirsmCe+U60hTA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.36 h1:fx2ujmozWn+C/GtfXfz5k6Ckzza40ElOpIW7d92fLWQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.36/go.mod h1:QT2ufGVJ+xTRxtXPHTQ1kHkAdWIKPCmD+BqYAXWv8/4= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.5 h1:0VTFBfOgPJrUSpGMgzoi8qLcXF5dbmiBuxpo14eBWUw= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.5/go.mod h1:sNZYlBxoohYMBYl47BO/bFtAM6I8HSsPa1qwwPPRGoQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.5 h1:jDQARFp1mJ2PEnllQf01nfFXGfWMJ59e0/HCHUTTZCk= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.5/go.mod h1:OcT2AhgTuxGAwZk5hgxaNLGpS33W8s8dUQadGVDVY9I= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.5 h1:8xo1q9ttkYqMJ6vOXX67FPSpVEI7BWKVTKh77g82w+8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.5/go.mod h1:hbBeEUrZg6VddXYZpbKPyF0tl4XEnM+Dbx92RW3vmZI= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.5 h1:eQ5BtXDrPg2wK0AjtVPzeBhUpYPeqHE/ptiH7xJRGek= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.5/go.mod h1:f9ImhnOISY7BuTZLM8qHepCYnglHBVLk5wVzatmP++w= +github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= +github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= diff --git a/ingestion-service/main.go b/ingestion-service/main.go index 2e301d8..058ba60 100644 --- a/ingestion-service/main.go +++ b/ingestion-service/main.go @@ -1,9 +1,13 @@ // ingestion-service/main.go — SentinelAI inference-log ingestion endpoint. // // Environment variables: -// DATABASE_URL — Postgres DSN (required when WAREHOUSE_MODE=postgres) -// WAREHOUSE_MODE — "postgres" (default) | "snowflake" -// PORT — listen port (default 8080) +// DATABASE_URL — Postgres DSN (required when WAREHOUSE_MODE=postgres) +// WAREHOUSE_MODE — "postgres" (default) | "snowflake" +// FIREHOSE_ENABLED — mirror accepted telemetry to Amazon Data Firehose when true +// FIREHOSE_DELIVERY_STREAM — Firehose stream name (required when enabled) +// FIREHOSE_QUEUE_SIZE — bounded in-memory mirror queue size (default 1000) +// AWS_REGION — AWS region for Firehose (default us-east-1) +// PORT — listen port (default 8080) package main import ( @@ -62,6 +66,7 @@ type InferenceLog struct { // --------------------------------------------------------------------------- var db *sql.DB +var firehoseMirror *firehoseDispatcher // --------------------------------------------------------------------------- // Handlers @@ -75,6 +80,8 @@ func healthHandler(w http.ResponseWriter, _ *http.Request) { } // readyHandler reports whether this replica can accept durable ingestion writes. +// Firehose is an optional fail-open telemetry mirror, so its health is exposed by +// metrics/logs rather than making the primary ingestion readiness depend on AWS. func readyHandler(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") if db != nil { @@ -133,7 +140,7 @@ func logHandler(w http.ResponseWriter, r *http.Request) { entry.Status = "ok" } if entry.Timestamp.IsZero() { - entry.Timestamp = time.Now() + entry.Timestamp = time.Now().UTC() } warehouseMode := os.Getenv("WAREHOUSE_MODE") @@ -158,11 +165,18 @@ func logHandler(w http.ResponseWriter, r *http.Request) { return } } else { - // Snowflake or no DB — just log for now + // Snowflake or no DB — just log for now. log.Printf("[%s] model=%s latency=%dms status=%s", warehouseMode, entry.ModelID, entry.LatencyMs, entry.Status) } + // The AWS path is intentionally a non-blocking mirror. Queue pressure or a + // transient Firehose failure is visible in metrics and logs but does not turn + // telemetry delivery into a failure of the primary ingestion request. + if firehoseMirror != nil { + firehoseMirror.enqueue(entry) + } + ingestTotal.WithLabelValues("ok").Inc() w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) @@ -201,6 +215,15 @@ func main() { log.Println("connected to postgres") } + var err error + firehoseMirror, err = newFirehoseDispatcher() + if err != nil { + log.Fatalf("could not configure Firehose mirror: %v", err) + } + if firehoseMirror != nil { + log.Printf("Amazon Data Firehose mirror enabled (stream=%s)", firehoseMirror.publisher.streamName) + } + port := os.Getenv("PORT") if port == "" { port = "8080" diff --git a/terraform/main,TF b/terraform/main,TF deleted file mode 100644 index dbfc1d7..0000000 --- a/terraform/main,TF +++ /dev/null @@ -1,11 +0,0 @@ -provider "aws" { - region = "us-east-1" -} - -resource "aws_s3_bucket" "sentinel_logs" { - bucket = "sentinelai-log-storage" -} - -resource "aws_ecr_repository" "sentinel_repo" { - name = "sentinelai-repo" -} diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..a18dd76 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,209 @@ +terraform { + required_version = ">= 1.6.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } +} + +provider "aws" { + region = var.aws_region +} + +data "aws_caller_identity" "current" {} + +locals { + telemetry_bucket_name = var.telemetry_bucket_name != "" ? var.telemetry_bucket_name : "${var.project_name}-telemetry-${data.aws_caller_identity.current.account_id}-${var.aws_region}" + common_tags = { + Project = var.project_name + ManagedBy = "Terraform" + Component = "telemetry" + } +} + +resource "aws_s3_bucket" "sentinel_logs" { + bucket = local.telemetry_bucket_name + force_destroy = var.telemetry_bucket_force_destroy + tags = local.common_tags +} + +resource "aws_s3_bucket_public_access_block" "sentinel_logs" { + bucket = aws_s3_bucket.sentinel_logs.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_ownership_controls" "sentinel_logs" { + bucket = aws_s3_bucket.sentinel_logs.id + + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_versioning" "sentinel_logs" { + bucket = aws_s3_bucket.sentinel_logs.id + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "sentinel_logs" { + bucket = aws_s3_bucket.sentinel_logs.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_lifecycle_configuration" "sentinel_logs" { + bucket = aws_s3_bucket.sentinel_logs.id + + rule { + id = "expire-telemetry" + status = "Enabled" + + filter {} + + expiration { + days = var.telemetry_retention_days + } + + noncurrent_version_expiration { + noncurrent_days = var.telemetry_retention_days + } + } +} + +resource "aws_ecr_repository" "sentinel_repo" { + name = "${var.project_name}-repo" + image_tag_mutability = "IMMUTABLE" + + image_scanning_configuration { + scan_on_push = true + } + + tags = local.common_tags +} + +resource "aws_cloudwatch_log_group" "firehose" { + name = "/aws/firehose/${var.project_name}-telemetry" + retention_in_days = 14 + tags = local.common_tags +} + +resource "aws_cloudwatch_log_stream" "firehose" { + name = "S3Delivery" + log_group_name = aws_cloudwatch_log_group.firehose.name +} + +data "aws_iam_policy_document" "firehose_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["firehose.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "firehose" { + name = "${var.project_name}-firehose-delivery" + assume_role_policy = data.aws_iam_policy_document.firehose_assume_role.json + tags = local.common_tags +} + +data "aws_iam_policy_document" "firehose_delivery" { + statement { + sid = "S3BucketMetadata" + effect = "Allow" + actions = [ + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:ListBucketMultipartUploads" + ] + resources = [aws_s3_bucket.sentinel_logs.arn] + } + + statement { + sid = "S3ObjectDelivery" + effect = "Allow" + actions = [ + "s3:AbortMultipartUpload", + "s3:GetObject", + "s3:PutObject" + ] + resources = ["${aws_s3_bucket.sentinel_logs.arn}/*"] + } + + statement { + sid = "CloudWatchDeliveryLogs" + effect = "Allow" + actions = ["logs:PutLogEvents"] + resources = [ + "${aws_cloudwatch_log_group.firehose.arn}:*" + ] + } +} + +resource "aws_iam_role_policy" "firehose_delivery" { + name = "${var.project_name}-firehose-delivery" + role = aws_iam_role.firehose.id + policy = data.aws_iam_policy_document.firehose_delivery.json +} + +resource "aws_kinesis_firehose_delivery_stream" "sentinel_telemetry" { + name = "${var.project_name}-telemetry" + destination = "extended_s3" + tags = local.common_tags + + extended_s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.sentinel_logs.arn + buffering_interval = 60 + buffering_size = 5 + compression_format = "GZIP" + + prefix = "inference/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/" + error_output_prefix = "errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/" + + cloudwatch_logging_options { + enabled = true + log_group_name = aws_cloudwatch_log_group.firehose.name + log_stream_name = aws_cloudwatch_log_stream.firehose.name + } + } + + depends_on = [aws_iam_role_policy.firehose_delivery] +} + +data "aws_iam_policy_document" "firehose_writer" { + statement { + sid = "PublishSentinelTelemetry" + effect = "Allow" + actions = [ + "firehose:PutRecord", + "firehose:PutRecordBatch" + ] + resources = [aws_kinesis_firehose_delivery_stream.sentinel_telemetry.arn] + } +} + +resource "aws_iam_policy" "firehose_writer" { + name = "${var.project_name}-firehose-writer" + description = "Allows SentinelAI ingestion workloads to publish telemetry to Amazon Data Firehose." + policy = data.aws_iam_policy_document.firehose_writer.json + tags = local.common_tags +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000..b3cfa46 --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,24 @@ +output "firehose_delivery_stream_name" { + description = "Amazon Data Firehose stream used by the SentinelAI ingestion mirror." + value = aws_kinesis_firehose_delivery_stream.sentinel_telemetry.name +} + +output "firehose_delivery_stream_arn" { + description = "ARN of the SentinelAI Amazon Data Firehose stream." + value = aws_kinesis_firehose_delivery_stream.sentinel_telemetry.arn +} + +output "firehose_writer_policy_arn" { + description = "IAM policy ARN to attach to the SentinelAI workload role (for example an EKS IRSA role)." + value = aws_iam_policy.firehose_writer.arn +} + +output "telemetry_bucket_name" { + description = "S3 bucket receiving compressed SentinelAI telemetry objects." + value = aws_s3_bucket.sentinel_logs.bucket +} + +output "telemetry_bucket_arn" { + description = "ARN of the S3 telemetry bucket." + value = aws_s3_bucket.sentinel_logs.arn +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..ca61341 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,34 @@ +variable "project_name" { + description = "Prefix used for SentinelAI AWS resources." + type = string + default = "sentinelai" +} + +variable "aws_region" { + description = "AWS region for SentinelAI telemetry infrastructure." + type = string + default = "us-east-1" +} + +variable "telemetry_bucket_name" { + description = "Optional globally unique S3 bucket name. Leave empty to derive one from the AWS account and region." + type = string + default = "" +} + +variable "telemetry_retention_days" { + description = "Number of days to retain delivered telemetry objects in S3." + type = number + default = 30 + + validation { + condition = var.telemetry_retention_days >= 1 + error_message = "telemetry_retention_days must be at least 1." + } +} + +variable "telemetry_bucket_force_destroy" { + description = "Whether Terraform may delete the S3 bucket while it still contains telemetry objects." + type = bool + default = false +}