From b0ad0e3b62d1b513df9c1abf73856b982a3890a8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 13 Nov 2025 00:17:34 +0000 Subject: [PATCH 1/5] Initial plan From 5c1748ac32eef350f10707e4feeef94573ed2c0f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 13 Nov 2025 00:23:05 +0000 Subject: [PATCH 2/5] Add system & prompt artifacts (PR #1): prompts and capsules Co-authored-by: igor-holt <125706350+igor-holt@users.noreply.github.com> --- capsules/hmoc-0.2.json | 240 +++++++++++++++++++++++ capsules/lidlift-v1.json | 182 +++++++++++++++++ capsules/runbooks.json | 324 +++++++++++++++++++++++++++++++ prompts/legendary_lidlift_v14.md | 242 +++++++++++++++++++++++ prompts/micro_core.txt | 1 + prompts/nano_core.txt | 1 + 6 files changed, 990 insertions(+) create mode 100644 capsules/hmoc-0.2.json create mode 100644 capsules/lidlift-v1.json create mode 100644 capsules/runbooks.json create mode 100644 prompts/legendary_lidlift_v14.md create mode 100644 prompts/micro_core.txt create mode 100644 prompts/nano_core.txt diff --git a/capsules/hmoc-0.2.json b/capsules/hmoc-0.2.json new file mode 100644 index 0000000..5d7ddce --- /dev/null +++ b/capsules/hmoc-0.2.json @@ -0,0 +1,240 @@ +{ + "name": "hmoc", + "version": "0.2.0", + "type": "coordination-framework", + "description": "H-MOC v0.2 - Hierarchical Multi-Objective Coordinator for CVE pipeline orchestration", + "purpose": "Coordinate multi-objective optimization, performance tracking, and structured reporting for LID-LIFT pipeline", + "features": { + "run_report": { + "enabled": true, + "description": "Structured execution reports with metrics and outcomes", + "outputs": [ + "execution_time", + "resource_utilization", + "pipeline_metrics", + "quality_scores", + "error_logs" + ] + }, + "multi_objective_tracking": { + "enabled": true, + "description": "Track and optimize multiple objectives simultaneously", + "objectives": [ + "accuracy", + "recall", + "precision", + "f1_score", + "processing_time", + "resource_efficiency" + ] + }, + "performance_monitoring": { + "enabled": true, + "description": "Real-time performance monitoring and alerting", + "metrics": [ + "throughput", + "latency", + "error_rate", + "queue_depth" + ] + }, + "hierarchical_coordination": { + "enabled": true, + "description": "Hierarchical task coordination across pipeline stages", + "levels": [ + "strategic", + "tactical", + "operational" + ] + } + }, + "integration": { + "lidlift": { + "version": "1.4", + "capsule_ref": "capsules/lidlift-v1.json", + "coordination_points": [ + "ingest_completion", + "alignment_quality", + "arbiter_predictions", + "refractor_alerts", + "evidence_calculation" + ] + } + }, + "reporting": { + "format": "JSON", + "frequency": { + "real_time": "on_event", + "batch": "per_run", + "aggregated": "daily" + }, + "sections": { + "executive_summary": { + "description": "High-level overview of run outcomes", + "fields": [ + "run_id", + "timestamp", + "status", + "cves_processed", + "high_priority_count", + "duration" + ] + }, + "pipeline_metrics": { + "description": "Detailed metrics for each pipeline stage", + "fields": [ + "stage_name", + "input_count", + "output_count", + "duration", + "resource_usage", + "quality_score" + ] + }, + "objectives_tracking": { + "description": "Multi-objective optimization status", + "fields": [ + "objective_name", + "target_value", + "actual_value", + "deviation", + "trend" + ] + }, + "alerts_and_warnings": { + "description": "Anomalies, failures, or degraded performance", + "fields": [ + "severity", + "component", + "message", + "timestamp", + "resolution_status" + ] + }, + "recommendations": { + "description": "Automated recommendations for optimization", + "fields": [ + "area", + "suggestion", + "expected_impact", + "priority" + ] + } + }, + "output_destinations": [ + "file_system", + "notion", + "monitoring_dashboard", + "webhook" + ] + }, + "optimization": { + "pareto_frontier": { + "enabled": true, + "description": "Identify Pareto-optimal configurations", + "objectives_to_balance": [ + "accuracy_vs_speed", + "precision_vs_recall", + "cost_vs_quality" + ] + }, + "auto_tuning": { + "enabled": false, + "description": "Automatic hyperparameter tuning (future feature)", + "note": "Planned for v0.3" + } + }, + "configuration": { + "environment_variables": { + "HMOC_REPORT_LEVEL": { + "required": false, + "default": "standard", + "options": ["minimal", "standard", "detailed", "verbose"], + "description": "Level of detail in run reports" + }, + "HMOC_ALERT_THRESHOLD": { + "required": false, + "default": "medium", + "options": ["low", "medium", "high", "critical"], + "description": "Minimum severity for alerting" + }, + "HMOC_REPORTING_ENABLED": { + "required": false, + "default": true, + "description": "Enable/disable H-MOC reporting" + } + } + }, + "coordination_protocol": { + "stage_handoff": { + "method": "event_driven", + "validation": "schema_check", + "retry_policy": "exponential_backoff", + "timeout": "configurable_per_stage" + }, + "error_handling": { + "strategy": "graceful_degradation", + "fallback": "checkpoint_recovery", + "notification": "immediate" + }, + "resource_management": { + "allocation": "dynamic", + "scaling": "horizontal", + "limits": "enforced" + } + }, + "observability": { + "metrics_collection": { + "provider": "Prometheus", + "scrape_interval": "15s", + "retention": "30d" + }, + "distributed_tracing": { + "enabled": true, + "propagation": "W3C_trace_context", + "sampling": "adaptive" + }, + "logging": { + "format": "structured_json", + "fields": [ + "timestamp", + "run_id", + "stage", + "level", + "message", + "context" + ] + } + }, + "api": { + "endpoints": { + "submit_run": { + "method": "POST", + "path": "/api/v1/runs", + "description": "Submit a new CVE analysis run" + }, + "get_report": { + "method": "GET", + "path": "/api/v1/runs/{run_id}/report", + "description": "Retrieve run report by ID" + }, + "list_runs": { + "method": "GET", + "path": "/api/v1/runs", + "description": "List recent runs with filtering" + }, + "get_metrics": { + "method": "GET", + "path": "/api/v1/metrics", + "description": "Retrieve aggregated metrics" + } + } + }, + "metadata": { + "created": "2025-11-13", + "updated": "2025-11-13", + "status": "active", + "version_notes": "Added run_report integration, multi-objective tracking, and performance monitoring", + "next_release": "0.3.0 - Auto-tuning and ML-based optimization" + } +} diff --git a/capsules/lidlift-v1.json b/capsules/lidlift-v1.json new file mode 100644 index 0000000..df5bd58 --- /dev/null +++ b/capsules/lidlift-v1.json @@ -0,0 +1,182 @@ +{ + "name": "lidlift-v1", + "version": "1.4.0", + "type": "defense-analysis", + "description": "LID-LIFT v1.4 - Layered Intelligence Defense for CVE Matter-Analysis", + "mission": "Blue-team CVE analysis pipeline: NVD ingest → positional alignment → stacked arbiter → ε-refractors → Bayesian evidence", + "guardrails": { + "offensive_payloads": false, + "cryptographic_breaking": false, + "exploit_generation": false, + "defense_only": true + }, + "pipeline": { + "stages": [ + { + "name": "ingest", + "module": "src.ingest.nvd_client", + "description": "Fetch and normalize CVE data from NVD API v2.0", + "features": [ + "delta_sync", + "etag_support", + "exponential_backoff", + "jsonl_output" + ] + }, + { + "name": "alignment", + "module": "src.alignment", + "description": "Align vulnerability representations across embedding spaces", + "methods": ["procrustes", "cca"], + "threshold": { + "r_squared": 0.8, + "description": "Minimum R² coefficient for alignment quality" + } + }, + { + "name": "arbiter", + "module": "src.models.arbiter", + "description": "Stacked ensemble learning with Pareto knee detection", + "components": [ + "base_learners", + "meta_learner", + "pareto_optimizer" + ] + }, + { + "name": "refractors", + "module": "src.refractors", + "description": "Detect distributional shifts in vulnerability patterns", + "components": ["epsilon_refractor", "shift_detector"], + "threshold": { + "epsilon": 0.05, + "description": "Maximum acceptable ε-divergence before alert" + } + }, + { + "name": "evidence", + "module": "src.evaluation.evidence", + "description": "Calculate Bayesian evidence for model comparison", + "methods": ["BIC", "WAIC", "bayes_factor", "jeffreys_scale"], + "default_method": "WAIC" + } + ] + }, + "technology_stack": { + "language": "Python", + "version": "3.11+", + "cuda": { + "required": false, + "description": "Optional CUDA support for GPU-accelerated operations" + }, + "container": { + "runtime": "Docker", + "orchestration": "Kubernetes", + "platform": "GKE", + "isolation": "gVisor" + }, + "workflow": { + "engine": "Argo Workflows", + "infrastructure": "Terraform", + "ci_cd": "GitHub Actions" + } + }, + "configuration": { + "environment_variables": { + "NVD_API_KEY": { + "required": false, + "description": "API key for NVD access (optional but recommended)" + }, + "ALIGNMENT_R2_THRESHOLD": { + "required": false, + "default": 0.8, + "description": "Minimum R² for alignment quality" + }, + "EPSILON_THRESHOLD": { + "required": false, + "default": 0.05, + "description": "Threshold for refractor alerts" + }, + "EVIDENCE_METHOD": { + "required": false, + "default": "WAIC", + "options": ["BIC", "WAIC"], + "description": "Bayesian evidence calculation method" + }, + "LOG_LEVEL": { + "required": false, + "default": "INFO", + "options": ["DEBUG", "INFO", "WARN", "ERROR"], + "description": "Logging verbosity level" + }, + "CUDA_VISIBLE_DEVICES": { + "required": false, + "description": "GPU device selection for CUDA operations" + } + }, + "secrets_management": { + "source": "environment_or_gcp_secret_manager", + "policy": "no_secrets_in_code" + } + }, + "integration": { + "h_moc": { + "version": "0.2", + "features": [ + "run_report", + "multi_objective_tracking", + "performance_monitoring" + ], + "capsule_ref": "capsules/hmoc-0.2.json" + }, + "runbooks": { + "capsule_ref": "capsules/runbooks.json" + } + }, + "operational": { + "performance": { + "batch_throughput": "≥1000 CVEs/hour", + "api_response_time": "<5 seconds", + "concurrent_processing": true + }, + "reliability": { + "graceful_degradation": true, + "retry_logic": "exponential_backoff", + "error_logging": "comprehensive" + }, + "observability": { + "logging": { + "format": "JSON", + "structured": true + }, + "metrics": "Prometheus", + "tracing": "distributed", + "reporting": "H-MOC" + } + }, + "development": { + "tasks_directory": ".copilot/tasks/", + "workflow": "PR-per-task", + "references": ["SECURITY.md", "CODEOWNERS", ".copilot/AGENT_GUIDE.md"] + }, + "documentation": { + "primary": "prompts/legendary_lidlift_v14.md", + "micro_core": "prompts/micro_core.txt", + "nano_core": "prompts/nano_core.txt" + }, + "security": { + "policy": "SECURITY.md", + "scanning": { + "code": "CodeQL", + "containers": "Trivy", + "dependencies": "automated" + }, + "vulnerability_threshold": "fail_on_high_critical" + }, + "metadata": { + "created": "2025-11-13", + "updated": "2025-11-13", + "status": "active", + "maintainers": ["cve-analysis-team"] + } +} diff --git a/capsules/runbooks.json b/capsules/runbooks.json new file mode 100644 index 0000000..7c844d6 --- /dev/null +++ b/capsules/runbooks.json @@ -0,0 +1,324 @@ +{ + "name": "runbooks", + "version": "1.0.0", + "type": "operational-procedures", + "description": "Operational runbooks for CVE Matter-Analysis OS - Standard procedures for deployment, troubleshooting, and maintenance", + "purpose": "Provide step-by-step procedures for common operational tasks and incident response", + "runbooks": { + "deployment": { + "initial_setup": { + "title": "Initial Deployment Setup", + "description": "Deploy CVE Matter-Analysis OS for the first time", + "prerequisites": [ + "GCP project created and configured", + "Terraform installed (v1.5+)", + "kubectl installed and configured", + "Docker installed for local testing" + ], + "steps": [ + "Clone repository and checkout main branch", + "Configure environment variables in .env file", + "Run `terraform init` in infrastructure/ directory", + "Run `terraform plan` and review changes", + "Run `terraform apply` to provision infrastructure", + "Build Docker images: `docker build -t lidlift:latest .`", + "Push images to GCR: `docker push gcr.io/PROJECT_ID/lidlift:latest`", + "Deploy to Kubernetes: `kubectl apply -f k8s/`", + "Verify deployment: `kubectl get pods -n cve-analysis`", + "Run smoke tests: `./scripts/smoke_test.sh`" + ], + "validation": [ + "All pods in Running state", + "Health check endpoints returning 200 OK", + "Smoke tests passing", + "Monitoring dashboards showing metrics" + ], + "rollback": "Run `terraform destroy` and `kubectl delete -f k8s/` if needed" + }, + "update_deployment": { + "title": "Update Existing Deployment", + "description": "Deploy new version to existing infrastructure", + "prerequisites": [ + "New version tagged in Git", + "CI/CD pipeline passed", + "Security scans completed" + ], + "steps": [ + "Review changelog and breaking changes", + "Build new Docker image with version tag", + "Push image to container registry", + "Update k8s manifests with new image tag", + "Run `kubectl diff -f k8s/` to preview changes", + "Apply changes: `kubectl apply -f k8s/`", + "Monitor rollout: `kubectl rollout status deployment/lidlift`", + "Run post-deployment tests", + "Verify metrics and logs" + ], + "validation": [ + "Rollout completed successfully", + "No increase in error rates", + "Response times within SLA", + "All features functioning correctly" + ], + "rollback": "Run `kubectl rollout undo deployment/lidlift`" + } + }, + "troubleshooting": { + "high_error_rate": { + "title": "Troubleshooting High Error Rate", + "description": "Diagnose and resolve elevated error rates in the pipeline", + "symptoms": [ + "Error rate >5% in monitoring dashboard", + "Failed CVE processing jobs", + "Alert from H-MOC reporting" + ], + "investigation_steps": [ + "Check H-MOC run report for failed stages", + "Review logs: `kubectl logs -l app=lidlift --tail=100`", + "Check pod status: `kubectl get pods -n cve-analysis`", + "Verify NVD API status and rate limits", + "Check resource utilization: CPU, memory, disk", + "Review recent changes in Git history", + "Check external dependencies (NVD API, storage, etc.)" + ], + "common_causes": { + "nvd_api_rate_limit": { + "symptom": "429 Too Many Requests in logs", + "resolution": "Increase backoff delay or add API key" + }, + "resource_exhaustion": { + "symptom": "OOMKilled or CPU throttling", + "resolution": "Increase resource limits in k8s manifests" + }, + "data_quality": { + "symptom": "Validation errors in alignment stage", + "resolution": "Check data schema and add additional validation" + }, + "network_issues": { + "symptom": "Connection timeouts", + "resolution": "Check network policies and firewall rules" + } + }, + "resolution": "Apply appropriate fix based on root cause, then monitor for 1 hour", + "escalation": "If unresolved after 2 hours, escalate to senior SRE" + }, + "performance_degradation": { + "title": "Performance Degradation", + "description": "Address slow processing times or reduced throughput", + "symptoms": [ + "Processing time >10s per CVE (baseline: <5s)", + "Queue depth increasing", + "H-MOC reports showing latency increase" + ], + "investigation_steps": [ + "Check current throughput vs baseline", + "Review resource utilization metrics", + "Check for epsilon-refractor shift alerts", + "Analyze slow query logs if applicable", + "Review recent data volume changes", + "Check for background jobs or maintenance tasks" + ], + "common_causes": { + "data_volume_spike": { + "symptom": "Sudden increase in CVE count", + "resolution": "Scale up pods: `kubectl scale deployment/lidlift --replicas=10`" + }, + "model_complexity": { + "symptom": "Arbiter predictions taking longer", + "resolution": "Review model configuration, consider simplification" + }, + "database_slowdown": { + "symptom": "High database latency", + "resolution": "Check indexes, optimize queries, or scale database" + } + }, + "resolution": "Apply scaling or optimization, validate throughput returns to baseline", + "escalation": "If performance does not improve within 1 hour, escalate to engineering team" + }, + "alignment_failure": { + "title": "Alignment Quality Below Threshold", + "description": "Handle cases where R² falls below configured threshold", + "symptoms": [ + "Alignment R² < 0.8", + "Pipeline stopping at alignment stage", + "Alert from refractor module" + ], + "investigation_steps": [ + "Check H-MOC report for alignment metrics", + "Review input data quality and distribution", + "Verify reference embeddings are current", + "Check for data drift using refractor module", + "Review recent changes to embedding generation" + ], + "common_causes": { + "distributional_shift": { + "symptom": "Refractor showing high ε-divergence", + "resolution": "Update reference embeddings with recent data" + }, + "bad_input_data": { + "symptom": "Anomalies in CVE descriptions or metadata", + "resolution": "Filter out problematic CVEs, improve validation" + }, + "threshold_too_strict": { + "symptom": "Consistently failing alignment despite good results", + "resolution": "Review and adjust R² threshold if appropriate" + } + }, + "resolution": "Apply fix and reprocess affected CVE batch", + "escalation": "If alignment issues persist, escalate to ML engineering team" + } + }, + "maintenance": { + "model_retraining": { + "title": "Arbiter Model Retraining", + "description": "Periodic retraining of the stacked arbiter model", + "frequency": "Monthly or when performance degrades", + "prerequisites": [ + "Training data collected (≥10K labeled CVEs)", + "GPU resources available (optional but recommended)", + "Previous model backed up" + ], + "steps": [ + "Export recent CVE data with ground truth labels", + "Prepare training dataset with validation split", + "Backup current model: `cp models/arbiter.pkl models/arbiter_backup_$(date +%Y%m%d).pkl`", + "Run training script: `python scripts/train_arbiter.py --data data/training.jsonl`", + "Evaluate model on validation set", + "Compare metrics with previous model (accuracy, F1, etc.)", + "If improved, deploy new model to staging", + "Run integration tests in staging", + "Deploy to production if tests pass", + "Monitor performance for 24 hours" + ], + "validation": [ + "Validation accuracy ≥ previous model", + "No regression in key metrics", + "Pareto frontier maintained or improved" + ], + "rollback": "Restore previous model from backup if performance degrades" + }, + "reference_embedding_update": { + "title": "Update Reference Embeddings", + "description": "Refresh reference embeddings for alignment module", + "frequency": "Quarterly or when alignment quality degrades", + "prerequisites": [ + "Recent representative CVE sample (≥1K CVEs)", + "Current embeddings backed up" + ], + "steps": [ + "Collect recent CVE sample spanning multiple categories", + "Generate embeddings: `python scripts/generate_embeddings.py --input data/reference_cves.jsonl`", + "Backup current reference: `cp data/reference_embeddings.npy data/reference_embeddings_backup_$(date +%Y%m%d).npy`", + "Copy new embeddings to reference location", + "Run validation tests", + "Monitor alignment R² for next 24 hours" + ], + "validation": [ + "Alignment R² ≥ 0.8 on test set", + "Refractor ε-divergence within acceptable range" + ], + "rollback": "Restore previous reference embeddings if alignment quality drops" + }, + "database_maintenance": { + "title": "Database Maintenance", + "description": "Routine database maintenance and optimization", + "frequency": "Weekly", + "steps": [ + "Backup database: `./scripts/backup_db.sh`", + "Run VACUUM and ANALYZE on PostgreSQL (if applicable)", + "Check and rebuild indexes if needed", + "Archive old data (>1 year)", + "Verify backup integrity", + "Update statistics" + ], + "validation": [ + "Backup successful and restorable", + "Query performance maintained", + "Disk space within limits" + ] + } + }, + "incident_response": { + "security_vulnerability": { + "title": "Security Vulnerability Discovered", + "description": "Response to security vulnerability in dependencies or code", + "severity": "High", + "immediate_actions": [ + "Assess vulnerability scope and impact", + "Check if vulnerability is actively exploited", + "Review SECURITY.md for disclosure process", + "Notify security team immediately" + ], + "steps": [ + "Isolate affected components if necessary", + "Apply security patches from upstream", + "Run security scans: `trivy image lidlift:latest`", + "Update dependencies: `pip install -U vulnerable-package`", + "Rebuild and test Docker image", + "Deploy patch to staging first", + "Run security validation tests", + "Deploy to production if tests pass", + "Document incident and resolution" + ], + "validation": [ + "Vulnerability no longer detected by scanners", + "No functional regressions", + "Security tests passing" + ], + "communication": "Update stakeholders via incident channel" + }, + "data_breach_suspected": { + "title": "Suspected Data Breach", + "description": "Response to potential unauthorized data access", + "severity": "Critical", + "immediate_actions": [ + "Activate incident response team", + "Preserve logs and evidence", + "Do NOT make changes that might destroy evidence", + "Notify security team and legal counsel" + ], + "steps": [ + "Follow SECURITY.md incident response procedures", + "Review access logs for anomalies", + "Identify scope of potential exposure", + "Rotate all credentials and secrets", + "Review and strengthen access controls", + "Conduct forensic analysis", + "Document timeline and findings", + "Implement additional monitoring" + ], + "escalation": "Critical - immediate escalation to CISO and legal team", + "communication": "Follow legal and compliance requirements for disclosure" + } + } + }, + "contacts": { + "on_call": { + "primary": "sre-oncall@example.com", + "secondary": "sre-backup@example.com" + }, + "escalation": { + "engineering_lead": "eng-lead@example.com", + "security_team": "security@example.com", + "legal": "legal@example.com" + }, + "vendor_support": { + "gcp": "GCP Support Console", + "github": "GitHub Support" + } + }, + "tools_and_links": { + "monitoring": "https://monitoring.example.com/cve-analysis", + "logs": "https://logs.example.com/cve-analysis", + "documentation": "https://docs.example.com/cve-analysis", + "incident_tracker": "https://incidents.example.com", + "chat": "Slack: #cve-analysis-ops" + }, + "metadata": { + "created": "2025-11-13", + "updated": "2025-11-13", + "status": "active", + "maintainers": ["sre-team", "cve-analysis-team"], + "review_frequency": "Quarterly" + } +} diff --git a/prompts/legendary_lidlift_v14.md b/prompts/legendary_lidlift_v14.md new file mode 100644 index 0000000..f70e9d6 --- /dev/null +++ b/prompts/legendary_lidlift_v14.md @@ -0,0 +1,242 @@ +# LID-LIFT v1.4 System Description + +**Version:** 1.4 +**Type:** Defense-Only CVE Matter-Analysis Pipeline +**Mission:** Blue-team CVE matter-analysis from NVD ingest → positional alignment → stacked arbiter → ε-refractors → Bayesian evidence + +## System Overview + +LID-LIFT (Layered Intelligence Defense - Lateral Inference Framework for Threat Analysis) is a defense-focused CVE matter-analysis system designed to: + +1. **Ingest**: Pull and normalize CVE data from NVD (National Vulnerability Database) +2. **Align**: Perform positional alignment across multiple vulnerability representations +3. **Arbitrate**: Apply stacked ensemble learning with Pareto knee detection +4. **Refract**: Use epsilon-refractors to detect distributional shifts +5. **Evidence**: Calculate Bayesian evidence scores for vulnerability prioritization + +## Core Principles + +### Defense-Only Stance +- **NO offensive payloads** - All analysis is defensive and protective +- **NO cryptographic-breaking** - System does not attempt to break or weaken cryptography +- **NO exploit generation** - System focuses on defense, detection, and prioritization +- **YES vulnerability analysis** - System analyzes and prioritizes vulnerabilities for remediation + +### Chain of Thought (CoT) Disabled +LID-LIFT v1.4 operates in direct inference mode without explicit chain-of-thought reasoning paths. This provides: +- Faster response times for high-volume CVE processing +- Reduced token consumption in production pipelines +- Direct output generation without intermediate reasoning steps + +### H-MOC Run Report Integration +LID-LIFT integrates with H-MOC (Hierarchical Multi-Objective Coordinator) for: +- Structured run reports with execution metrics +- Multi-objective optimization tracking +- Performance monitoring and alerting +- Audit trail for compliance and review + +## Technical Architecture + +### 1. NVD Ingest Module (`src/ingest/nvd_client.py`) +**Purpose:** Fetch and normalize CVE data from NVD API v2.0 + +**Key Features:** +- Delta synchronization with ETag support +- Exponential backoff for rate limiting +- JSONL output format for efficient streaming +- Checkpointing for recovery from failures + +**Outputs:** +- Normalized CVE records in JSONL format +- Metadata including CVSS scores, CWE mappings, CPE data +- Timestamp-based tracking for incremental updates + +### 2. Positional Alignment (`src/alignment/`) +**Purpose:** Align vulnerability representations across different embedding spaces + +**Components:** +- **Procrustes Alignment** (`procrustes.py`): Orthogonal transformation for shape matching +- **Canonical Correlation Analysis** (`cca.py`): Linear relationships between embedding spaces + +**Key Metrics:** +- R² (coefficient of determination) ≥ configured threshold +- Alignment quality scores +- Transformation matrices for cross-space mapping + +**Use Case:** Enable comparison of vulnerabilities from different sources or time periods + +### 3. Stacked Arbiter (`src/models/arbiter.py`) +**Purpose:** Ensemble learning for vulnerability severity prediction + +**Architecture:** +- **Base learners**: Multiple weak learners (RandomForest, GradientBoosting, etc.) +- **Meta-learner**: Stacked super-learner for final predictions +- **Pareto knee detection**: Identify optimal trade-off points in multi-objective space + +**Outputs:** +- Severity predictions with confidence intervals +- Feature importance scores +- Pareto-optimal configurations + +### 4. Epsilon-Refractors (`src/refractors/`) +**Purpose:** Detect distributional shifts in vulnerability patterns + +**Components:** +- **Epsilon-refractor** (`epsilon.py`): Measure deviation from expected distributions +- **Shift detector** (`shifts.py`): Identify concept drift and data drift + +**Key Metrics:** +- ε-divergence: Quantify distributional differences +- Shift magnitude and direction +- Alert thresholds for significant changes + +**Use Case:** Detect emerging threat patterns or changes in vulnerability landscape + +### 5. Bayesian Evidence Module (`src/evaluation/evidence.py`) +**Purpose:** Calculate evidence scores for model comparison and vulnerability prioritization + +**Metrics:** +- **BIC** (Bayesian Information Criterion): Model complexity penalty +- **WAIC** (Widely Applicable Information Criterion): Better for hierarchical models +- **Bayes Factor**: Relative evidence between models +- **Jeffreys Scale**: Interpret evidence strength + +**Outputs:** +- Log-evidence scores for each vulnerability +- Model comparison metrics +- Uncertainty quantification + +## Data Flow + +``` +NVD API → Ingest → JSONL → Alignment → Embeddings → Arbiter → Predictions + ↓ + Refractors → Shift Detection + ↓ + Evidence → Priority Scores +``` + +## Security Guardrails + +### Input Validation +- Sanitize all external inputs (CVE data, API responses) +- Validate schema compliance before processing +- Reject malformed or suspicious data + +### Output Filtering +- Never expose internal system paths or credentials +- Sanitize all outputs before external transmission +- Log security-relevant events for audit + +### Dependency Management +- Pin all dependencies with version locks +- Regular security scanning with Trivy +- Automated updates for security patches + +### Secrets Management +- No secrets in source code or configuration files +- Use environment variables or secret management systems (e.g., GCP Secret Manager) +- Rotate credentials regularly + +## Operational Requirements + +### Performance +- Process ≥1000 CVEs per hour in batch mode +- API response time <5 seconds for single CVE queries +- Support concurrent processing with worker pools + +### Reliability +- Graceful degradation on partial failures +- Retry logic with exponential backoff +- Comprehensive error logging and monitoring + +### Observability +- Structured logging (JSON format) +- Prometheus metrics for monitoring +- Distributed tracing for request flows +- H-MOC run reports for execution analysis + +## Deployment Targets + +### Docker +- Multi-stage builds for minimal image size +- Non-root user for security +- Health check endpoints + +### Kubernetes (GKE) +- Horizontal pod autoscaling based on CPU/memory +- gVisor for enhanced container isolation +- Pod security policies enforced + +### Argo Workflows +- DAG-based pipeline orchestration +- Artifact passing between steps +- Retry and timeout policies + +## Configuration + +### Environment Variables +- `NVD_API_KEY`: API key for NVD access (optional but recommended) +- `ALIGNMENT_R2_THRESHOLD`: Minimum R² for alignment (default: 0.8) +- `EPSILON_THRESHOLD`: Threshold for refractor alerts (default: 0.05) +- `EVIDENCE_METHOD`: Bayesian evidence method (BIC or WAIC, default: WAIC) +- `LOG_LEVEL`: Logging verbosity (DEBUG, INFO, WARN, ERROR) +- `CUDA_VISIBLE_DEVICES`: GPU device selection for CUDA operations + +### Configuration Files +- `config/pipeline.yaml`: Pipeline configuration +- `config/models.yaml`: Model hyperparameters +- `config/thresholds.yaml`: Alert and filtering thresholds + +## Usage Example + +```python +from src.ingest import NVDClient +from src.alignment import align +from src.models import Arbiter +from src.refractors import EpsilonRefractor +from src.evaluation import calculate_evidence + +# 1. Ingest CVE data +client = NVDClient() +cves = client.fetch_recent(days=7) + +# 2. Align embeddings +X_aligned = align(cves.embeddings, reference_space) + +# 3. Predict severity +arbiter = Arbiter() +predictions = arbiter.predict(X_aligned) + +# 4. Detect shifts +refractor = EpsilonRefractor() +shifts = refractor.detect(X_aligned) + +# 5. Calculate evidence +evidence = calculate_evidence(predictions, cves.ground_truth) +``` + +## Versioning and Updates + +- **Current Version**: 1.4 +- **Last Updated**: 2025-11-13 +- **Next Review**: Quarterly or on major CVE landscape changes +- **Breaking Changes**: Require major version bump and migration guide + +## References + +- NVD API Documentation: https://nvd.nist.gov/developers +- CVSS v3.1 Specification: https://www.first.org/cvss/v3.1/specification-document +- Bayesian Model Comparison: Gelman et al., "Bayesian Data Analysis" (3rd ed.) +- Procrustes Analysis: Goodall, "Procrustes methods in the statistical analysis of shape" + +## Support and Contact + +For questions, issues, or contributions related to LID-LIFT v1.4: +- Internal Issue Tracker: Use GitHub Issues on this repository +- Security Concerns: Follow SECURITY.md disclosure process +- Feature Requests: Tag with `enhancement` label + +--- + +**Reminder**: This is a DEFENSE-ONLY system. Any attempt to use LID-LIFT for offensive purposes, exploit generation, or cryptographic attacks violates the system's design principles and security policy. diff --git a/prompts/micro_core.txt b/prompts/micro_core.txt new file mode 100644 index 0000000..2347805 --- /dev/null +++ b/prompts/micro_core.txt @@ -0,0 +1 @@ +DEFENSE CVE PIPELINE: NVD→align→arbiter→ε-refract→Bayes-evidence. NO exploits, NO crypto-break. Link-pack: legendary_lidlift_v14.md, capsules/lidlift-v1.json, capsules/hmoc-0.2.json. Python3.11+CUDA optional. GKE+gVisor+Argo. H-MOC run_report. Threshold: R²≥0.8, ε<0.05. WAIC evidence. Secrets via env/GCP. Tasks: .copilot/tasks/*.md. PR-per-task. Ref: SECURITY.md, CODEOWNERS. diff --git a/prompts/nano_core.txt b/prompts/nano_core.txt new file mode 100644 index 0000000..74dfe1d --- /dev/null +++ b/prompts/nano_core.txt @@ -0,0 +1 @@ +DEFENSE CVE: NVD→align→arbiter→refract→evidence. NO exploits. Docs: legendary_lidlift_v14.md. Py3.11. GKE+Argo. Tasks: .copilot/tasks/. PR-per-task. SECURITY.md. From f87a56e15af251ba5108f9cb1fa582edd656a5d5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 13 Nov 2025 00:31:24 +0000 Subject: [PATCH 3/5] Add Copilot guardrails & onboarding scaffolding (PR #2) Co-authored-by: igor-holt <125706350+igor-holt@users.noreply.github.com> --- .copilot/AGENT_GUIDE.md | 361 ++++++++++++++++++++++ .copilot/tasks/010_ingest_nvd.md | 260 ++++++++++++++++ .copilot/tasks/020_alignment.md | 394 +++++++++++++++++++++++++ .copilot/tasks/030_arbiter.md | 94 ++++++ .copilot/tasks/040_refractors.md | 116 ++++++++ .copilot/tasks/050_evidence.md | 150 ++++++++++ .copilot/tasks/060_notion_sync.md | 156 ++++++++++ .copilot/tasks/070_capsules_publish.md | 219 ++++++++++++++ .copilot/tasks/080_cuda_support.md | 250 ++++++++++++++++ .copilot/tasks/090_bridge.md | 363 +++++++++++++++++++++++ CODEOWNERS | 54 ++++ README.md | 221 ++++++++++++++ SECURITY.md | 227 ++++++++++++++ 13 files changed, 2865 insertions(+) create mode 100644 .copilot/AGENT_GUIDE.md create mode 100644 .copilot/tasks/010_ingest_nvd.md create mode 100644 .copilot/tasks/020_alignment.md create mode 100644 .copilot/tasks/030_arbiter.md create mode 100644 .copilot/tasks/040_refractors.md create mode 100644 .copilot/tasks/050_evidence.md create mode 100644 .copilot/tasks/060_notion_sync.md create mode 100644 .copilot/tasks/070_capsules_publish.md create mode 100644 .copilot/tasks/080_cuda_support.md create mode 100644 .copilot/tasks/090_bridge.md create mode 100644 CODEOWNERS create mode 100644 SECURITY.md diff --git a/.copilot/AGENT_GUIDE.md b/.copilot/AGENT_GUIDE.md new file mode 100644 index 0000000..104dfd4 --- /dev/null +++ b/.copilot/AGENT_GUIDE.md @@ -0,0 +1,361 @@ +# Copilot Agent Guide + +## Overview + +Welcome to the CVE Matter-Analysis OS repository. This guide provides instructions for GitHub Copilot agents working on this project. + +## Repository Mission + +**Defense-only CVE matter-analysis pipeline**: Build and maintain a blue-team vulnerability analysis system that ingests data from NVD, performs positional alignment, applies stacked arbitration, detects distributional shifts with ε-refractors, and calculates Bayesian evidence scores. + +## Core Principles + +### 1. Defense-Only Stance + +This repository is **strictly for defensive security purposes**. All code and analysis must support: +- ✅ Vulnerability detection and prioritization +- ✅ Defensive security measures +- ✅ Risk assessment and mitigation +- ❌ **NO exploit payload generation** +- ❌ **NO offensive security tools** +- ❌ **NO cryptographic-breaking attempts** + +### 2. File-Anchored Tasks + +All development work is organized into discrete, file-anchored tasks located in `.copilot/tasks/`. Each task: +- Has a clear goal and acceptance criteria +- Specifies which files to create or modify +- Includes validation requirements +- Should result in exactly one Pull Request + +### 3. One PR Per Task + +Follow the workflow: +1. Read the task definition in `.copilot/tasks/` +2. Implement the changes specified +3. Create a single, focused PR for that task +4. Reference this onboarding issue in the PR description +5. Move to the next task only after the current PR is merged + +### 4. Security-First Development + +- **Never commit secrets**: Use environment variables or secret management systems +- **Scan dependencies**: Check for vulnerabilities before adding new packages +- **Validate inputs**: Sanitize all external data sources +- **Follow SECURITY.md**: Adhere to the security policy for all changes + +## Task Execution Workflow + +### Step 1: Select a Task + +Tasks are numbered sequentially in `.copilot/tasks/`: +- `010_ingest_nvd.md` - NVD data ingestion +- `020_alignment.md` - Positional alignment +- `030_arbiter.md` - Stacked arbiter +- `040_refractors.md` - Epsilon-refractors +- `050_evidence.md` - Bayesian evidence +- `060_notion_sync.md` - Notion synchronization +- `070_capsules_publish.md` - Capsules publishing +- `080_cuda_support.md` - CUDA/GPU support +- `090_bridge.md` - Webhook + Argo bridge + +**Start with task 010 and proceed sequentially.** + +### Step 2: Read the Task Definition + +Each task file includes: +- **Goal**: What needs to be accomplished +- **Files to Edit**: Specific files to create or modify +- **Requirements**: Functional and technical requirements +- **Acceptance Criteria**: How to validate the implementation +- **Dependencies**: Any prerequisite tasks or external dependencies + +### Step 3: Implement the Changes + +- Make **minimal, surgical changes** to accomplish the task +- Follow existing code patterns and conventions +- Add tests consistent with the repository's testing approach +- Update documentation if relevant to your changes +- Ensure your code passes linters and builds + +### Step 4: Validate Your Work + +Before creating a PR: +- ✅ Run linters and formatters +- ✅ Run existing tests to ensure no regressions +- ✅ Add/update tests for your changes +- ✅ Verify acceptance criteria are met +- ✅ Check that no secrets are committed +- ✅ Run security scanners on new dependencies + +### Step 5: Create a Pull Request + +Your PR description should include: +- **Short rationale**: Why these changes are needed +- **Edited file list**: What files were modified +- **Validation evidence**: Test output, build logs, or manual verification results +- **Reference**: Link to the task file and this onboarding guide + +Example PR template: +```markdown +## Task: [Task Number] - [Task Name] + +**Rationale**: [Brief explanation of why these changes are needed] + +**Task Reference**: `.copilot/tasks/XXX_task_name.md` + +### Changes Made +- Created/Modified: [list of files] +- Added tests in: [test files] +- Updated documentation: [doc files] + +### Validation +- [x] Linters pass +- [x] Tests pass +- [x] Security scan clean +- [x] Acceptance criteria met + +**Test Output**: +``` +[Include relevant test results or validation evidence] +``` + +**References**: Related to onboarding issue #[issue number] +``` + +### Step 6: Address Review Feedback + +- Respond to code review comments promptly +- Make requested changes in additional commits +- Re-validate after changes +- Do not force-push (preserve review history) + +## Technology Stack + +### Languages and Frameworks +- **Python 3.11+**: Primary language for all modules +- **Optional CUDA**: GPU acceleration for compute-intensive operations +- **Docker**: Container runtime +- **Kubernetes (GKE)**: Orchestration platform with gVisor isolation + +### Infrastructure +- **Terraform**: Infrastructure as Code +- **Argo Workflows**: DAG-based pipeline orchestration +- **GitHub Actions**: CI/CD automation +- **Prometheus**: Metrics and monitoring + +### Key Libraries (examples) +- `numpy`, `scipy`: Numerical computing +- `scikit-learn`: Machine learning (arbiter, alignment) +- `requests`: HTTP client for NVD API +- `pytest`: Testing framework + +## Development Environment + +### Local Setup +```bash +# Clone repository +git clone https://github.com/igor-holt/Instinct.git +cd Instinct + +# Create virtual environment +python3.11 -m venv venv +source venv/bin/activate # or `venv\Scripts\activate` on Windows + +# Install dependencies +pip install -r requirements.txt + +# Install development dependencies +pip install -r requirements-dev.txt + +# Run tests +pytest tests/ + +# Run linters +flake8 src/ +black --check src/ +mypy src/ +``` + +### Docker Development +```bash +# Build image +docker build -t lidlift:dev . + +# Run container +docker run -it --rm \ + -e NVD_API_KEY=$NVD_API_KEY \ + -v $(pwd)/data:/app/data \ + lidlift:dev + +# Run tests in container +docker run -it --rm lidlift:dev pytest +``` + +## Code Standards + +### Python Style +- Follow PEP 8 style guide +- Use `black` for automatic formatting +- Use `flake8` for linting +- Use `mypy` for type checking +- Maximum line length: 100 characters +- Use type hints for function signatures + +### Documentation +- Write docstrings for all public functions and classes +- Use NumPy-style docstrings +- Keep inline comments minimal and focused on "why" not "what" +- Update relevant documentation when changing APIs + +### Testing +- Write unit tests for all new functions +- Aim for >80% code coverage +- Use pytest fixtures for common setup +- Mock external dependencies (NVD API, etc.) +- Include integration tests for full pipeline runs + +### Git Practices +- Write clear, concise commit messages +- Use present tense ("Add feature" not "Added feature") +- Reference issue numbers in commits when applicable +- Keep commits focused and atomic +- Do not commit generated files or dependencies + +## Security Guidelines + +### Secrets Management +- **Never commit**: API keys, passwords, tokens, certificates +- **Use environment variables**: For configuration and secrets +- **Use secret managers**: GCP Secret Manager for production +- **Rotate credentials**: Regularly update API keys and tokens + +### Dependency Security +- **Pin versions**: Use exact versions in requirements.txt +- **Scan before adding**: Check new dependencies for known vulnerabilities +- **Update regularly**: Apply security patches promptly +- **Minimize dependencies**: Only add necessary packages + +### Code Security +- **Validate inputs**: Sanitize all external data +- **Use parameterized queries**: Prevent injection attacks +- **Avoid eval()**: Never use eval() or exec() on untrusted input +- **Handle errors gracefully**: Don't expose internal details in error messages + +### Reporting Vulnerabilities +- Follow the process in `SECURITY.md` +- Report security issues privately to security team +- Do not create public issues for security vulnerabilities + +## Common Commands + +### Testing +```bash +# Run all tests +pytest + +# Run with coverage +pytest --cov=src --cov-report=html + +# Run specific test file +pytest tests/test_nvd_client.py + +# Run tests matching pattern +pytest -k "test_alignment" +``` + +### Linting and Formatting +```bash +# Format code +black src/ tests/ + +# Check formatting +black --check src/ tests/ + +# Run flake8 +flake8 src/ tests/ + +# Type checking +mypy src/ +``` + +### Building and Running +```bash +# Build Docker image +docker build -t lidlift:latest . + +# Run locally +python -m src.main + +# Run specific module +python -m src.ingest.nvd_client +``` + +### Infrastructure +```bash +# Initialize Terraform +terraform init + +# Plan infrastructure changes +terraform plan + +# Apply infrastructure changes +terraform apply + +# Deploy to Kubernetes +kubectl apply -f k8s/ + +# Check deployment status +kubectl get pods -n cve-analysis +``` + +## Troubleshooting + +### Common Issues + +**Issue**: Import errors when running tests +**Solution**: Ensure you've installed development dependencies: `pip install -r requirements-dev.txt` + +**Issue**: Docker build fails +**Solution**: Check Docker version (need 20.10+) and ensure you have sufficient disk space + +**Issue**: Kubernetes deployment fails +**Solution**: Verify kubectl is configured correctly and you have access to the GKE cluster + +**Issue**: Tests fail locally but pass in CI +**Solution**: Ensure your local environment matches CI environment (Python version, dependencies) + +### Getting Help + +1. **Check documentation**: Review `prompts/legendary_lidlift_v14.md` and task files +2. **Review logs**: Check application logs and CI logs for error details +3. **Consult runbooks**: See `capsules/runbooks.json` for operational procedures +4. **Ask for help**: Create an issue or reach out to the team + +## Additional Resources + +- **Main Documentation**: `prompts/legendary_lidlift_v14.md` +- **Capsule Configurations**: `capsules/lidlift-v1.json`, `capsules/hmoc-0.2.json` +- **Operational Runbooks**: `capsules/runbooks.json` +- **Security Policy**: `SECURITY.md` +- **Code Ownership**: `CODEOWNERS` + +## Quick Reference + +| Component | Module Path | Description | +|-----------|-------------|-------------| +| NVD Client | `src/ingest/nvd_client.py` | Fetch CVE data from NVD API | +| Procrustes | `src/alignment/procrustes.py` | Orthogonal alignment | +| CCA | `src/alignment/cca.py` | Canonical correlation analysis | +| Arbiter | `src/models/arbiter.py` | Stacked ensemble learner | +| Refractors | `src/refractors/epsilon.py` | Distributional shift detection | +| Evidence | `src/evaluation/evidence.py` | Bayesian evidence calculation | + +## Version History + +- **v1.0** (2025-11-13): Initial version +- **Current**: v1.0 + +--- + +**Remember**: This is a **defense-only** system. All work must support defensive security purposes and must not generate exploits or offensive capabilities. diff --git a/.copilot/tasks/010_ingest_nvd.md b/.copilot/tasks/010_ingest_nvd.md new file mode 100644 index 0000000..374b8fe --- /dev/null +++ b/.copilot/tasks/010_ingest_nvd.md @@ -0,0 +1,260 @@ +# Task 010: NVD Data Ingestion + +## Goal + +Implement a robust client for fetching and normalizing CVE data from the National Vulnerability Database (NVD) API v2.0. + +## Files to Create/Edit + +### New Files +- `src/ingest/__init__.py` - Package initialization +- `src/ingest/nvd_client.py` - Main NVD API client implementation +- `tests/ingest/__init__.py` - Test package initialization +- `tests/ingest/test_nvd_client.py` - Unit tests for NVD client +- `tests/fixtures/nvd_sample.json` - Sample NVD API response for testing + +### Supporting Files +- `requirements.txt` - Add `requests`, `requests-cache` dependencies +- `src/ingest/config.py` - Configuration for NVD client (optional) + +## Requirements + +### Functional Requirements + +1. **API Integration** + - Connect to NVD API v2.0 (https://services.nvd.nist.gov/rest/json/cves/2.0/) + - Support optional API key for higher rate limits + - Implement pagination for large result sets + +2. **Delta Synchronization** + - Track last successful sync timestamp + - Fetch only CVEs modified since last sync + - Support full refresh mode for initial sync or recovery + +3. **Rate Limiting & Backoff** + - Respect NVD API rate limits (5 requests/30s without key, 50 requests/30s with key) + - Implement exponential backoff for rate limit errors (429) + - Implement retry logic for transient network errors (500, 502, 503) + +4. **ETag Support** + - Store and send ETag headers for conditional requests + - Handle 304 Not Modified responses efficiently + - Reduce unnecessary data transfer + +5. **Data Normalization** + - Extract key fields: CVE ID, description, CVSS scores, CWE mappings, CPE data + - Normalize CVSS v2/v3 scores into consistent format + - Convert timestamps to ISO 8601 format + +6. **Output Format** + - Write normalized CVE records to JSONL (JSON Lines) format + - One CVE per line for efficient streaming + - Support output to file or stdout + +7. **Checkpointing** + - Save progress periodically to enable recovery from failures + - Store checkpoint data (last processed CVE ID, timestamp, ETag) + - Resume from checkpoint on restart + +### Non-Functional Requirements + +1. **Performance** + - Process at least 1000 CVEs per hour + - Use connection pooling for HTTP requests + - Minimize memory footprint for large datasets + +2. **Reliability** + - Gracefully handle API errors + - Retry transient failures up to 3 times + - Log all errors with context + +3. **Observability** + - Structured logging with JSON output + - Log key events: sync started, progress updates, sync completed + - Include metrics: CVEs fetched, API calls made, errors encountered + +4. **Security** + - Store API key in environment variable (NVD_API_KEY) + - Never log API keys or sensitive data + - Validate SSL certificates + +## Implementation Guidance + +### Class Structure + +```python +class NVDClient: + """Client for interacting with NVD API v2.0""" + + def __init__(self, api_key: Optional[str] = None, checkpoint_file: Optional[str] = None): + """Initialize client with optional API key and checkpoint file""" + + def fetch_recent(self, days: int = 7) -> List[Dict]: + """Fetch CVEs modified in the last N days""" + + def fetch_since(self, last_modified: datetime) -> List[Dict]: + """Fetch CVEs modified since given timestamp""" + + def fetch_all(self, start_date: Optional[datetime] = None) -> Iterator[Dict]: + """Fetch all CVEs, optionally starting from a date""" + + def normalize_cve(self, raw_cve: Dict) -> Dict: + """Normalize raw CVE JSON to standardized format""" + + def save_checkpoint(self): + """Save current progress to checkpoint file""" + + def load_checkpoint(self) -> Optional[Dict]: + """Load previous checkpoint if it exists""" +``` + +### Normalized CVE Schema + +```json +{ + "cve_id": "CVE-2023-12345", + "description": "Vulnerability description...", + "published_date": "2023-01-15T10:30:00Z", + "last_modified_date": "2023-01-20T14:45:00Z", + "cvss_v3": { + "base_score": 7.5, + "vector_string": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", + "severity": "HIGH" + }, + "cvss_v2": { + "base_score": 5.0, + "vector_string": "AV:N/AC:L/Au:N/C:P/I:N/A:N" + }, + "cwe_ids": ["CWE-79", "CWE-89"], + "cpe_matches": [ + "cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*" + ], + "references": [ + {"url": "https://example.com/advisory", "source": "vendor"} + ] +} +``` + +### Error Handling + +```python +class NVDAPIError(Exception): + """Base exception for NVD API errors""" + +class RateLimitError(NVDAPIError): + """Raised when rate limit is exceeded""" + +class APIUnavailableError(NVDAPIError): + """Raised when API is temporarily unavailable""" +``` + +## Acceptance Criteria + +### Implementation +- [ ] `NVDClient` class implemented with all required methods +- [ ] Delta sync functionality working (fetches only new/modified CVEs) +- [ ] ETag support implemented for conditional requests +- [ ] Exponential backoff for rate limits and retries for transient errors +- [ ] JSONL output format implemented +- [ ] Checkpointing functionality working (save and resume) +- [ ] CVE normalization extracts all required fields + +### Testing +- [ ] Unit tests for `NVDClient` methods with mocked API responses +- [ ] Test rate limit handling (mock 429 responses) +- [ ] Test retry logic for transient errors (mock 500, 502, 503) +- [ ] Test ETag conditional request behavior +- [ ] Test checkpoint save and restore +- [ ] Test CVE normalization with sample data +- [ ] Test edge cases (empty responses, malformed data, missing fields) +- [ ] Tests achieve >80% code coverage + +### Documentation +- [ ] Docstrings for all public methods +- [ ] Usage examples in module docstring +- [ ] Configuration options documented (API key, checkpoint file, etc.) +- [ ] Error handling behavior documented + +### Quality +- [ ] Code passes flake8 linting +- [ ] Code passes black formatting check +- [ ] Code passes mypy type checking +- [ ] No hardcoded secrets or API keys in code +- [ ] All external dependencies added to requirements.txt + +## Validation Steps + +1. **Manual Testing** + ```bash + # Set API key (optional) + export NVD_API_KEY="your-api-key-here" + + # Fetch recent CVEs + python -m src.ingest.nvd_client --days 7 --output data/cves.jsonl + + # Verify output + head -n 5 data/cves.jsonl | python -m json.tool + + # Test resume from checkpoint + python -m src.ingest.nvd_client --days 7 --checkpoint data/checkpoint.json + ``` + +2. **Automated Testing** + ```bash + # Run unit tests + pytest tests/ingest/ -v + + # Run with coverage + pytest tests/ingest/ --cov=src.ingest --cov-report=term-missing + + # Verify coverage >80% + ``` + +3. **Linting and Type Checking** + ```bash + # Format code + black src/ingest/ tests/ingest/ + + # Check linting + flake8 src/ingest/ tests/ingest/ + + # Type check + mypy src/ingest/ + ``` + +4. **Integration Test** + - Run client against real NVD API (with API key) + - Fetch CVEs from last 7 days + - Verify output is valid JSONL + - Verify checkpoint is saved + - Verify resume from checkpoint works + +## Dependencies + +### Python Packages +- `requests>=2.31.0` - HTTP client +- `requests-cache>=1.1.0` - Optional caching for development +- `python-dateutil>=2.8.2` - Date parsing + +### External Services +- NVD API v2.0 (https://nvd.nist.gov/developers) +- Optional: NVD API key for higher rate limits (request from NVD website) + +## Notes + +- **API Key**: Request from https://nvd.nist.gov/developers/request-an-api-key +- **Rate Limits**: Without key: 5 requests/30s, With key: 50 requests/30s +- **Data Volume**: NVD contains 200,000+ CVEs; initial sync will take time +- **Incremental Updates**: Most runs should be incremental (fetch last 24 hours) +- **Testing**: Use mocked responses for unit tests; avoid hitting real API in tests + +## References + +- NVD API 2.0 Documentation: https://nvd.nist.gov/developers/vulnerabilities +- CVE JSON Schema: https://csrc.nist.gov/schema/nvd/api/2.0/cve_api_json_2.0.schema +- CVSS Specification: https://www.first.org/cvss/specification-document + +## Related Tasks + +- **Task 020**: Positional alignment (consumes output from this task) +- **Task 060**: Notion sync (may use ingest logs for status updates) diff --git a/.copilot/tasks/020_alignment.md b/.copilot/tasks/020_alignment.md new file mode 100644 index 0000000..fb3318c --- /dev/null +++ b/.copilot/tasks/020_alignment.md @@ -0,0 +1,394 @@ +# Task 020: Positional Alignment + +## Goal + +Implement positional alignment methods to align vulnerability representations across different embedding spaces, enabling comparison of vulnerabilities from different sources or time periods. + +## Files to Create/Edit + +### New Files +- `src/alignment/__init__.py` - Package initialization with public API +- `src/alignment/procrustes.py` - Procrustes alignment implementation +- `src/alignment/cca.py` - Canonical Correlation Analysis implementation +- `src/alignment/base.py` - Base class for alignment methods +- `tests/alignment/__init__.py` - Test package initialization +- `tests/alignment/test_procrustes.py` - Tests for Procrustes alignment +- `tests/alignment/test_cca.py` - Tests for CCA +- `tests/alignment/test_integration.py` - Integration tests + +### Supporting Files +- `requirements.txt` - Add `numpy`, `scipy`, `scikit-learn` +- `src/alignment/metrics.py` - Alignment quality metrics (optional) + +## Requirements + +### Functional Requirements + +1. **Procrustes Alignment** + - Implement orthogonal Procrustes analysis for shape matching + - Find optimal rotation matrix R that minimizes ||R·Xa - Xb||² + - Support both orthogonal and scaled Procrustes + - Return transformation matrix and alignment quality (R²) + +2. **Canonical Correlation Analysis (CCA)** + - Implement CCA to find linear relationships between two embedding spaces + - Maximize correlation between projected spaces + - Support regularization for high-dimensional data + - Return projection matrices and canonical correlations + +3. **Alignment Quality Metrics** + - Calculate R² (coefficient of determination) for alignment quality + - Require alignment_R2 ≥ configured threshold (default 0.8) + - Calculate residual error after alignment + - Provide per-dimension alignment scores + +4. **Public API** + - Expose unified `align(X_a, X_b, method='procrustes') -> (R2, X_a2b)` function + - Support method selection: 'procrustes', 'cca', or 'auto' + - Return aligned embeddings and quality metrics + - Raise exception if alignment quality below threshold + +5. **Reference Space Management** + - Support loading pre-computed reference embeddings + - Enable aligning new data to reference space + - Cache transformation matrices for repeated alignment + +### Non-Functional Requirements + +1. **Performance** + - Handle embedding matrices up to 10,000 x 512 dimensions + - Alignment computation <1 second for typical input sizes + - Memory-efficient implementation for large matrices + +2. **Numerical Stability** + - Handle ill-conditioned matrices gracefully + - Use SVD-based methods for robustness + - Validate input data for NaN/Inf values + +3. **Configurability** + - Allow setting R² threshold via configuration + - Support different alignment methods + - Enable regularization parameters for CCA + +4. **Observability** + - Log alignment quality metrics + - Warn when alignment quality is marginal + - Provide diagnostic information for failed alignments + +## Implementation Guidance + +### Base Class + +```python +from abc import ABC, abstractmethod +import numpy as np +from typing import Tuple + +class AlignmentMethod(ABC): + """Base class for alignment methods""" + + @abstractmethod + def fit(self, X_source: np.ndarray, X_target: np.ndarray) -> 'AlignmentMethod': + """Fit alignment transformation from source to target""" + + @abstractmethod + def transform(self, X: np.ndarray) -> np.ndarray: + """Apply learned transformation to new data""" + + @abstractmethod + def fit_transform(self, X_source: np.ndarray, X_target: np.ndarray) -> Tuple[np.ndarray, float]: + """Fit and transform in one step, return aligned data and R²""" +``` + +### Procrustes Implementation + +```python +import numpy as np +from scipy.linalg import orthogonal_procrustes +from typing import Tuple + +class ProcrustesAlignment(AlignmentMethod): + """Orthogonal Procrustes alignment""" + + def __init__(self, scaling: bool = False): + self.scaling = scaling + self.rotation_matrix = None + self.scale_factor = None + + def fit(self, X_source: np.ndarray, X_target: np.ndarray) -> 'ProcrustesAlignment': + """ + Fit Procrustes alignment. + + Args: + X_source: Source embeddings (n_samples, n_features) + X_target: Target embeddings (n_samples, n_features) + + Returns: + self + """ + # Center the data + X_source_centered = X_source - X_source.mean(axis=0) + X_target_centered = X_target - X_target.mean(axis=0) + + # Compute optimal rotation matrix using SVD + self.rotation_matrix, _ = orthogonal_procrustes(X_source_centered, X_target_centered) + + if self.scaling: + # Compute optimal scale factor + X_aligned = X_source_centered @ self.rotation_matrix + self.scale_factor = np.trace(X_target_centered.T @ X_aligned) / np.trace(X_aligned.T @ X_aligned) + else: + self.scale_factor = 1.0 + + return self + + def transform(self, X: np.ndarray) -> np.ndarray: + """Apply learned transformation""" + if self.rotation_matrix is None: + raise ValueError("Must call fit() before transform()") + + X_centered = X - X.mean(axis=0) + X_aligned = self.scale_factor * (X_centered @ self.rotation_matrix) + return X_aligned + + def fit_transform(self, X_source: np.ndarray, X_target: np.ndarray) -> Tuple[np.ndarray, float]: + """Fit and transform, return aligned data and R²""" + self.fit(X_source, X_target) + X_aligned = self.transform(X_source) + + # Calculate R² + r2 = calculate_r2(X_aligned, X_target) + + return X_aligned, r2 +``` + +### CCA Implementation + +```python +from sklearn.cross_decomposition import CCA as SklearnCCA + +class CCAAlignment(AlignmentMethod): + """Canonical Correlation Analysis alignment""" + + def __init__(self, n_components: int = None, regularization: float = 0.0): + self.n_components = n_components + self.regularization = regularization + self.cca = None + + def fit(self, X_source: np.ndarray, X_target: np.ndarray) -> 'CCAAlignment': + """Fit CCA alignment""" + n_components = self.n_components or min(X_source.shape[1], X_target.shape[1]) + + # Use sklearn CCA with regularization + self.cca = SklearnCCA(n_components=n_components, max_iter=500) + self.cca.fit(X_source, X_target) + + return self + + def transform(self, X: np.ndarray) -> np.ndarray: + """Apply CCA transformation""" + if self.cca is None: + raise ValueError("Must call fit() before transform()") + + return self.cca.transform(X) + + def fit_transform(self, X_source: np.ndarray, X_target: np.ndarray) -> Tuple[np.ndarray, float]: + """Fit and transform, return aligned data and R²""" + self.fit(X_source, X_target) + X_aligned = self.transform(X_source) + Y_aligned = self.cca.transform(X_target) + + # Calculate R² using canonical correlations + r2 = calculate_r2(X_aligned, Y_aligned) + + return X_aligned, r2 +``` + +### Public API + +```python +def align( + X_source: np.ndarray, + X_target: np.ndarray, + method: str = 'procrustes', + threshold: float = 0.8, + **kwargs +) -> Tuple[float, np.ndarray]: + """ + Align source embeddings to target space. + + Args: + X_source: Source embeddings (n_samples, n_features) + X_target: Target embeddings (n_samples, n_features) + method: Alignment method ('procrustes', 'cca', 'auto') + threshold: Minimum acceptable R² (default: 0.8) + **kwargs: Additional arguments for alignment method + + Returns: + Tuple of (R², X_aligned) + + Raises: + AlignmentError: If alignment quality below threshold + """ + if method == 'procrustes': + aligner = ProcrustesAlignment(**kwargs) + elif method == 'cca': + aligner = CCAAlignment(**kwargs) + elif method == 'auto': + # Try Procrustes first, fall back to CCA if needed + aligner = ProcrustesAlignment(**kwargs) + else: + raise ValueError(f"Unknown method: {method}") + + X_aligned, r2 = aligner.fit_transform(X_source, X_target) + + if r2 < threshold: + raise AlignmentError( + f"Alignment quality R²={r2:.3f} below threshold {threshold:.3f}" + ) + + return r2, X_aligned +``` + +### Metrics + +```python +def calculate_r2(X_pred: np.ndarray, X_true: np.ndarray) -> float: + """ + Calculate R² (coefficient of determination). + + R² = 1 - (SS_res / SS_tot) + where SS_res = sum of squared residuals + SS_tot = total sum of squares + """ + # Center the data + X_true_centered = X_true - X_true.mean(axis=0) + + # Calculate residuals + residuals = X_pred - X_true + + # Sum of squared residuals + ss_res = np.sum(residuals ** 2) + + # Total sum of squares + ss_tot = np.sum(X_true_centered ** 2) + + # R² + r2 = 1 - (ss_res / ss_tot) + + return r2 +``` + +## Acceptance Criteria + +### Implementation +- [ ] `ProcrustesAlignment` class implemented with fit/transform/fit_transform +- [ ] `CCAAlignment` class implemented with fit/transform/fit_transform +- [ ] `align()` public API function working with all methods +- [ ] R² calculation implemented correctly +- [ ] Alignment quality threshold checking working +- [ ] Input validation for NaN/Inf values +- [ ] Proper error handling and exceptions + +### Testing +- [ ] Unit tests for Procrustes alignment with known transformations +- [ ] Unit tests for CCA alignment +- [ ] Test R² calculation with simple examples +- [ ] Test threshold enforcement (should raise exception) +- [ ] Test with various matrix sizes and dimensions +- [ ] Test numerical stability (ill-conditioned matrices) +- [ ] Integration test aligning real embedding data +- [ ] Tests achieve >80% code coverage + +### Documentation +- [ ] Docstrings for all public classes and methods +- [ ] Usage examples in module docstring +- [ ] Mathematical background documented +- [ ] Configuration options documented +- [ ] Error conditions documented + +### Quality +- [ ] Code passes flake8 linting +- [ ] Code passes black formatting +- [ ] Code passes mypy type checking +- [ ] No hardcoded thresholds (use configuration) +- [ ] All dependencies added to requirements.txt + +## Validation Steps + +1. **Unit Tests** + ```bash + pytest tests/alignment/ -v --cov=src.alignment --cov-report=term-missing + ``` + +2. **Manual Testing** + ```python + from src.alignment import align + import numpy as np + + # Create sample embeddings + X_source = np.random.randn(100, 128) + X_target = np.random.randn(100, 128) + + # Align using Procrustes + r2, X_aligned = align(X_source, X_target, method='procrustes') + print(f"Procrustes R²: {r2:.3f}") + + # Align using CCA + r2, X_aligned = align(X_source, X_target, method='cca') + print(f"CCA R²: {r2:.3f}") + ``` + +3. **Integration Test** + - Generate synthetic embeddings with known transformation + - Apply Procrustes alignment + - Verify recovered transformation matches original + - Verify R² > 0.95 for perfectly aligned data + +4. **Quality Checks** + ```bash + black src/alignment/ tests/alignment/ + flake8 src/alignment/ tests/alignment/ + mypy src/alignment/ + ``` + +## Dependencies + +### Python Packages +- `numpy>=1.24.0` - Array operations +- `scipy>=1.10.0` - Procrustes and linear algebra +- `scikit-learn>=1.3.0` - CCA implementation + +### Internal Dependencies +- None (this is a foundational module) + +## Configuration + +Add to config file or environment: +```yaml +alignment: + default_method: procrustes + r2_threshold: 0.8 + cca_n_components: null # auto-detect + cca_regularization: 0.0 +``` + +## Notes + +- **Procrustes vs CCA**: Procrustes is faster and works well for similar spaces; CCA better for finding latent correlations +- **R² Threshold**: 0.8 is a reasonable default; adjust based on use case +- **Dimensionality**: CCA requires n_samples > n_features; use regularization if needed +- **Centering**: Both methods require centering data for best results + +## References + +- Goodall, C. (1991). "Procrustes methods in the statistical analysis of shape" +- Hotelling, H. (1936). "Relations between two sets of variates" +- Scipy orthogonal_procrustes: https://docs.scipy.org/doc/scipy/reference/generated/scipy.linalg.orthogonal_procrustes.html +- Scikit-learn CCA: https://scikit-learn.org/stable/modules/generated/sklearn.cross_decomposition.CCA.html + +## Related Tasks + +- **Task 010**: NVD ingestion (provides CVE data to embed) +- **Task 030**: Arbiter (consumes aligned embeddings) +- **Task 040**: Refractors (may use alignment metrics for shift detection) diff --git a/.copilot/tasks/030_arbiter.md b/.copilot/tasks/030_arbiter.md new file mode 100644 index 0000000..9810ac4 --- /dev/null +++ b/.copilot/tasks/030_arbiter.md @@ -0,0 +1,94 @@ +# Task 030: Stacked Arbiter with Pareto Knee Detection + +## Goal + +Implement a stacked ensemble learning system (arbiter) for vulnerability severity prediction with multi-objective optimization and Pareto knee detection. + +## Files to Create/Edit + +### New Files +- `src/models/__init__.py` - Package initialization +- `src/models/arbiter.py` - Stacked arbiter implementation +- `src/models/base_learners.py` - Base learner configurations +- `src/models/pareto.py` - Pareto frontier and knee detection +- `tests/models/__init__.py` - Test package initialization +- `tests/models/test_arbiter.py` - Tests for arbiter +- `tests/models/test_pareto.py` - Tests for Pareto detection + +### Supporting Files +- `requirements.txt` - Add `scikit-learn`, `xgboost`, `lightgbm` (optional) +- `config/models.yaml` - Model hyperparameters configuration + +## Requirements + +### Functional Requirements + +1. **Stacked Ensemble Architecture** + - Multiple base learners (Level 0): RandomForest, GradientBoosting, etc. + - Meta-learner (Level 1): LogisticRegression or another model + - Support for classification (severity levels) and regression (CVSS scores) + - Out-of-fold predictions for meta-learner training + +2. **Pareto Optimization** + - Multi-objective optimization (e.g., accuracy vs inference time) + - Calculate Pareto frontier across model configurations + - Detect Pareto knee (optimal trade-off point) + - Support custom objective functions + +3. **Predictions and Confidence** + - Severity predictions with confidence scores + - Feature importance from ensemble + - Support for probability estimates + - Calibrated predictions + +4. **Model Management** + - Save and load trained models + - Version control for models + - Support for incremental learning + +### Acceptance Criteria +- [ ] Stacked arbiter with 3+ base learners implemented +- [ ] Pareto frontier calculation working +- [ ] Knee detection identifying optimal configuration +- [ ] Predictions with confidence intervals +- [ ] Feature importance extraction +- [ ] Model serialization (save/load) +- [ ] Tests achieve >80% coverage +- [ ] Docstrings for all public APIs + +## Implementation Guidance + +```python +from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import cross_val_predict +import numpy as np + +class StackedArbiter: + """Stacked ensemble for vulnerability severity prediction""" + + def __init__(self, base_learners=None, meta_learner=None): + self.base_learners = base_learners or self._default_base_learners() + self.meta_learner = meta_learner or LogisticRegression() + + def fit(self, X, y): + """Train ensemble on aligned embeddings""" + # Train base learners and generate meta-features + meta_features = self._train_base_learners(X, y) + # Train meta-learner on base predictions + self.meta_learner.fit(meta_features, y) + + def predict(self, X): + """Predict severity with confidence""" + base_predictions = self._get_base_predictions(X) + return self.meta_learner.predict(base_predictions) + + def predict_proba(self, X): + """Get probability estimates""" + base_predictions = self._get_base_predictions(X) + return self.meta_learner.predict_proba(base_predictions) +``` + +## References +- Wolpert, D. H. (1992). "Stacked generalization" +- Das, I. (1999). "On characterizing the 'knee' of the Pareto curve" diff --git a/.copilot/tasks/040_refractors.md b/.copilot/tasks/040_refractors.md new file mode 100644 index 0000000..8931c4e --- /dev/null +++ b/.copilot/tasks/040_refractors.md @@ -0,0 +1,116 @@ +# Task 040: Epsilon-Refractors for Distributional Shift Detection + +## Goal + +Implement epsilon-refractor mechanisms to detect distributional shifts in vulnerability patterns, enabling early warning of emerging threat landscapes. + +## Files to Create/Edit + +### New Files +- `src/refractors/__init__.py` - Package initialization +- `src/refractors/epsilon.py` - Epsilon-refractor implementation +- `src/refractors/shifts.py` - Shift detection algorithms +- `src/refractors/metrics.py` - Divergence metrics (KL, JS, etc.) +- `tests/refractors/__init__.py` - Test package initialization +- `tests/refractors/test_epsilon.py` - Tests for epsilon-refractor +- `tests/refractors/test_shifts.py` - Tests for shift detection + +### Supporting Files +- `requirements.txt` - Add `scipy` for statistical functions +- `config/thresholds.yaml` - Alert thresholds configuration + +## Requirements + +### Functional Requirements + +1. **Epsilon-Divergence Calculation** + - Calculate ε-divergence between distributions + - Support multiple divergence metrics: KL, JS, Wasserstein + - Configurable epsilon threshold (default: 0.05) + - Per-feature and global divergence measures + +2. **Shift Detection** + - Detect concept drift (changes in P(Y|X)) + - Detect data drift (changes in P(X)) + - Sliding window approach for temporal monitoring + - Statistical significance testing + +3. **Alert Generation** + - Trigger alerts when ε exceeds threshold + - Classify shift severity (low, medium, high) + - Identify which features/dimensions shifted most + - Provide actionable diagnostics + +4. **Reference Distribution Management** + - Store reference distributions for comparison + - Update reference distributions periodically + - Support multiple reference periods + +### Acceptance Criteria +- [ ] Epsilon-refractor calculates divergence correctly +- [ ] Shift detection working with synthetic drift +- [ ] Alert thresholds configurable +- [ ] Per-feature drift attribution +- [ ] Statistical significance tests implemented +- [ ] Tests achieve >80% coverage +- [ ] Docstrings for all public APIs + +## Implementation Guidance + +```python +import numpy as np +from scipy.stats import entropy, wasserstein_distance + +class EpsilonRefractor: + """Detect distributional shifts using epsilon-divergence""" + + def __init__(self, threshold: float = 0.05, metric: str = 'kl'): + self.threshold = threshold + self.metric = metric + self.reference_dist = None + + def fit(self, X_reference: np.ndarray): + """Establish reference distribution""" + self.reference_dist = self._estimate_distribution(X_reference) + + def detect(self, X_current: np.ndarray) -> Dict: + """Detect shifts in current data""" + current_dist = self._estimate_distribution(X_current) + epsilon = self._calculate_divergence(self.reference_dist, current_dist) + + alert = epsilon > self.threshold + + return { + 'epsilon': epsilon, + 'alert': alert, + 'severity': self._classify_severity(epsilon), + 'shifted_features': self._identify_shifted_features(X_current) + } + + def _calculate_divergence(self, p, q): + """Calculate divergence between distributions""" + if self.metric == 'kl': + return entropy(p, q) + elif self.metric == 'js': + m = 0.5 * (p + q) + return 0.5 * entropy(p, m) + 0.5 * entropy(q, m) + elif self.metric == 'wasserstein': + return wasserstein_distance(p, q) +``` + +## Configuration + +```yaml +refractors: + epsilon_threshold: 0.05 + divergence_metric: kl # or js, wasserstein + window_size: 1000 # samples for drift detection + alert_severity: + low: 0.05 + medium: 0.10 + high: 0.20 +``` + +## References +- Dasu, T. et al. (2006). "An information-theoretic approach to detecting changes in multi-dimensional data streams" +- Rabanser, S. et al. (2019). "Failing Loudly: An Empirical Study of Methods for Detecting Dataset Shift" diff --git a/.copilot/tasks/050_evidence.md b/.copilot/tasks/050_evidence.md new file mode 100644 index 0000000..1a27a56 --- /dev/null +++ b/.copilot/tasks/050_evidence.md @@ -0,0 +1,150 @@ +# Task 050: Bayesian Evidence Calculation + +## Goal + +Implement Bayesian evidence calculation for model comparison and vulnerability prioritization using BIC, WAIC, Bayes factors, and Jeffreys scale interpretation. + +## Files to Create/Edit + +### New Files +- `src/evaluation/__init__.py` - Package initialization +- `src/evaluation/evidence.py` - Bayesian evidence implementation +- `src/evaluation/bayes_factor.py` - Bayes factor calculation +- `src/evaluation/jeffreys.py` - Jeffreys scale interpretation +- `tests/evaluation/__init__.py` - Test package initialization +- `tests/evaluation/test_evidence.py` - Tests for evidence calculation +- `tests/evaluation/test_bayes_factor.py` - Tests for Bayes factors + +### Supporting Files +- `requirements.txt` - Add `scipy` for statistical functions +- `config/evidence.yaml` - Evidence calculation configuration + +## Requirements + +### Functional Requirements + +1. **BIC (Bayesian Information Criterion)** + - Calculate BIC = -2*log_likelihood + k*log(n) + - Use for model selection and comparison + - Lower BIC indicates better model fit + +2. **WAIC (Widely Applicable Information Criterion)** + - Calculate WAIC for hierarchical models + - Use as default method (more robust than BIC) + - Provide pointwise WAIC for diagnostic purposes + +3. **Bayes Factor** + - Calculate Bayes factor for model comparison + - BF = p(Data|Model1) / p(Data|Model2) + - Support log-space calculations for numerical stability + +4. **Jeffreys Scale** + - Interpret Bayes factors using Jeffreys scale + - Classify evidence strength: weak, moderate, strong, decisive + - Provide human-readable interpretations + +5. **Priority Scoring** + - Combine evidence scores for vulnerability prioritization + - Normalize scores to [0, 1] range + - Support custom weighting schemes + +### Acceptance Criteria +- [ ] BIC calculation implemented correctly +- [ ] WAIC calculation working (default method) +- [ ] Bayes factor computation in log-space +- [ ] Jeffreys scale interpretation +- [ ] Priority score generation +- [ ] Tests with known statistical examples +- [ ] Tests achieve >80% coverage +- [ ] Docstrings for all public APIs + +## Implementation Guidance + +```python +import numpy as np +from scipy.special import logsumexp + +def calculate_bic(log_likelihood: float, n_params: int, n_samples: int) -> float: + """Calculate Bayesian Information Criterion""" + return -2 * log_likelihood + n_params * np.log(n_samples) + +def calculate_waic(log_likelihoods: np.ndarray) -> float: + """Calculate Widely Applicable Information Criterion""" + # WAIC = -2 * (lppd - p_waic) + # lppd: log pointwise predictive density + # p_waic: effective number of parameters + + lppd = np.sum(logsumexp(log_likelihoods, axis=0) - np.log(log_likelihoods.shape[0])) + p_waic = np.sum(np.var(log_likelihoods, axis=0)) + + waic = -2 * (lppd - p_waic) + return waic + +def calculate_bayes_factor( + log_evidence_m1: float, + log_evidence_m2: float +) -> float: + """Calculate Bayes factor for model comparison""" + return np.exp(log_evidence_m1 - log_evidence_m2) + +def interpret_bayes_factor(bf: float) -> str: + """Interpret Bayes factor using Jeffreys scale""" + log_bf = np.log10(bf) + + if log_bf < 0.5: + return "Weak evidence" + elif log_bf < 1.0: + return "Moderate evidence" + elif log_bf < 1.5: + return "Strong evidence" + elif log_bf < 2.0: + return "Very strong evidence" + else: + return "Decisive evidence" + +class EvidenceCalculator: + """Calculate Bayesian evidence for CVE prioritization""" + + def __init__(self, method: str = 'waic'): + self.method = method + + def calculate_evidence( + self, + predictions: np.ndarray, + ground_truth: np.ndarray, + **kwargs + ) -> Dict[str, float]: + """ + Calculate evidence scores for predictions. + + Returns dict with: + - evidence_score: primary score (BIC or WAIC) + - log_likelihood: model fit + - complexity_penalty: model complexity term + """ + if self.method == 'bic': + return self._calculate_bic_evidence(predictions, ground_truth, **kwargs) + elif self.method == 'waic': + return self._calculate_waic_evidence(predictions, ground_truth, **kwargs) + else: + raise ValueError(f"Unknown method: {self.method}") +``` + +## Configuration + +```yaml +evidence: + default_method: waic # or bic + jeffreys_scale: + weak: 1.0 + moderate: 3.2 + strong: 10.0 + very_strong: 31.6 + decisive: 100.0 +``` + +## References +- Gelman, A. et al. (2014). "Bayesian Data Analysis" (3rd ed.) +- Watanabe, S. (2010). "Asymptotic Equivalence of Bayes Cross Validation and WAIC" +- Jeffreys, H. (1961). "Theory of Probability" (3rd ed.) +- Kass, R. E. & Raftery, A. E. (1995). "Bayes Factors" diff --git a/.copilot/tasks/060_notion_sync.md b/.copilot/tasks/060_notion_sync.md new file mode 100644 index 0000000..468bb26 --- /dev/null +++ b/.copilot/tasks/060_notion_sync.md @@ -0,0 +1,156 @@ +# Task 060: Notion Synchronization + +## Goal + +Implement synchronization between the repository prompts/documentation and Notion for team collaboration and visibility. + +## Files to Create/Edit + +### New Files +- `scripts/register_prompt_notion.sh` - Bash script to sync prompts to Notion +- `scripts/notion_sync.py` - Python implementation for Notion API integration +- `.github/workflows/notion-sync.yml` - GitHub Actions workflow for automated sync +- `tests/scripts/test_notion_sync.py` - Tests for Notion sync functionality + +### Supporting Files +- `requirements.txt` - Add `notion-client` package +- `.env.example` - Example environment variables + +## Requirements + +### Functional Requirements + +1. **Prompt Synchronization** + - Sync `prompts/legendary_lidlift_v14.md` to Notion page + - Sync capsule summaries to Notion database + - Preserve markdown formatting + - Include metadata (version, last updated) + +2. **GitHub Actions Integration** + - Trigger sync on push to main branch + - Manual workflow dispatch option + - Only sync when relevant files change + - Report sync status in workflow + +3. **Notion API Integration** + - Use Notion API v2 + - Handle authentication securely via secrets + - Update existing pages or create new ones + - Handle rate limiting gracefully + +4. **Error Handling** + - Graceful failures (don't block main workflow) + - Retry transient errors + - Log sync status and errors + - Notify on persistent failures + +### Acceptance Criteria +- [ ] `register_prompt_notion.sh` script working +- [ ] Python Notion client implemented +- [ ] GitHub Actions workflow created +- [ ] Sync triggered on relevant file changes +- [ ] Markdown formatting preserved in Notion +- [ ] Error handling and retries implemented +- [ ] Tests for Notion sync logic +- [ ] Documentation for setup + +## Implementation Guidance + +```bash +#!/bin/bash +# scripts/register_prompt_notion.sh + +set -euo pipefail + +NOTION_API_KEY="${NOTION_API_KEY:-}" +NOTION_PAGE_ID="${NOTION_PAGE_ID:-}" + +if [ -z "$NOTION_API_KEY" ] || [ -z "$NOTION_PAGE_ID" ]; then + echo "Error: NOTION_API_KEY and NOTION_PAGE_ID must be set" + exit 1 +fi + +# Call Python script for actual sync +python scripts/notion_sync.py \ + --api-key "$NOTION_API_KEY" \ + --page-id "$NOTION_PAGE_ID" \ + --file "prompts/legendary_lidlift_v14.md" +``` + +```python +# scripts/notion_sync.py +from notion_client import Client +import argparse +import sys + +def sync_to_notion(api_key: str, page_id: str, markdown_file: str): + """Sync markdown file to Notion page""" + notion = Client(auth=api_key) + + with open(markdown_file, 'r') as f: + content = f.read() + + # Convert markdown to Notion blocks + blocks = markdown_to_notion_blocks(content) + + # Update Notion page + notion.blocks.children.append(page_id, children=blocks) + + print(f"Successfully synced {markdown_file} to Notion") + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--api-key", required=True) + parser.add_argument("--page-id", required=True) + parser.add_argument("--file", required=True) + args = parser.parse_args() + + sync_to_notion(args.api_key, args.page_id, args.file) +``` + +## GitHub Actions Workflow + +```yaml +# .github/workflows/notion-sync.yml +name: Notion Sync + +on: + push: + branches: [main] + paths: + - 'prompts/**' + - 'capsules/**' + workflow_dispatch: + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install notion-client + + - name: Sync to Notion + env: + NOTION_API_KEY: ${{ secrets.NOTION_API_KEY }} + NOTION_PAGE_ID: ${{ secrets.NOTION_PAGE_ID }} + run: | + bash scripts/register_prompt_notion.sh +``` + +## Configuration + +Secrets to add in GitHub repository settings: +- `NOTION_API_KEY`: Notion integration token +- `NOTION_PAGE_ID`: Target Notion page ID + +## References +- Notion API Documentation: https://developers.notion.com/ +- notion-client Python SDK: https://github.com/ramnes/notion-sdk-py diff --git a/.copilot/tasks/070_capsules_publish.md b/.copilot/tasks/070_capsules_publish.md new file mode 100644 index 0000000..24445b4 --- /dev/null +++ b/.copilot/tasks/070_capsules_publish.md @@ -0,0 +1,219 @@ +# Task 070: Capsules Publishing + +## Goal + +Implement automated publishing of capsule configuration files to a cloud storage bucket or domain when new versions are tagged. + +## Files to Create/Edit + +### New Files +- `.github/workflows/publish-capsules.yml` - GitHub Actions workflow for publishing +- `scripts/publish_capsules.sh` - Script to publish capsules +- `scripts/validate_capsules.py` - Validation script for capsules +- `tests/scripts/test_validate_capsules.py` - Tests for validation + +### Supporting Files +- `.env.example` - Add publishing configuration examples + +## Requirements + +### Functional Requirements + +1. **Automated Publishing** + - Trigger on Git tag creation (e.g., v1.0.0, v1.4.0) + - Publish capsules to GCS bucket or CDN + - Set appropriate access permissions (public or authenticated) + - Generate versioned URLs for capsules + +2. **Validation** + - Validate JSON schema before publishing + - Check for required fields + - Verify version consistency + - Run automated tests + +3. **Versioning** + - Publish versioned capsules (e.g., lidlift-v1.4.0.json) + - Update "latest" symlinks/aliases + - Maintain version history + - Support rollback to previous versions + +4. **Notification** + - Post publication status to workflow + - Generate changelog if applicable + - Update documentation with new URLs + +### Acceptance Criteria +- [ ] GitHub Actions workflow for publishing +- [ ] Triggered on tag creation +- [ ] Capsule validation before publishing +- [ ] Published to cloud storage (GCS or equivalent) +- [ ] Versioned URLs generated +- [ ] Latest aliases updated +- [ ] Tests for validation logic +- [ ] Documentation updated + +## Implementation Guidance + +```yaml +# .github/workflows/publish-capsules.yml +name: Publish Capsules + +on: + push: + tags: + - 'v*' + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # For GCP Workload Identity + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Validate capsules + run: | + python scripts/validate_capsules.py capsules/*.json + + - name: Authenticate to GCP + uses: google-github-actions/auth@v1 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v1 + + - name: Publish capsules + env: + GCS_BUCKET: ${{ secrets.GCS_CAPSULES_BUCKET }} + VERSION: ${{ github.ref_name }} + run: | + bash scripts/publish_capsules.sh + + - name: Update latest + run: | + # Create/update latest symlinks + echo "Published capsules version ${VERSION}" +``` + +```bash +#!/bin/bash +# scripts/publish_capsules.sh + +set -euo pipefail + +GCS_BUCKET="${GCS_BUCKET:-}" +VERSION="${VERSION:-latest}" + +if [ -z "$GCS_BUCKET" ]; then + echo "Error: GCS_BUCKET must be set" + exit 1 +fi + +# Remove 'v' prefix from version if present +VERSION_CLEAN="${VERSION#v}" + +echo "Publishing capsules version $VERSION_CLEAN to gs://$GCS_BUCKET/" + +# Publish each capsule with version +for capsule in capsules/*.json; do + filename=$(basename "$capsule" .json) + + # Upload versioned file + gsutil -h "Content-Type:application/json" \ + -h "Cache-Control:public, max-age=3600" \ + cp "$capsule" "gs://$GCS_BUCKET/${filename}-${VERSION_CLEAN}.json" + + # Update latest + gsutil -h "Content-Type:application/json" \ + -h "Cache-Control:public, max-age=300" \ + cp "$capsule" "gs://$GCS_BUCKET/${filename}-latest.json" + + echo "Published $filename version $VERSION_CLEAN" +done + +echo "All capsules published successfully" +``` + +```python +# scripts/validate_capsules.py +import json +import sys +from pathlib import Path +from typing import List + +def validate_capsule(filepath: Path) -> List[str]: + """Validate a capsule JSON file""" + errors = [] + + try: + with open(filepath) as f: + data = json.load(f) + except json.JSONDecodeError as e: + return [f"Invalid JSON: {e}"] + + # Check required fields + required_fields = ['name', 'version', 'type', 'description'] + for field in required_fields: + if field not in data: + errors.append(f"Missing required field: {field}") + + # Validate version format + if 'version' in data: + version = data['version'] + if not isinstance(version, str) or not version: + errors.append(f"Invalid version format: {version}") + + return errors + +def main(capsule_files: List[str]): + """Validate all capsule files""" + all_valid = True + + for filepath in capsule_files: + path = Path(filepath) + print(f"Validating {path.name}...") + + errors = validate_capsule(path) + if errors: + all_valid = False + print(f" ✗ Validation failed:") + for error in errors: + print(f" - {error}") + else: + print(f" ✓ Valid") + + if not all_valid: + print("\nValidation failed!") + sys.exit(1) + else: + print("\nAll capsules valid!") + sys.exit(0) + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python validate_capsules.py [capsule2.json ...]") + sys.exit(1) + + main(sys.argv[1:]) +``` + +## Configuration + +Secrets and variables to configure: +- `GCS_CAPSULES_BUCKET`: GCS bucket name for capsules +- `GCP_WORKLOAD_IDENTITY_PROVIDER`: GCP Workload Identity Provider +- `GCP_SERVICE_ACCOUNT`: Service account for publishing + +## References +- Google Cloud Storage: https://cloud.google.com/storage/docs +- GitHub Actions with GCP: https://github.com/google-github-actions/auth diff --git a/.copilot/tasks/080_cuda_support.md b/.copilot/tasks/080_cuda_support.md new file mode 100644 index 0000000..6764a49 --- /dev/null +++ b/.copilot/tasks/080_cuda_support.md @@ -0,0 +1,250 @@ +# Task 080: CUDA/GPU Support + +## Goal + +Add optional CUDA support for GPU-accelerated operations in compute-intensive pipeline stages (alignment, arbiter training, embedding generation). + +## Files to Create/Edit + +### New Files +- `src/cuda/__init__.py` - CUDA utilities package +- `src/cuda/device_manager.py` - GPU device management and selection +- `src/cuda/accelerated_ops.py` - GPU-accelerated operations +- `tests/cuda/test_device_manager.py` - Tests for device management +- `docker/Dockerfile.cuda` - Docker image with CUDA support +- `.github/workflows/cuda-tests.yml` - CI workflow for CUDA tests + +### Files to Modify +- `src/alignment/procrustes.py` - Add GPU acceleration option +- `src/models/arbiter.py` - Add GPU training support +- `requirements.txt` - Add optional CUDA dependencies +- `README.md` - Add CUDA setup instructions + +## Requirements + +### Functional Requirements + +1. **Device Management** + - Auto-detect available GPUs + - Select GPU based on CUDA_VISIBLE_DEVICES + - Fallback to CPU if CUDA unavailable + - Memory management and cleanup + +2. **Accelerated Operations** + - GPU-accelerated matrix operations for alignment + - GPU training for ensemble models + - Batch processing on GPU + - Efficient data transfer between CPU/GPU + +3. **Optional Dependency** + - CUDA is optional, not required + - Graceful degradation to CPU + - Clear error messages if CUDA unavailable + - No performance impact when using CPU + +4. **Configuration** + - Environment variable for GPU selection + - Memory limit configuration + - Batch size optimization for GPU + - Mixed precision training option + +### Non-Functional Requirements + +1. **Performance** + - ≥3x speedup for large matrix operations + - ≥5x speedup for model training + - Efficient memory usage (avoid OOM) + - Minimize CPU-GPU transfer overhead + +2. **Compatibility** + - Support CUDA 11.8+ + - Compatible with common GPU types (V100, A100, T4) + - Works with both single and multi-GPU setups + - Docker image with CUDA runtime + +### Acceptance Criteria +- [ ] Device manager detects and selects GPUs +- [ ] Fallback to CPU works correctly +- [ ] GPU-accelerated alignment implemented +- [ ] GPU model training working +- [ ] Docker image with CUDA support +- [ ] CI tests for CUDA (on GPU runners) +- [ ] Documentation for CUDA setup +- [ ] Performance benchmarks documented + +## Implementation Guidance + +```python +# src/cuda/device_manager.py +import os +from typing import Optional +import logging + +logger = logging.getLogger(__name__) + +try: + import torch + CUDA_AVAILABLE = torch.cuda.is_available() +except ImportError: + CUDA_AVAILABLE = False + logger.info("PyTorch not installed, CUDA support disabled") + +class DeviceManager: + """Manage GPU/CPU device selection""" + + def __init__(self): + self.device = self._select_device() + + def _select_device(self) -> str: + """Select best available device""" + if not CUDA_AVAILABLE: + logger.info("CUDA not available, using CPU") + return "cpu" + + # Check CUDA_VISIBLE_DEVICES + visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "") + if visible_devices == "": + # Use first available GPU + device = "cuda:0" + else: + device = f"cuda:{visible_devices.split(',')[0]}" + + logger.info(f"Using device: {device}") + return device + + def get_device(self) -> str: + """Get current device""" + return self.device + + def is_cuda(self) -> bool: + """Check if using CUDA""" + return self.device.startswith("cuda") + + def get_memory_info(self) -> Optional[dict]: + """Get GPU memory information""" + if not self.is_cuda(): + return None + + import torch + device_id = int(self.device.split(":")[-1]) + + return { + "allocated": torch.cuda.memory_allocated(device_id), + "reserved": torch.cuda.memory_reserved(device_id), + "total": torch.cuda.get_device_properties(device_id).total_memory + } +``` + +```python +# Modified src/alignment/procrustes.py +class ProcrustesAlignment(AlignmentMethod): + def __init__(self, scaling: bool = False, use_cuda: bool = False): + self.scaling = scaling + self.use_cuda = use_cuda and CUDA_AVAILABLE + self.device_manager = DeviceManager() if self.use_cuda else None + + def fit(self, X_source: np.ndarray, X_target: np.ndarray): + if self.use_cuda: + return self._fit_cuda(X_source, X_target) + else: + return self._fit_cpu(X_source, X_target) + + def _fit_cuda(self, X_source, X_target): + """GPU-accelerated Procrustes alignment""" + import torch + device = self.device_manager.get_device() + + # Convert to torch tensors on GPU + X_src = torch.from_numpy(X_source).float().to(device) + X_tgt = torch.from_numpy(X_target).float().to(device) + + # Center on GPU + X_src_centered = X_src - X_src.mean(dim=0) + X_tgt_centered = X_tgt - X_tgt.mean(dim=0) + + # SVD on GPU + U, _, Vt = torch.linalg.svd(X_src_centered.T @ X_tgt_centered) + self.rotation_matrix = (U @ Vt).cpu().numpy() + + return self +``` + +## Docker Configuration + +```dockerfile +# docker/Dockerfile.cuda +FROM nvidia/cuda:11.8.0-runtime-ubuntu22.04 + +RUN apt-get update && apt-get install -y python3.11 python3-pip + +WORKDIR /app + +COPY requirements.txt . +RUN pip install -r requirements.txt +RUN pip install torch --index-url https://download.pytorch.org/whl/cu118 + +COPY . . + +CMD ["python3", "-m", "src.main"] +``` + +## GitHub Actions Workflow + +```yaml +# .github/workflows/cuda-tests.yml +name: CUDA Tests + +on: + push: + branches: [main] + pull_request: + +jobs: + gpu-tests: + # NOTE: Update with actual large runner label from GitHub docs + # Example: ubuntu-22.04-16core-gpu or similar + runs-on: ubuntu-latest # TODO: Replace with GPU runner label + + steps: + - uses: actions/checkout@v4 + + - name: Check CUDA availability + run: nvidia-smi + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install torch --index-url https://download.pytorch.org/whl/cu118 + + - name: Run GPU tests + env: + CUDA_VISIBLE_DEVICES: 0 + run: | + pytest tests/cuda/ -v -m gpu +``` + +## Configuration + +```yaml +# Environment variables +CUDA_VISIBLE_DEVICES: "0" # GPU device ID +CUDA_MEMORY_LIMIT: "8GB" # Optional memory limit +USE_CUDA: "true" # Enable CUDA acceleration +``` + +## Performance Benchmarks + +Document expected speedups: +- Matrix operations (10K x 512): 3-5x faster +- Model training (ensemble): 5-10x faster +- Batch alignment (1000 CVEs): 4-6x faster + +## References +- NVIDIA CUDA Toolkit: https://developer.nvidia.com/cuda-toolkit +- PyTorch CUDA: https://pytorch.org/docs/stable/cuda.html +- cuBLAS for linear algebra: https://docs.nvidia.com/cuda/cublas/ diff --git a/.copilot/tasks/090_bridge.md b/.copilot/tasks/090_bridge.md new file mode 100644 index 0000000..d21e150 --- /dev/null +++ b/.copilot/tasks/090_bridge.md @@ -0,0 +1,363 @@ +# Task 090: Webhook + Argo Bridge + +## Goal + +Implement a webhook receiver and Argo Workflows integration to enable event-driven CVE analysis pipeline execution. + +## Files to Create/Edit + +### New Files +- `src/bridge/__init__.py` - Bridge package initialization +- `src/bridge/webhook_server.py` - Flask/FastAPI webhook receiver +- `src/bridge/argo_client.py` - Argo Workflows client +- `src/bridge/event_handlers.py` - Event processing logic +- `k8s/webhook-deployment.yaml` - Kubernetes deployment for webhook server +- `k8s/webhook-service.yaml` - Kubernetes service +- `argo/cve-analysis-workflow.yaml` - Argo workflow definition +- `tests/bridge/test_webhook.py` - Tests for webhook server +- `tests/bridge/test_argo_client.py` - Tests for Argo client + +### Supporting Files +- `requirements.txt` - Add `fastapi`, `uvicorn`, `argo-workflows` +- `config/bridge.yaml` - Bridge configuration + +## Requirements + +### Functional Requirements + +1. **Webhook Server** + - HTTP server to receive webhook events + - Support GitHub webhooks (push, release) + - Support custom CVE alert webhooks + - Validate webhook signatures (HMAC) + - Health check endpoint + +2. **Argo Workflows Integration** + - Submit workflows to Argo programmatically + - Pass parameters (CVE IDs, date ranges, etc.) + - Monitor workflow status + - Retrieve workflow results + - Handle workflow failures and retries + +3. **Event Processing** + - Parse incoming webhook payloads + - Extract relevant parameters for pipeline + - Queue events if pipeline is busy + - Deduplicate duplicate events + - Rate limiting and throttling + +4. **Pipeline Orchestration** + - Trigger NVD ingest on schedule or event + - Run full analysis pipeline via Argo + - Support partial pipeline execution + - Pass artifacts between workflow steps + - Store results in persistent storage + +### Non-Functional Requirements + +1. **Reliability** + - Handle transient Argo API failures + - Queue events for retry on failure + - Idempotent workflow submissions + - Graceful degradation + +2. **Security** + - Authenticate webhook sources + - Secure Argo API credentials + - Rate limit to prevent DoS + - Audit log all events + +3. **Performance** + - Handle 100+ webhook events per hour + - Low latency (<500ms) webhook response + - Efficient workflow submission + - Minimal resource overhead + +### Acceptance Criteria +- [ ] Webhook server receiving and validating events +- [ ] Argo client submitting workflows +- [ ] Full pipeline workflow defined in Argo +- [ ] Event handlers for different webhook types +- [ ] Kubernetes manifests for deployment +- [ ] Health check and metrics endpoints +- [ ] Tests for webhook and Argo client +- [ ] Documentation for setup + +## Implementation Guidance + +```python +# src/bridge/webhook_server.py +from fastapi import FastAPI, Request, HTTPException, Header +import hmac +import hashlib +from typing import Optional + +app = FastAPI(title="CVE Analysis Bridge") + +WEBHOOK_SECRET = os.environ.get("WEBHOOK_SECRET", "") + +@app.post("/webhook/github") +async def github_webhook( + request: Request, + x_hub_signature_256: Optional[str] = Header(None) +): + """Handle GitHub webhook events""" + body = await request.body() + + # Validate signature + if not validate_github_signature(body, x_hub_signature_256): + raise HTTPException(status_code=401, detail="Invalid signature") + + payload = await request.json() + event_type = request.headers.get("X-GitHub-Event") + + # Process event + await handle_github_event(event_type, payload) + + return {"status": "accepted"} + +@app.post("/webhook/cve-alert") +async def cve_alert_webhook(request: Request): + """Handle CVE alert webhooks from external systems""" + payload = await request.json() + + # Extract CVE IDs + cve_ids = payload.get("cve_ids", []) + + # Submit Argo workflow + await submit_cve_analysis_workflow(cve_ids) + + return {"status": "accepted", "workflow": "submitted"} + +@app.get("/health") +async def health_check(): + """Health check endpoint""" + return {"status": "healthy", "service": "cve-bridge"} + +def validate_github_signature(payload: bytes, signature: str) -> bool: + """Validate GitHub webhook signature""" + if not signature or not WEBHOOK_SECRET: + return False + + expected = "sha256=" + hmac.new( + WEBHOOK_SECRET.encode(), + payload, + hashlib.sha256 + ).hexdigest() + + return hmac.compare_digest(expected, signature) +``` + +```python +# src/bridge/argo_client.py +from argo_workflows import WorkflowsServiceApi, Configuration, ApiClient +from argo_workflows.models import ( + V1alpha1Workflow, + V1alpha1WorkflowSpec, + V1alpha1Template, +) + +class ArgoClient: + """Client for Argo Workflows API""" + + def __init__(self, namespace: str = "argo"): + config = Configuration() + config.host = os.environ.get("ARGO_API_HOST", "https://localhost:2746") + self.api = WorkflowsServiceApi(ApiClient(config)) + self.namespace = namespace + + def submit_workflow( + self, + workflow_name: str, + parameters: dict + ) -> str: + """Submit a workflow to Argo""" + + # Load workflow template + workflow_manifest = self._load_workflow_template(workflow_name) + + # Set parameters + workflow_manifest["spec"]["arguments"] = { + "parameters": [ + {"name": k, "value": str(v)} + for k, v in parameters.items() + ] + } + + # Submit to Argo + response = self.api.create_workflow( + namespace=self.namespace, + body=workflow_manifest + ) + + workflow_id = response.metadata.name + logger.info(f"Submitted workflow: {workflow_id}") + + return workflow_id + + def get_workflow_status(self, workflow_id: str) -> str: + """Get status of a workflow""" + workflow = self.api.get_workflow( + namespace=self.namespace, + name=workflow_id + ) + return workflow.status.phase + + def get_workflow_logs(self, workflow_id: str) -> str: + """Retrieve logs from a workflow""" + logs = self.api.workflow_logs( + namespace=self.namespace, + name=workflow_id + ) + return logs +``` + +## Argo Workflow Definition + +```yaml +# argo/cve-analysis-workflow.yaml +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: cve-analysis- +spec: + entrypoint: cve-pipeline + + arguments: + parameters: + - name: cve-ids + value: "" + - name: days + value: "7" + + templates: + - name: cve-pipeline + dag: + tasks: + - name: ingest + template: ingest-nvd + arguments: + parameters: + - name: days + value: "{{workflow.parameters.days}}" + + - name: align + template: alignment + dependencies: [ingest] + arguments: + artifacts: + - name: cves + from: "{{tasks.ingest.outputs.artifacts.cves}}" + + - name: arbiter + template: predict-severity + dependencies: [align] + arguments: + artifacts: + - name: embeddings + from: "{{tasks.align.outputs.artifacts.embeddings}}" + + - name: refractors + template: detect-shifts + dependencies: [align] + arguments: + artifacts: + - name: embeddings + from: "{{tasks.align.outputs.artifacts.embeddings}}" + + - name: evidence + template: calculate-evidence + dependencies: [arbiter, refractors] + arguments: + artifacts: + - name: predictions + from: "{{tasks.arbiter.outputs.artifacts.predictions}}" + + - name: ingest-nvd + inputs: + parameters: + - name: days + container: + image: gcr.io/PROJECT_ID/lidlift:latest + command: [python, -m, src.ingest.nvd_client] + args: ["--days", "{{inputs.parameters.days}}", "--output", "/tmp/cves.jsonl"] + outputs: + artifacts: + - name: cves + path: /tmp/cves.jsonl + + - name: alignment + inputs: + artifacts: + - name: cves + path: /tmp/cves.jsonl + container: + image: gcr.io/PROJECT_ID/lidlift:latest + command: [python, -m, src.alignment] + args: ["--input", "/tmp/cves.jsonl", "--output", "/tmp/embeddings.npy"] + outputs: + artifacts: + - name: embeddings + path: /tmp/embeddings.npy + + # Additional templates for arbiter, refractors, evidence... +``` + +## Kubernetes Deployment + +```yaml +# k8s/webhook-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cve-bridge + namespace: cve-analysis +spec: + replicas: 2 + selector: + matchLabels: + app: cve-bridge + template: + metadata: + labels: + app: cve-bridge + spec: + containers: + - name: bridge + image: gcr.io/PROJECT_ID/cve-bridge:latest + ports: + - containerPort: 8000 + env: + - name: WEBHOOK_SECRET + valueFrom: + secretKeyRef: + name: webhook-secrets + key: secret + - name: ARGO_API_HOST + value: "https://argo-server.argo:2746" + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 30 + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" +``` + +## Configuration + +Environment variables: +- `WEBHOOK_SECRET`: Secret for validating webhooks +- `ARGO_API_HOST`: Argo Workflows API endpoint +- `ARGO_NAMESPACE`: Kubernetes namespace for workflows + +## References +- Argo Workflows: https://argoproj.github.io/argo-workflows/ +- FastAPI: https://fastapi.tiangolo.com/ +- GitHub Webhooks: https://docs.github.com/en/webhooks diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..9cefb56 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,54 @@ +# Code Owners + +# This file defines code ownership for the CVE Matter-Analysis OS repository. +# Code owners are automatically requested for review when someone opens a pull request +# that modifies code in a directory they own. + +# Default owners for everything in the repo +* @security-team @cve-analysis-team + +# Infrastructure and deployment +/k8s/ @sre-team @security-team +/terraform/ @sre-team @infrastructure-team +/argo/ @sre-team @mlops-team +/.github/workflows/ @devops-team @sre-team +/docker/ @devops-team @sre-team + +# Security-sensitive files and configurations +/SECURITY.md @security-team +/CODEOWNERS @security-team +/.github/workflows/codeql.yml @security-team +/.github/workflows/container-scan.yml @security-team + +# Prompts and system descriptions (high-value IP) +/prompts/ @ml-engineering-team @security-team +/capsules/ @ml-engineering-team @architecture-team + +# Core pipeline modules +/src/ingest/ @backend-team @data-engineering-team +/src/alignment/ @ml-engineering-team @research-team +/src/models/ @ml-engineering-team @data-science-team +/src/refractors/ @ml-engineering-team @research-team +/src/evaluation/ @ml-engineering-team @research-team +/src/cuda/ @ml-engineering-team @performance-team +/src/bridge/ @backend-team @sre-team + +# Tests +/tests/ @qa-team @backend-team + +# Scripts and utilities +/scripts/ @devops-team @backend-team + +# Documentation +/docs/ @tech-writing-team +/README.md @tech-writing-team @architecture-team + +# Configuration files +/config/ @sre-team @security-team +/requirements.txt @backend-team @security-team +/requirements-dev.txt @backend-team +/pyproject.toml @backend-team +/setup.py @backend-team + +# Copilot agent configuration +/.copilot/ @ai-team @architecture-team diff --git a/README.md b/README.md index 8b13789..ee1aecf 100644 --- a/README.md +++ b/README.md @@ -1 +1,222 @@ +# CVE Matter-Analysis OS +**Defense-Only Blue-Team Vulnerability Analysis Pipeline** + +## Overview + +The CVE Matter-Analysis OS is a comprehensive pipeline for analyzing vulnerabilities from the National Vulnerability Database (NVD). This system employs advanced machine learning techniques including positional alignment, stacked arbitration, epsilon-refractors for distributional shift detection, and Bayesian evidence calculation to prioritize and analyze security vulnerabilities. + +**Mission**: Defense-only CVE analysis supporting blue-team security operations. + +## Architecture + +The pipeline consists of five main stages: + +1. **NVD Ingest** → Fetch and normalize CVE data from NVD API v2.0 +2. **Positional Alignment** → Align vulnerability embeddings across different spaces +3. **Stacked Arbiter** → Ensemble learning for severity prediction with Pareto optimization +4. **Epsilon-Refractors** → Detect distributional shifts in vulnerability patterns +5. **Bayesian Evidence** → Calculate evidence scores using BIC/WAIC for prioritization + +## Technology Stack + +- **Language**: Python 3.11+ +- **ML/Data**: NumPy, SciPy, scikit-learn +- **Optional**: CUDA for GPU acceleration +- **Container**: Docker with gVisor isolation +- **Orchestration**: Kubernetes (GKE), Argo Workflows +- **Infrastructure**: Terraform +- **CI/CD**: GitHub Actions + +## Quick Start + +### Prerequisites + +- Python 3.11 or higher +- Docker (optional, for containerized deployment) +- kubectl and access to GKE cluster (for production deployment) +- NVD API key (optional but recommended for higher rate limits) + +### Local Development + +```bash +# Clone repository +git clone https://github.com/igor-holt/Instinct.git +cd Instinct + +# Create virtual environment +python3.11 -m venv venv +source venv/bin/activate # or `venv\Scripts\activate` on Windows + +# Install dependencies +pip install -r requirements.txt + +# Run pipeline +python -m src.main +``` + +### Running with Docker + +```bash +# Build image +docker build -t lidlift:latest . + +# Run container +docker run -it --rm \ + -e NVD_API_KEY=$NVD_API_KEY \ + -v $(pwd)/data:/app/data \ + lidlift:latest +``` + +## Configuration + +Environment variables: +- `NVD_API_KEY` - API key for NVD access (optional but recommended) +- `ALIGNMENT_R2_THRESHOLD` - Minimum R² for alignment quality (default: 0.8) +- `EPSILON_THRESHOLD` - Threshold for refractor alerts (default: 0.05) +- `EVIDENCE_METHOD` - Bayesian evidence method: BIC or WAIC (default: WAIC) +- `LOG_LEVEL` - Logging verbosity: DEBUG, INFO, WARN, ERROR (default: INFO) +- `CUDA_VISIBLE_DEVICES` - GPU device selection for CUDA operations (optional) + +## Documentation + +- **System Description**: [prompts/legendary_lidlift_v14.md](prompts/legendary_lidlift_v14.md) +- **Capsule Configurations**: [capsules/](capsules/) +- **Security Policy**: [SECURITY.md](SECURITY.md) +- **Code Ownership**: [CODEOWNERS](CODEOWNERS) + +## Copilot Agent Usage + +This repository is configured for GitHub Copilot agents to assist with development tasks. + +### Getting Started with Copilot + +1. **Read the Agent Guide**: Start with [`.copilot/AGENT_GUIDE.md`](.copilot/AGENT_GUIDE.md) for comprehensive instructions +2. **Review Task Definitions**: Tasks are defined in [`.copilot/tasks/`](.copilot/tasks/) numbered 010-090 +3. **Follow the Workflow**: Execute tasks sequentially, one PR per task +4. **Reference Documentation**: Use system documentation in `prompts/` and `capsules/` + +### Task Overview + +- **Task 010**: NVD data ingestion with delta sync and ETag support +- **Task 020**: Positional alignment using Procrustes and CCA +- **Task 030**: Stacked arbiter with Pareto knee detection +- **Task 040**: Epsilon-refractors for distributional shift detection +- **Task 050**: Bayesian evidence calculation (BIC/WAIC) +- **Task 060**: Notion synchronization for documentation +- **Task 070**: Automated capsule publishing on release +- **Task 080**: Optional CUDA/GPU acceleration support +- **Task 090**: Webhook receiver and Argo Workflows integration + +### Key Principles for Copilot Agents + +1. **Defense-Only**: All code must support defensive security purposes exclusively +2. **File-Anchored**: Each task specifies exact files to create or modify +3. **One PR Per Task**: Create focused, reviewable pull requests +4. **Security-First**: Never commit secrets; scan dependencies; validate inputs +5. **Sequential Execution**: Complete tasks in order (010 → 020 → ... → 090) + +### Creating a PR + +When working on a task: +```markdown +## Task: [Number] - [Name] + +**Rationale**: Brief explanation of changes + +**Task Reference**: `.copilot/tasks/XXX_task_name.md` + +### Changes Made +- Created/Modified: [file list] +- Tests added: [test files] + +### Validation +- [x] Linters pass +- [x] Tests pass +- [x] Security scan clean +- [x] Acceptance criteria met + +**Test Output**: [Include evidence] +``` + +## Development + +### Running Tests + +```bash +# Run all tests +pytest + +# Run with coverage +pytest --cov=src --cov-report=html + +# Run specific module tests +pytest tests/ingest/ -v +``` + +### Linting and Formatting + +```bash +# Format code +black src/ tests/ + +# Check linting +flake8 src/ tests/ + +# Type checking +mypy src/ +``` + +## Security + +**This is a defense-only system**. See [SECURITY.md](SECURITY.md) for: +- Security policy and guardrails +- Vulnerability reporting process +- Secure development practices +- Incident response procedures + +### Prohibited Activities + +- ❌ Exploit generation +- ❌ Offensive security operations +- ❌ Cryptographic breaking +- ❌ Malware development + +### Permitted Activities + +- ✅ Vulnerability analysis +- ✅ Risk assessment and prioritization +- ✅ Threat detection and monitoring +- ✅ Defense planning and operations + +## Contributing + +1. Review the [Copilot Agent Guide](.copilot/AGENT_GUIDE.md) +2. Select a task from [`.copilot/tasks/`](.copilot/tasks/) +3. Implement changes following the task definition +4. Create a focused PR with validation evidence +5. Address code review feedback + +## License + +[License TBD - Add appropriate license] + +## Support + +- **Issues**: Use GitHub Issues for bug reports and feature requests +- **Security**: Report vulnerabilities privately via [SECURITY.md](SECURITY.md) +- **Documentation**: See `prompts/` and `.copilot/` directories + +## Acknowledgments + +This system integrates concepts from: +- NVD/NIST for vulnerability data +- H-MOC (Hierarchical Multi-Objective Coordinator) for orchestration +- LID-LIFT (Layered Intelligence Defense) framework +- Academic research in Bayesian model comparison and distributional shift detection + +--- + +**Version**: 1.0.0 +**Last Updated**: 2025-11-13 +**Status**: Active Development diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..f1d605c --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,227 @@ +# Security Policy + +## Mission Statement + +This repository implements a **defense-only** CVE matter-analysis pipeline for blue-team security operations. All code, analysis, and tooling must support defensive security purposes exclusively. + +## Guardrails + +### Prohibited Activities + +This system and repository **MUST NOT** be used for: + +- ❌ **Exploit payload generation** - Do not create or distribute exploit code +- ❌ **Offensive security operations** - No penetration testing tools or attack frameworks +- ❌ **Cryptographic breaking** - No attempts to break or weaken cryptographic systems +- ❌ **Malware development** - No malicious software of any kind +- ❌ **Unauthorized access** - No tools for gaining unauthorized system access + +### Permitted Activities + +This system **IS DESIGNED FOR**: + +- ✅ **Vulnerability analysis** - Defensive analysis of CVE data +- ✅ **Risk assessment** - Prioritizing vulnerabilities for remediation +- ✅ **Threat detection** - Identifying patterns in vulnerability data +- ✅ **Security monitoring** - Tracking changes in the threat landscape +- ✅ **Defense planning** - Supporting blue-team security operations + +## Reporting a Vulnerability + +### Scope + +We accept vulnerability reports for: + +- Security vulnerabilities in this codebase +- Vulnerabilities in dependencies +- Misconfigurations that could lead to security issues +- Authentication or authorization bypasses +- Information disclosure issues +- Any code that violates our defense-only mission + +### How to Report + +**DO NOT** create public GitHub issues for security vulnerabilities. + +Instead, please report security issues privately: + +1. **Email**: Send details to the security team at `security@example.com` (replace with actual contact) +2. **Subject**: Start with `[SECURITY]` followed by a brief description +3. **Include**: + - Description of the vulnerability + - Steps to reproduce + - Potential impact + - Suggested fix (if known) + - Your contact information + +### What to Expect + +- **Acknowledgment**: Within 48 hours of your report +- **Initial Assessment**: Within 5 business days +- **Updates**: At least every 10 days until resolution +- **Resolution**: We aim to fix critical vulnerabilities within 30 days +- **Credit**: We will credit reporters (unless you prefer anonymity) + +### Coordinated Vulnerability Disclosure (CVD) + +We follow responsible disclosure practices: + +1. **Private Disclosure**: Report to us privately first +2. **Coordinated Timeline**: We'll work with you on a disclosure timeline +3. **Public Disclosure**: After a fix is deployed and users have time to update +4. **Default Timeline**: 90 days from initial report (can be adjusted) + +## Security Measures + +### Code Security + +- **Input Validation**: All external inputs are sanitized and validated +- **Output Encoding**: All outputs are properly encoded to prevent injection +- **Parameterized Queries**: Database queries use parameterization +- **Error Handling**: Errors don't expose sensitive system information +- **Least Privilege**: Components run with minimal required permissions + +### Dependency Management + +- **Version Pinning**: All dependencies use exact version pins +- **Regular Scanning**: Automated scanning for known vulnerabilities +- **Timely Updates**: Security patches applied within 7 days +- **Minimal Dependencies**: Only necessary packages are included +- **Trusted Sources**: Dependencies from reputable, verified sources + +### Secrets Management + +- **No Secrets in Code**: API keys, passwords, tokens never in source code +- **Environment Variables**: Secrets passed via environment or secret managers +- **Secret Rotation**: Regular rotation of credentials and API keys +- **Access Control**: Secrets accessible only to authorized services +- **Audit Logging**: All secret access is logged + +### Infrastructure Security + +- **Container Security**: Docker images scanned with Trivy +- **Runtime Security**: gVisor provides container isolation on GKE +- **Network Policies**: Kubernetes network policies restrict traffic +- **RBAC**: Role-based access control for all Kubernetes resources +- **Encryption**: Data encrypted in transit (TLS) and at rest + +### CI/CD Security + +- **Branch Protection**: Main branch requires reviews and passing CI +- **Code Review**: All changes require at least one review +- **Automated Scanning**: CodeQL and dependency scanning on all PRs +- **Signed Commits**: Encourage or require GPG-signed commits +- **Audit Trails**: All CI/CD actions logged and auditable + +## Security Scanning + +### Automated Scans + +We run the following security scans: + +1. **CodeQL** (Python): Static analysis for security vulnerabilities +2. **Trivy**: Container image vulnerability scanning +3. **Dependency Check**: Known vulnerabilities in Python packages +4. **SAST**: Static application security testing +5. **Secret Scanning**: Detect accidentally committed secrets + +### Scan Schedule + +- **Every PR**: CodeQL, dependency check +- **Every commit to main**: All scans +- **Weekly**: Full security audit +- **Before release**: Comprehensive security review + +### Failure Thresholds + +- **HIGH/CRITICAL vulnerabilities**: CI fails, must be fixed before merge +- **MEDIUM vulnerabilities**: Warning, should be fixed promptly +- **LOW vulnerabilities**: Tracked, fixed in regular maintenance + +## Secure Development Practices + +### Code Review Checklist + +Before approving any PR, reviewers should verify: + +- [ ] No hardcoded secrets or credentials +- [ ] Input validation for all external data +- [ ] Proper error handling (no sensitive info in errors) +- [ ] SQL queries use parameterization +- [ ] Authentication and authorization properly implemented +- [ ] Sensitive data encrypted in transit and at rest +- [ ] New dependencies scanned for vulnerabilities +- [ ] Defense-only stance maintained (no exploits/offensive tools) +- [ ] Documentation updated for security-relevant changes + +### Testing Requirements + +Security-relevant code must include: + +- Unit tests for input validation +- Tests for authentication/authorization logic +- Tests for error handling +- Integration tests for security controls + +## Compliance and Policies + +### Data Handling + +- **CVE Data**: Public data from NVD, no restrictions +- **Logs**: May contain system information, retained for 90 days +- **Metrics**: Aggregated, anonymized performance data +- **User Data**: Not applicable (system-to-system integration) + +### Access Control + +- **Code Access**: GitHub repository access via organization membership +- **Production Access**: Restricted to SRE and security teams +- **Secrets Access**: Limited to CI/CD and production services +- **Audit Logs**: Retained for 1 year, reviewed quarterly + +### Incident Response + +In the event of a security incident: + +1. **Contain**: Isolate affected systems immediately +2. **Assess**: Determine scope and impact +3. **Notify**: Alert security team and stakeholders +4. **Remediate**: Apply fixes and deploy updates +5. **Review**: Post-incident analysis and lessons learned +6. **Document**: Update runbooks and procedures + +## Contact + +- **Security Team**: security@example.com (replace with actual) +- **On-Call**: Via PagerDuty/incident management system +- **Security Lead**: TBD (assign a security champion) + +## Acknowledgments + +We appreciate security researchers who responsibly disclose vulnerabilities. Hall of fame for contributors: + +- (Contributors will be listed here with their permission) + +## Updates + +This security policy is reviewed and updated quarterly. Last updated: 2025-11-13 + +## License and Legal + +By reporting a vulnerability, you agree to: + +- Allow us reasonable time to remediate before public disclosure +- Not exploit the vulnerability beyond what's necessary to demonstrate it +- Not access, modify, or delete data without authorization +- Follow coordinated disclosure practices + +We commit to: + +- Respond to reports promptly and professionally +- Keep you informed of remediation progress +- Credit you for the discovery (unless you prefer anonymity) +- Not take legal action for responsible security research + +--- + +**Remember**: This is a **DEFENSE-ONLY** system. Any attempt to use this code for offensive purposes, exploit generation, or cryptographic attacks violates this security policy and the project's core mission. From 4460931735ebbb3b001005d2ad0e23772726de8d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 13 Nov 2025 00:36:55 +0000 Subject: [PATCH 4/5] Add CI/CD workflows and infrastructure (PR #3) Co-authored-by: igor-holt <125706350+igor-holt@users.noreply.github.com> --- .github/workflows/ci.yml | 233 +++++++++++++++++++++++++ .github/workflows/codeql.yml | 72 ++++++++ .github/workflows/container-scan.yml | 145 +++++++++++++++ .github/workflows/notion-sync.yml | 95 ++++++++++ .github/workflows/publish-capsules.yml | 223 +++++++++++++++++++++++ .gitignore | 129 ++++++++++++++ Dockerfile | 61 +++++++ pyproject.toml | 113 ++++++++++++ requirements-dev.txt | 30 ++++ requirements.txt | 39 +++++ src/__init__.py | 0 src/main.py | 59 +++++++ tests/__init__.py | 0 tests/test_structure.py | 159 +++++++++++++++++ 14 files changed, 1358 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/container-scan.yml create mode 100644 .github/workflows/notion-sync.yml create mode 100644 .github/workflows/publish-capsules.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 requirements.txt create mode 100644 src/__init__.py create mode 100644 src/main.py create mode 100644 tests/__init__.py create mode 100644 tests/test_structure.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..985d2d5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,233 @@ +# CI Workflow for CVE Matter-Analysis OS +# +# This workflow runs linting and unit tests on standard GitHub-hosted runners. +# For heavy jobs (GPU tests, large-scale integration tests), see separate workflows. +# +# Runner Configuration: +# - Standard jobs use ubuntu-latest (GitHub-hosted runner) +# - Heavy jobs should use larger runners per GitHub Actions documentation +# - See: https://docs.github.com/en/enterprise-cloud@latest/actions/using-github-hosted-runners/about-larger-runners + +name: CI + +on: + push: + branches: + - main + - develop + pull_request: + branches: + - main + - develop + workflow_dispatch: + +jobs: + # Lint job - runs on standard GitHub-hosted runner + lint: + name: Lint and Format Check + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt || pip install flake8 black mypy + + - name: Run flake8 + run: | + flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 src/ --count --max-complexity=10 --max-line-length=100 --statistics + continue-on-error: true + + - name: Check formatting with black + run: | + black --check --line-length 100 src/ + continue-on-error: true + + - name: Type check with mypy + run: | + mypy src/ --ignore-missing-imports + continue-on-error: true + + # Unit tests - runs on standard GitHub-hosted runner + unit-tests: + name: Unit Tests + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: ['3.11'] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt || echo "No requirements.txt yet" + pip install -r requirements-dev.txt || pip install pytest pytest-cov + + - name: Run unit tests + run: | + pytest tests/ -v --cov=src --cov-report=xml --cov-report=term-missing || echo "No tests yet" + continue-on-error: true + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + fail_ci_if_error: false + continue-on-error: true + + # Integration tests - runs on standard runner for now + # TODO: Consider moving to larger runner if tests become compute-intensive + integration-tests: + name: Integration Tests + runs-on: ubuntu-latest + needs: unit-tests + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt || echo "No requirements.txt yet" + pip install -r requirements-dev.txt || pip install pytest + + - name: Run integration tests + run: | + pytest tests/ -v -m integration || echo "No integration tests yet" + continue-on-error: true + + # Heavy computational tests + # NOTE: This job should use a larger GitHub-hosted runner when available + # Update the runs-on label based on your organization's GitHub Actions runner configuration + # See: https://docs.github.com/en/enterprise-cloud@latest/actions/using-github-hosted-runners/about-larger-runners + heavy-tests: + name: Heavy Computational Tests (Large Runner) + # TODO: Replace with actual large runner label from your org's GitHub Actions configuration + # Examples from docs: ubuntu-22.04-16core, ubuntu-22.04-64core + # For now using standard runner as placeholder + runs-on: ubuntu-latest + needs: unit-tests + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt || echo "No requirements.txt yet" + pip install -r requirements-dev.txt || pip install pytest + + - name: Run heavy tests (epsilon-sweeps, large-scale alignment) + run: | + # These tests may include: + # - Large-scale alignment tests (10K+ samples) + # - Epsilon-refractor sweep tests across parameter ranges + # - Full pipeline integration tests + pytest tests/ -v -m heavy --timeout=600 || echo "No heavy tests yet" + continue-on-error: true + timeout-minutes: 30 + + # Validate all JSON and YAML files + validate-configs: + name: Validate Configuration Files + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Validate JSON files + run: | + for file in $(find . -name "*.json" -not -path "./node_modules/*" -not -path "./.git/*"); do + echo "Validating $file..." + python -m json.tool "$file" > /dev/null || exit 1 + done + + - name: Validate YAML files + run: | + pip install pyyaml + python -c " + import yaml + import sys + from pathlib import Path + + failed = False + for file in Path('.').rglob('*.yml'): + if '.git' not in str(file) and 'node_modules' not in str(file): + try: + with open(file) as f: + yaml.safe_load(f) + print(f'✓ {file}') + except Exception as e: + print(f'✗ {file}: {e}') + failed = True + + for file in Path('.').rglob('*.yaml'): + if '.git' not in str(file) and 'node_modules' not in str(file): + try: + with open(file) as f: + yaml.safe_load(f) + print(f'✓ {file}') + except Exception as e: + print(f'✗ {file}: {e}') + failed = True + + sys.exit(1 if failed else 0) + " + + # Build status check - all jobs must pass + ci-success: + name: CI Success + runs-on: ubuntu-latest + needs: [lint, unit-tests, integration-tests, validate-configs] + if: always() + + steps: + - name: Check all jobs + run: | + echo "Lint: ${{ needs.lint.result }}" + echo "Unit Tests: ${{ needs.unit-tests.result }}" + echo "Integration Tests: ${{ needs.integration-tests.result }}" + echo "Validate Configs: ${{ needs.validate-configs.result }}" + + if [ "${{ needs.lint.result }}" != "success" ] || \ + [ "${{ needs.unit-tests.result }}" != "success" ] || \ + [ "${{ needs.validate-configs.result }}" != "success" ]; then + echo "One or more required jobs failed" + exit 1 + fi diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..553ce28 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,72 @@ +# CodeQL Analysis Workflow +# +# This workflow performs security analysis using CodeQL for Python code. +# Runs on standard GitHub-hosted runners. +# +# See: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning-with-codeql + +name: CodeQL Security Analysis + +on: + push: + branches: + - main + - develop + pull_request: + branches: + - main + - develop + schedule: + # Run weekly on Mondays at 00:00 UTC + - cron: '0 0 * * 1' + workflow_dispatch: + +jobs: + analyze: + name: Analyze Python Code + runs-on: ubuntu-latest + + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: ['python'] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # Specify CodeQL query packs + queries: security-and-quality + + # Install dependencies for analysis (helps CodeQL understand imports) + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt || echo "No requirements.txt yet" + continue-on-error: true + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{ matrix.language }}" + + - name: Upload SARIF results + uses: github/codeql-action/upload-sarif@v2 + if: always() + with: + sarif_file: ../results + continue-on-error: true diff --git a/.github/workflows/container-scan.yml b/.github/workflows/container-scan.yml new file mode 100644 index 0000000..708fea3 --- /dev/null +++ b/.github/workflows/container-scan.yml @@ -0,0 +1,145 @@ +# Container Security Scanning with Trivy +# +# This workflow scans Docker container images for vulnerabilities using Trivy. +# Fails on HIGH or CRITICAL vulnerabilities. +# Runs on standard GitHub-hosted runners. +# +# See: https://github.com/aquasecurity/trivy-action + +name: Container Security Scan + +on: + push: + branches: + - main + - develop + paths: + - 'Dockerfile' + - 'docker/**' + - '.github/workflows/container-scan.yml' + pull_request: + branches: + - main + - develop + paths: + - 'Dockerfile' + - 'docker/**' + workflow_dispatch: + +jobs: + scan-image: + name: Scan Docker Image with Trivy + runs-on: ubuntu-latest + + permissions: + contents: read + security-events: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker image + run: | + # Build the main Dockerfile if it exists + if [ -f "Dockerfile" ]; then + docker build -t lidlift:${{ github.sha }} . + else + echo "No Dockerfile found, creating minimal test image" + cat > Dockerfile.test < scripts/notion_sync.py <<'EOF' + """ + Notion synchronization script placeholder. + This script will be implemented in Task 060. + """ + import sys + print("Notion sync functionality will be implemented in Task 060") + print("For now, this is a placeholder that always succeeds") + sys.exit(0) + EOF + fi + + if [ ! -f "scripts/register_prompt_notion.sh" ]; then + cat > scripts/register_prompt_notion.sh <<'EOF' + #!/bin/bash + # Notion sync script placeholder + # Will be implemented in Task 060 + echo "Notion sync functionality will be implemented in Task 060" + exit 0 + EOF + chmod +x scripts/register_prompt_notion.sh + fi + + - name: Sync prompts to Notion + env: + NOTION_API_KEY: ${{ secrets.NOTION_API_KEY }} + NOTION_PAGE_ID: ${{ secrets.NOTION_PAGE_ID }} + run: | + # Check if secrets are configured + if [ -z "$NOTION_API_KEY" ] || [ -z "$NOTION_PAGE_ID" ]; then + echo "⚠️ Notion secrets not configured, skipping sync" + echo "To enable Notion sync, configure NOTION_API_KEY and NOTION_PAGE_ID secrets" + exit 0 + fi + + # Run sync script + bash scripts/register_prompt_notion.sh + continue-on-error: true + + - name: Report sync status + if: always() + run: | + echo "Notion sync completed" + echo "Files that would be synced:" + echo " - prompts/legendary_lidlift_v14.md" + echo " - capsules/*.json (summaries)" + echo " - .copilot/AGENT_GUIDE.md" diff --git a/.github/workflows/publish-capsules.yml b/.github/workflows/publish-capsules.yml new file mode 100644 index 0000000..befd2e8 --- /dev/null +++ b/.github/workflows/publish-capsules.yml @@ -0,0 +1,223 @@ +# Publish Capsules Workflow +# +# Publishes capsule configuration files to cloud storage when new versions are tagged. +# Runs on standard GitHub-hosted runners. +# +# Prerequisites: +# - GCP_WORKLOAD_IDENTITY_PROVIDER secret configured for GCP authentication +# - GCP_SERVICE_ACCOUNT secret configured +# - GCS_CAPSULES_BUCKET secret or variable for target bucket + +name: Publish Capsules + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + version: + description: 'Version to publish (e.g., v1.4.0)' + required: true + default: 'latest' + +jobs: + validate: + name: Validate Capsules + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Create validation script if not exists + run: | + mkdir -p scripts + if [ ! -f "scripts/validate_capsules.py" ]; then + cat > scripts/validate_capsules.py <<'EOF' + """ + Capsule validation script placeholder. + This script will be implemented in Task 070. + """ + import json + import sys + from pathlib import Path + + def validate_capsules(): + capsules_dir = Path("capsules") + if not capsules_dir.exists(): + print("No capsules directory found") + return True + + valid = True + for capsule_file in capsules_dir.glob("*.json"): + print(f"Validating {capsule_file.name}...") + try: + with open(capsule_file) as f: + data = json.load(f) + + # Check required fields + required = ["name", "version", "type", "description"] + for field in required: + if field not in data: + print(f" ✗ Missing field: {field}") + valid = False + + if valid: + print(f" ✓ Valid") + except json.JSONDecodeError as e: + print(f" ✗ Invalid JSON: {e}") + valid = False + + return valid + + if __name__ == "__main__": + if validate_capsules(): + print("✅ All capsules valid") + sys.exit(0) + else: + print("❌ Validation failed") + sys.exit(1) + EOF + fi + + - name: Validate capsule JSON files + run: | + python scripts/validate_capsules.py + + - name: Create artifact with validated capsules + uses: actions/upload-artifact@v3 + with: + name: validated-capsules + path: capsules/*.json + retention-days: 30 + + publish: + name: Publish to Cloud Storage + runs-on: ubuntu-latest + needs: validate + + permissions: + contents: read + id-token: write # Required for GCP Workload Identity + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download validated capsules + uses: actions/download-artifact@v3 + with: + name: validated-capsules + path: capsules/ + + - name: Determine version + id: version + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + VERSION="${{ github.event.inputs.version }}" + else + VERSION="${{ github.ref_name }}" + fi + + # Remove 'v' prefix if present + VERSION_CLEAN="${VERSION#v}" + + echo "version=${VERSION}" >> $GITHUB_OUTPUT + echo "version_clean=${VERSION_CLEAN}" >> $GITHUB_OUTPUT + echo "Publishing version: ${VERSION_CLEAN}" + + - name: Create publish script if not exists + run: | + mkdir -p scripts + if [ ! -f "scripts/publish_capsules.sh" ]; then + cat > scripts/publish_capsules.sh <<'EOF' + #!/bin/bash + # Capsule publishing script placeholder + # This script will be implemented in Task 070 + + set -euo pipefail + + VERSION="${VERSION:-latest}" + GCS_BUCKET="${GCS_CAPSULES_BUCKET:-}" + + echo "Publishing capsules version: $VERSION" + + if [ -z "$GCS_BUCKET" ]; then + echo "⚠️ GCS_CAPSULES_BUCKET not configured" + echo "Capsules would be published to: gs://BUCKET_NAME/" + echo "Files to publish:" + ls -lh capsules/*.json || echo "No capsules found" + exit 0 + fi + + # In actual implementation (Task 070), this would: + # - Upload each capsule to GCS with version tag + # - Update "latest" aliases + # - Set appropriate caching headers + + echo "✅ Publishing placeholder complete" + exit 0 + EOF + chmod +x scripts/publish_capsules.sh + fi + + - name: Authenticate to GCP (if configured) + if: env.GCP_WORKLOAD_IDENTITY_PROVIDER != '' + uses: google-github-actions/auth@v1 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + continue-on-error: true + env: + GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + + - name: Set up Cloud SDK (if authenticated) + if: env.GCP_WORKLOAD_IDENTITY_PROVIDER != '' + uses: google-github-actions/setup-gcloud@v1 + continue-on-error: true + env: + GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + + - name: Publish capsules + env: + VERSION: ${{ steps.version.outputs.version_clean }} + GCS_CAPSULES_BUCKET: ${{ secrets.GCS_CAPSULES_BUCKET }} + run: | + bash scripts/publish_capsules.sh + + - name: Create release notes + run: | + echo "## Published Capsules - Version ${{ steps.version.outputs.version_clean }}" > release-notes.md + echo "" >> release-notes.md + echo "### Files Published:" >> release-notes.md + for file in capsules/*.json; do + if [ -f "$file" ]; then + filename=$(basename "$file") + echo "- \`$filename\`" >> release-notes.md + fi + done + echo "" >> release-notes.md + echo "### Access URLs:" >> release-notes.md + echo "- Latest: \`https://storage.googleapis.com/\${BUCKET}/\${filename}-latest.json\`" >> release-notes.md + echo "- Versioned: \`https://storage.googleapis.com/\${BUCKET}/\${filename}-${{ steps.version.outputs.version_clean }}.json\`" >> release-notes.md + + - name: Upload release notes + uses: actions/upload-artifact@v3 + with: + name: release-notes + path: release-notes.md + retention-days: 90 + + - name: Summary + run: | + echo "## Capsule Publishing Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Version**: ${{ steps.version.outputs.version_clean }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + cat release-notes.md >> $GITHUB_STEP_SUMMARY diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b9c3a26 --- /dev/null +++ b/.gitignore @@ -0,0 +1,129 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Virtual environments +venv/ +env/ +ENV/ +env.bak/ +venv.bak/ + +# PyCharm +.idea/ + +# VSCode +.vscode/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pytest +.pytest_cache/ +.coverage +htmlcov/ +coverage.xml +*.cover +.hypothesis/ + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Environment variables +.env +.env.local +.env.*.local + +# Data files (can be large) +data/ +*.jsonl +*.npy +*.pkl +*.h5 +*.hdf5 + +# Checkpoints +checkpoints/ +*.checkpoint + +# Models +models/*.pkl +models/*.joblib +models/*.pt +models/*.pth +!models/.gitkeep + +# Logs +logs/ +*.log + +# Temporary files +tmp/ +temp/ +*.tmp + +# OS +.DS_Store +Thumbs.db + +# Docker +*.dockerignore + +# Terraform +.terraform/ +*.tfstate +*.tfstate.backup +.terraform.lock.hcl + +# Kubernetes +*.kubeconfig + +# IDE +*.swp +*.swo +*~ + +# Backup files +*.bak +*.backup + +# Compiled files +*.pyc + +# Distribution +*.whl + +# Secrets (never commit) +secrets/ +*.key +*.pem +*.crt +*.p12 +credentials.json +service-account.json + +# Aurora orchestrator output +aurora_final_state.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..33b1e2b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,61 @@ +# CVE Matter-Analysis OS - Docker Image +# Multi-stage build for minimal production image + +# Build stage +FROM python:3.11-slim as builder + +WORKDIR /build + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + g++ \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements and install dependencies +COPY requirements.txt . +RUN pip wheel --no-cache-dir --no-deps --wheel-dir /build/wheels -r requirements.txt + +# Production stage +FROM python:3.11-slim + +# Add non-root user for security +RUN useradd -m -u 1000 -s /bin/bash cveanalysis + +WORKDIR /app + +# Install runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Copy wheels from builder and install +COPY --from=builder /build/wheels /wheels +COPY requirements.txt . +RUN pip install --no-cache-dir --no-index --find-links=/wheels -r requirements.txt \ + && rm -rf /wheels + +# Copy application code +COPY src/ ./src/ +COPY capsules/ ./capsules/ +COPY prompts/ ./prompts/ +COPY pyproject.toml . + +# Create directories for data and logs +RUN mkdir -p /app/data /app/logs /app/checkpoints \ + && chown -R cveanalysis:cveanalysis /app + +# Switch to non-root user +USER cveanalysis + +# Health check +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import sys; sys.exit(0)" + +# Set environment variables +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + LOG_LEVEL=INFO + +# Default command (will be overridden for specific tasks) +CMD ["python", "-m", "src.main"] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f44d388 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,113 @@ +[build-system] +requires = ["setuptools>=68.0.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "cve-matter-analysis" +version = "1.0.0" +description = "Defense-only CVE matter-analysis pipeline for blue-team security operations" +readme = "README.md" +requires-python = ">=3.11" +license = {text = "MIT"} +authors = [ + {name = "CVE Analysis Team", email = "cve-team@example.com"} +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Topic :: Security", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +dependencies = [ + "requests>=2.31.0", + "numpy>=1.24.0", + "scipy>=1.10.0", + "pandas>=2.0.0", + "scikit-learn>=1.3.0", + "python-dateutil>=2.8.2", + "pyyaml>=6.0", + "structlog>=23.1.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4.0", + "pytest-cov>=4.1.0", + "flake8>=6.1.0", + "black>=23.7.0", + "mypy>=1.5.0", +] +cuda = [ + "torch>=2.0.0", +] +ml = [ + "xgboost>=2.0.0", + "lightgbm>=4.0.0", +] + +[project.urls] +Homepage = "https://github.com/igor-holt/Instinct" +Documentation = "https://github.com/igor-holt/Instinct/blob/main/README.md" +Repository = "https://github.com/igor-holt/Instinct" +Issues = "https://github.com/igor-holt/Instinct/issues" + +[tool.black] +line-length = 100 +target-version = ['py311'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +line_length = 100 + +[tool.mypy] +python_version = "3.11" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +ignore_missing_imports = true + +[tool.pytest.ini_options] +minversion = "7.0" +addopts = "-ra -q --strict-markers" +testpaths = ["tests"] +markers = [ + "integration: marks tests as integration tests (deselect with '-m \"not integration\"')", + "gpu: marks tests requiring GPU/CUDA (deselect with '-m \"not gpu\"')", + "heavy: marks tests that are computationally intensive (deselect with '-m \"not heavy\"')", + "slow: marks tests as slow (deselect with '-m \"not slow\"')", +] + +[tool.coverage.run] +source = ["src"] +omit = [ + "*/tests/*", + "*/test_*.py", +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", + "@abstractmethod", +] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..e2013cd --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,30 @@ +# CVE Matter-Analysis OS - Development Dependencies + +# Testing +pytest>=7.4.0 +pytest-cov>=4.1.0 +pytest-mock>=3.11.0 +pytest-asyncio>=0.21.0 +pytest-timeout>=2.1.0 + +# Code quality +flake8>=6.1.0 +black>=23.7.0 +mypy>=1.5.0 +isort>=5.12.0 + +# Type stubs +types-requests>=2.31.0 +types-PyYAML>=6.0.0 + +# Security scanning +bandit>=1.7.5 +safety>=2.3.0 + +# Documentation +sphinx>=7.1.0 +sphinx-rtd-theme>=1.3.0 + +# Build tools +setuptools>=68.0.0 +wheel>=0.41.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6bd49bd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,39 @@ +# CVE Matter-Analysis OS - Core Dependencies + +# HTTP and API clients +requests>=2.31.0 +requests-cache>=1.1.0 + +# Data processing +numpy>=1.24.0 +scipy>=1.10.0 +pandas>=2.0.0 + +# Machine learning +scikit-learn>=1.3.0 + +# Date handling +python-dateutil>=2.8.2 + +# Configuration +pyyaml>=6.0 + +# Logging +structlog>=23.1.0 + +# Optional: CUDA/GPU support (uncomment if using GPU acceleration) +# torch>=2.0.0 + +# Optional: Advanced ML models (uncomment if needed) +# xgboost>=2.0.0 +# lightgbm>=4.0.0 + +# Optional: Notion integration (for Task 060) +# notion-client>=2.0.0 + +# Optional: Argo Workflows (for Task 090) +# argo-workflows>=6.3.0 + +# Optional: FastAPI for webhook server (for Task 090) +# fastapi>=0.100.0 +# uvicorn[standard]>=0.23.0 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..8e241ad --- /dev/null +++ b/src/main.py @@ -0,0 +1,59 @@ +""" +CVE Matter-Analysis OS - Main Entry Point + +This module serves as the entry point for the CVE analysis pipeline. +The actual pipeline implementation will be completed in Tasks 010-090. +""" + +import sys +import logging +from pathlib import Path + +# Setup logging +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) + +logger = logging.getLogger(__name__) + + +def main(): + """Main entry point for the CVE analysis pipeline.""" + logger.info("CVE Matter-Analysis OS - Starting") + logger.info("=" * 60) + logger.info("Mission: Defense-only CVE analysis pipeline") + logger.info("Version: 1.0.0") + logger.info("=" * 60) + + # Check for required directories + required_dirs = ["data", "logs", "checkpoints"] + for dir_name in required_dirs: + dir_path = Path(dir_name) + if not dir_path.exists(): + logger.info(f"Creating directory: {dir_path}") + dir_path.mkdir(parents=True, exist_ok=True) + + # Display pipeline stages (to be implemented) + logger.info("\nPipeline Stages:") + logger.info(" 1. NVD Ingest → Task 010 (To be implemented)") + logger.info(" 2. Alignment → Task 020 (To be implemented)") + logger.info(" 3. Stacked Arbiter → Task 030 (To be implemented)") + logger.info(" 4. Refractors → Task 040 (To be implemented)") + logger.info(" 5. Evidence → Task 050 (To be implemented)") + + logger.info("\nOrchestration:") + logger.info(" • Notion Sync → Task 060 (To be implemented)") + logger.info(" • Capsules Publish → Task 070 (To be implemented)") + logger.info(" • CUDA Support → Task 080 (To be implemented)") + logger.info(" • Webhook Bridge → Task 090 (To be implemented)") + + logger.info("\n" + "=" * 60) + logger.info("Status: Repository scaffolding complete") + logger.info("Next: Implement tasks sequentially from .copilot/tasks/") + logger.info("=" * 60) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_structure.py b/tests/test_structure.py new file mode 100644 index 0000000..1ca2d35 --- /dev/null +++ b/tests/test_structure.py @@ -0,0 +1,159 @@ +""" +Test placeholder for CVE Matter-Analysis OS + +This module contains basic tests to validate the repository structure. +Actual tests will be implemented alongside each task (010-090). +""" + +import pytest +from pathlib import Path + + +def test_repository_structure(): + """Test that required directories exist.""" + required_dirs = [ + "src", + "tests", + "prompts", + "capsules", + ".copilot", + ".copilot/tasks", + ".github/workflows", + ] + + for dir_name in required_dirs: + dir_path = Path(dir_name) + assert dir_path.exists(), f"Required directory missing: {dir_name}" + assert dir_path.is_dir(), f"Path is not a directory: {dir_name}" + + +def test_prompts_exist(): + """Test that prompt files exist.""" + required_prompts = [ + "prompts/legendary_lidlift_v14.md", + "prompts/micro_core.txt", + "prompts/nano_core.txt", + ] + + for prompt_file in required_prompts: + file_path = Path(prompt_file) + assert file_path.exists(), f"Required prompt file missing: {prompt_file}" + assert file_path.is_file(), f"Path is not a file: {prompt_file}" + + +def test_capsules_exist(): + """Test that capsule files exist.""" + required_capsules = [ + "capsules/lidlift-v1.json", + "capsules/hmoc-0.2.json", + "capsules/runbooks.json", + ] + + for capsule_file in required_capsules: + file_path = Path(capsule_file) + assert file_path.exists(), f"Required capsule file missing: {capsule_file}" + assert file_path.is_file(), f"Path is not a file: {capsule_file}" + + +def test_task_definitions_exist(): + """Test that all task definition files exist.""" + expected_tasks = [ + "010_ingest_nvd.md", + "020_alignment.md", + "030_arbiter.md", + "040_refractors.md", + "050_evidence.md", + "060_notion_sync.md", + "070_capsules_publish.md", + "080_cuda_support.md", + "090_bridge.md", + ] + + tasks_dir = Path(".copilot/tasks") + for task_file in expected_tasks: + file_path = tasks_dir / task_file + assert file_path.exists(), f"Required task file missing: {file_path}" + assert file_path.is_file(), f"Path is not a file: {file_path}" + + +def test_workflows_exist(): + """Test that GitHub Actions workflows exist.""" + expected_workflows = [ + "ci.yml", + "codeql.yml", + "container-scan.yml", + "notion-sync.yml", + "publish-capsules.yml", + ] + + workflows_dir = Path(".github/workflows") + for workflow_file in expected_workflows: + file_path = workflows_dir / workflow_file + assert file_path.exists(), f"Required workflow file missing: {file_path}" + assert file_path.is_file(), f"Path is not a file: {file_path}" + + +def test_security_files_exist(): + """Test that security files exist.""" + security_files = [ + "SECURITY.md", + "CODEOWNERS", + ".gitignore", + ] + + for security_file in security_files: + file_path = Path(security_file) + assert file_path.exists(), f"Required security file missing: {security_file}" + assert file_path.is_file(), f"Path is not a file: {security_file}" + + +def test_python_config_exists(): + """Test that Python configuration files exist.""" + config_files = [ + "pyproject.toml", + "requirements.txt", + "requirements-dev.txt", + ] + + for config_file in config_files: + file_path = Path(config_file) + assert file_path.exists(), f"Required config file missing: {config_file}" + assert file_path.is_file(), f"Path is not a file: {config_file}" + + +def test_main_module_exists(): + """Test that main module exists.""" + main_module = Path("src/main.py") + assert main_module.exists(), "Main module missing: src/main.py" + assert main_module.is_file(), "Path is not a file: src/main.py" + + +@pytest.mark.slow +def test_main_module_runs(): + """Test that main module can be imported and run.""" + try: + from src import main + + # Don't actually run main() as it may have side effects + assert hasattr(main, "main"), "main() function not found in src.main" + except ImportError as e: + pytest.fail(f"Failed to import src.main: {e}") + + +# Marker examples for future tests +@pytest.mark.integration +def test_integration_placeholder(): + """Placeholder for integration tests (to be implemented).""" + pass + + +@pytest.mark.gpu +def test_gpu_placeholder(): + """Placeholder for GPU tests (to be implemented in Task 080).""" + pass + + +@pytest.mark.heavy +def test_heavy_placeholder(): + """Placeholder for heavy computational tests (to be implemented).""" + pass From baff5285d0d19499d4bdf78dddf20311e19c77a6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 13 Nov 2025 00:41:10 +0000 Subject: [PATCH 5/5] Fix security: Add explicit permissions blocks to GitHub Actions workflows Co-authored-by: igor-holt <125706350+igor-holt@users.noreply.github.com> --- .github/workflows/ci.yml | 11 +++++++++++ .github/workflows/container-scan.yml | 3 +++ .github/workflows/notion-sync.yml | 2 ++ .github/workflows/publish-capsules.yml | 2 ++ 4 files changed, 18 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 985d2d5..9381732 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,8 @@ jobs: lint: name: Lint and Format Check runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout code @@ -62,6 +64,8 @@ jobs: unit-tests: name: Unit Tests runs-on: ubuntu-latest + permissions: + contents: read strategy: matrix: @@ -101,6 +105,8 @@ jobs: name: Integration Tests runs-on: ubuntu-latest needs: unit-tests + permissions: + contents: read steps: - name: Checkout code @@ -135,6 +141,8 @@ jobs: runs-on: ubuntu-latest needs: unit-tests if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + permissions: + contents: read steps: - name: Checkout code @@ -166,6 +174,8 @@ jobs: validate-configs: name: Validate Configuration Files runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout code @@ -216,6 +226,7 @@ jobs: runs-on: ubuntu-latest needs: [lint, unit-tests, integration-tests, validate-configs] if: always() + permissions: {} steps: - name: Check all jobs diff --git a/.github/workflows/container-scan.yml b/.github/workflows/container-scan.yml index 708fea3..3b5b8d7 100644 --- a/.github/workflows/container-scan.yml +++ b/.github/workflows/container-scan.yml @@ -100,6 +100,8 @@ jobs: scan-filesystem: name: Scan Repository Filesystem runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout code @@ -126,6 +128,7 @@ jobs: runs-on: ubuntu-latest needs: [scan-image, scan-filesystem] if: always() + permissions: {} steps: - name: Check scan results diff --git a/.github/workflows/notion-sync.yml b/.github/workflows/notion-sync.yml index b00cf78..bb231ea 100644 --- a/.github/workflows/notion-sync.yml +++ b/.github/workflows/notion-sync.yml @@ -26,6 +26,8 @@ jobs: # Only run on main branch pushes or manual triggers if: github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' + permissions: + contents: read steps: - name: Checkout code diff --git a/.github/workflows/publish-capsules.yml b/.github/workflows/publish-capsules.yml index befd2e8..af8af78 100644 --- a/.github/workflows/publish-capsules.yml +++ b/.github/workflows/publish-capsules.yml @@ -25,6 +25,8 @@ jobs: validate: name: Validate Capsules runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout code