diff --git a/.env.example b/.env.example index 65e1285..b466144 100644 --- a/.env.example +++ b/.env.example @@ -195,3 +195,9 @@ SOCIAL_TOKEN_REVERIFICATION_INTERVAL=60000 SOCIAL_TOKEN_CACHE_PREFIX=social_token: STELLAR_MAX_RETRIES=3 STELLAR_RETRY_DELAY=1000 + +# PII Scrubbing Configuration (GDPR/CCPA Compliance) +PII_SCRUBBING_SALT=your-secure-random-salt-for-pii-hashing-min-32-bytes +INACTIVE_RETENTION_YEARS=3 +PII_SCRUBBING_ENABLED=true +PII_SCRUBBING_CRON_SCHEDULE=0 2 * * 0 # Weekly on Sunday at 2 AM diff --git a/Dockerfile b/Dockerfile index 46271b5..438b1c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,6 +75,10 @@ COPY --chown=nodejs:nodejs index.js ./ COPY --chown=nodejs:nodejs knexfile.js ./ COPY --chown=nodejs:nodejs .env.example ./.env.example +# Copy Knex migrations for initContainer +COPY --chown=nodejs:nodejs migrations ./migrations +COPY --chown=nodejs:nodejs scripts ./scripts + # Create data directory with proper permissions RUN mkdir -p /app/data && \ chown nodejs:nodejs /app/data diff --git a/KUBERNETES_MIGRATION_AUTOMATION.md b/KUBERNETES_MIGRATION_AUTOMATION.md new file mode 100644 index 0000000..6e5b3dd --- /dev/null +++ b/KUBERNETES_MIGRATION_AUTOMATION.md @@ -0,0 +1,527 @@ +# Kubernetes Database Migration Automation + +This document describes the automated database schema migration system for the SubStream Protocol Backend, which ensures safe and reliable database updates during Kubernetes deployments. + +## Overview + +The migration automation system eliminates the need for manual database migrations during deployments. It integrates with Kubernetes to: +- Automatically run database migrations before application pods start +- Prevent application deployment if migrations fail +- Ensure database schema and application code are always synchronized +- Support distributed execution without race conditions +- Handle long-running migrations gracefully + +## Architecture + +### Components + +1. **Migration Script** (`scripts/migrate-init-container.js`) + - Idempotent migration runner for Kubernetes initContainers + - Distributed locking to prevent concurrent migrations + - Timeout protection for long-running operations + - Comprehensive logging and error handling + +2. **Dockerfile Updates** + - Packages Knex schema and migration files into the Docker image + - Ensures migrations are available at runtime + +3. **Helm Chart Configuration** + - Supports two deployment strategies: initContainer or Job hooks + - Configurable timeouts and resource limits + - Vault integration for secure credential management + +4. **Vault Integration** (Optional) + - Secure credential management for migration operations + - Dynamic credential provisioning + - Role-based access control + +## Deployment Strategies + +### Strategy 1: initContainer (Default) + +The initContainer runs migrations before the main application container starts. + +**Advantages:** +- Simple and straightforward +- Migrations complete before any application pod starts +- Failed migrations prevent pod startup automatically + +**Configuration:** +```yaml +migration: + enabled: true + strategy: "initContainer" + timeout: 1800 + lockTimeout: 300 +``` + +**How it works:** +1. Kubernetes creates a pod with the initContainer +2. initContainer runs the migration script +3. If migration succeeds, the main container starts +4. If migration fails, the pod fails and Kubernetes retries + +### Strategy 2: Helm Job Hooks + +The migration runs as a separate Kubernetes Job using Helm hooks. + +**Advantages:** +- Migration runs once per Helm release (not per pod) +- Better for large-scale deployments with many replicas +- Can be configured to run pre-install or pre-upgrade + +**Configuration:** +```yaml +migration: + enabled: true + strategy: "job" + timeout: 1800 + lockTimeout: 300 +``` + +**How it works:** +1. Helm creates a Job before the Deployment +2. Job runs the migration script +3. If migration succeeds, Helm proceeds with Deployment +4. If migration fails, Helm aborts the upgrade + +## Quick Start + +### 1. Update Dockerfile + +The Dockerfile has been updated to include migration files: + +```dockerfile +# Copy Knex migrations for initContainer +COPY --chown=nodejs:nodejs migrations ./migrations +COPY --chown=nodejs:nodejs scripts ./scripts +``` + +### 2. Build and Push Image + +```bash +docker build -t substream/backend:latest . +docker push substream/backend:latest +``` + +### 3. Configure Helm Values + +Update `helm/substream-backend/values.yaml`: + +```yaml +migration: + enabled: true + strategy: "initContainer" # or "job" + timeout: 1800 + lockTimeout: 300 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" +``` + +### 4. Deploy + +```bash +helm upgrade substream-backend ./helm/substream-backend \ + --namespace production \ + --values values-production.yaml +``` + +### 5. Monitor + +```bash +# Watch migration logs +kubectl logs -f deployment/substream-backend -c migration -n production + +# Check migration status +kubectl describe pod -n production +``` + +## Features + +### Idempotent Execution + +The migration script can be run multiple times safely: +- Checks if migrations are already applied +- Skips already-completed migrations +- Uses database-level locking to prevent race conditions + +### Distributed Locking + +Prevents concurrent migrations across multiple pods: +- Lock table in the database +- Automatic lock expiration +- Lock acquisition with timeout +- Automatic lock release on completion + +### Timeout Protection + +Configurable timeouts prevent indefinite hangs: +- Migration timeout (default: 30 minutes) +- Lock acquisition timeout (default: 5 minutes) +- Configurable via environment variables +- Warnings for long-running operations + +### Comprehensive Logging + +Detailed logging for debugging and monitoring: +- Migration start/end timestamps +- Individual migration names and status +- Lock acquisition/release events +- Error details and stack traces +- Progress tracking for long operations + +### Failure Handling + +Automatic failure handling to prevent deployment: +- Exit code 0 on success +- Exit code 1 on failure (prevents pod startup) +- Automatic rollback on failure +- Lock cleanup on error + +## Configuration + +### Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `DATABASE_FILENAME` | Path to SQLite database file | `/app/data/substream.db` | +| `MIGRATION_TIMEOUT` | Migration timeout in milliseconds | `1800000` (30 min) | +| `MIGRATION_LOCK_TIMEOUT` | Lock acquisition timeout in milliseconds | `300000` (5 min) | +| `POD_NAME` | Kubernetes pod name (auto-injected) | - | +| `NAMESPACE` | Kubernetes namespace (auto-injected) | - | +| `HOSTNAME` | Node hostname (auto-injected) | - | + +### Helm Values + +```yaml +migration: + enabled: true # Enable/disable migrations + strategy: "initContainer" # "initContainer" or "job" + timeout: 1800 # Migration timeout in seconds + lockTimeout: 300 # Lock timeout in seconds + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" + vault: + enabled: false # Enable Vault integration + role: "substream-migration" # Vault role name + secretPath: "database/creds/migration" + address: "http://vault:8200" +``` + +## Vault Integration + +For secure credential management, enable Vault integration: + +### Setup + +1. Configure Vault policy (see [VAULT_MIGRATION_SETUP.md](./docs/VAULT_MIGRATION_SETUP.md)) +2. Enable Vault in Helm values: + +```yaml +migration: + vault: + enabled: true + role: "substream-migration" + secretPath: "database/creds/migration" + address: "http://vault:8200" +``` + +3. Annotate service account for Vault authentication + +## Writing Migrations + +### Basic Migration + +```javascript +// migrations/knex/016_add_new_column.js +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + table.string('new_column').nullable(); + }); +}; + +exports.down = function(knex) { + return knex.schema.table('users', (table) => { + table.dropColumn('new_column'); + }); +}; +``` + +### Backwards-Compatible Migration + +Always write backwards-compatible migrations: + +```javascript +exports.up = function(knex) { + // Add nullable column + return knex.schema.table('users', (table) => { + table.string('email').nullable(); + }); +}; +``` + +See [BACKWARDS_COMPATIBLE_MIGRATIONS.md](./docs/BACKWARDS_COMPATIBLE_MIGRATIONS.md) for detailed guidelines. + +### Long-Running Migration + +For operations that take a long time: + +```javascript +exports.up = async function(knex) { + const batchSize = 10000; + let processed = 0; + + while (true) { + const updated = await knex('large_table') + .whereNull('new_column') + .limit(batchSize) + .update({ new_column: 'value' }); + + if (updated === 0) break; + + processed += updated; + console.log(`Processed ${processed} records`); + } +}; +``` + +See [LONG_RUNNING_MIGRATIONS.md](./docs/LONG_RUNNING_MIGRATIONS.md) for strategies. + +## Troubleshooting + +### Migration Fails + +**Symptoms:** +- Pod fails to start +- Job fails with error + +**Steps:** +1. Check logs: `kubectl logs -c migration -n production` +2. Review [Migration Failure Runbook](./docs/MIGRATION_FAILURE_RUNBOOK.md) +3. Check database connectivity +4. Verify migration file syntax +5. Check for lock contention + +### Lock Timeout + +**Symptoms:** +- "Timeout waiting for migration lock" +- Migration waits indefinitely + +**Steps:** +1. Check for existing locks in `_migration_locks` table +2. Manually release stale locks (emergency only) +3. Increase `lockTimeout` in values.yaml +4. Check for stuck migration jobs + +### Migration Timeout + +**Symptoms:** +- "Migration timeout after Xms" +- Long-running migration fails + +**Steps:** +1. Increase `timeout` in values.yaml +2. Optimize migration (use batches) +3. Use background job strategy +4. See [Long-Running Migrations](./docs/LONG_RUNNING_MIGRATIONS.md) + +## Monitoring + +### Kubernetes + +```bash +# Watch migration logs +kubectl logs -f deployment/substream-backend -c migration -n production + +# Check pod status +kubectl get pods -n production -l app=substream-backend + +# Describe pod for events +kubectl describe pod -n production +``` + +### Application Logs + +The migration script logs to stdout/stderr: +- `[InitContainer]` prefix for all messages +- Timestamps for all operations +- Progress tracking for long operations +- Error details with stack traces + +### Metrics + +Consider adding metrics for: +- Migration duration +- Migration success/failure rate +- Lock acquisition time +- Number of migrations run + +## Best Practices + +### Before Deployment + +1. **Test in Staging**: Always test migrations in staging first +2. **Backup Database**: Ensure recent backups are available +3. **Review Migration**: Check migration file for potential issues +4. **Estimate Duration**: Test with production-like data +5. **Plan Rollback**: Have a rollback strategy ready + +### During Deployment + +1. **Monitor Logs**: Watch migration logs in real-time +2. **Check Progress**: Verify migrations are progressing +3. **Watch Resources**: Monitor database and pod resources +4. **Be Ready**: Have runbook and team on standby + +### After Deployment + +1. **Verify Application**: Ensure application works correctly +2. **Check Data Integrity**: Verify data consistency +3. **Monitor Performance**: Watch for performance issues +4. **Document**: Document any issues or learnings + +## Security Considerations + +### Credentials + +- Never hardcode database credentials +- Use Vault for credential management +- Rotate credentials regularly +- Use least-privilege access + +### Database Access + +- Migration role should have minimal required privileges +- Revoke migration credentials after use +- Audit all migration operations +- Use read-only credentials for verification + +### Network + +- Use TLS for database connections +- Restrict database access to migration pods +- Use network policies to limit access +- Monitor for unauthorized access + +## CI/CD Integration + +### GitHub Actions Example + +```yaml +name: Deploy to Production + +on: + push: + branches: [main] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Build Docker Image + run: | + docker build -t substream/backend:${{ github.sha }} . + docker push substream/backend:${{ github.sha }} + + - name: Deploy with Helm + run: | + helm upgrade substream-backend ./helm/substream-backend \ + --namespace production \ + --set image.tag=${{ github.sha }} \ + --set migration.enabled=true +``` + +### GitLab CI Example + +```yaml +deploy: + stage: deploy + script: + - docker build -t substream/backend:$CI_COMMIT_SHA . + - docker push substream/backend:$CI_COMMIT_SHA + - helm upgrade substream-backend ./helm/substream-backend + --namespace production + --set image.tag=$CI_COMMIT_SHA + --set migration.enabled=true + only: + - main +``` + +## Rollback Procedure + +If a deployment fails: + +1. **Stop Deployment** + ```bash + helm rollback substream-backend -n production + ``` + +2. **Check Migration Status** + ```bash + kubectl exec -it -n production -- npx knex migrate:list + ``` + +3. **Rollback Migrations** (if needed) + ```bash + kubectl exec -it -n production -- npx knex migrate:rollback + ``` + +4. **Verify System** + ```bash + kubectl get pods -n production + kubectl logs deployment/substream-backend -n production + ``` + +See [Migration Failure Runbook](./docs/MIGRATION_FAILURE_RUNBOOK.md) for detailed procedures. + +## Documentation + +- [Migration Failure Runbook](./docs/MIGRATION_FAILURE_RUNBOOK.md) - Handling failed migrations +- [Backwards-Compatible Migrations](./docs/BACKWARDS_COMPATIBLE_MIGRATIONS.md) - Writing safe migrations +- [Long-Running Migrations](./docs/LONG_RUNNING_MIGRATIONS.md) - Handling slow operations +- [Vault Migration Setup](./docs/VAULT_MIGRATION_SETUP.md) - Vault integration guide + +## Acceptance Criteria + +### Acceptance 1: Autonomous and Secure Updates +✅ Database schemas are updated automatically during Kubernetes deployment +✅ Migrations run before application pods start +✅ Failed migrations prevent application deployment +✅ Secure credential management via Vault (optional) + +### Acceptance 2: Failed Migration Protection +✅ Failed migrations prevent pod startup +✅ Exit code 1 on failure triggers Kubernetes retry logic +✅ Automatic rollback on failure +✅ Comprehensive error logging + +### Acceptance 3: Helm Hooks and Distributed Execution +✅ Supports both initContainer and Job hook strategies +✅ Database-level locking prevents race conditions +✅ Idempotent execution safe for multiple pods +✅ Configurable timeouts and resources + +## Support + +For issues or questions: +- Review the troubleshooting section +- Check the detailed runbooks +- Consult the Knex.js documentation +- Contact the DevOps team + +## License + +MIT diff --git a/PII_SCRUBBING_README.md b/PII_SCRUBBING_README.md new file mode 100644 index 0000000..862d391 --- /dev/null +++ b/PII_SCRUBBING_README.md @@ -0,0 +1,506 @@ +# PII Scrubbing System - Right to be Forgotten + +## Overview + +The PII (Personally Identifiable Information) Scrubbing System implements GDPR/CCPA compliant data deletion for the SubStream Protocol. It automatically scrubs user personal data while preserving financial records for tax compliance. + +## Features + +- **Cryptographic Hashing**: One-way SHA-256 HMAC with secure salt prevents reversal +- **Financial Data Preservation**: Billing events retained with anonymized user identity +- **Automated Retention Policy**: Inactive users scrubbed after 3 years +- **Redis Cache Scrubbing**: Removes PII from all cache layers +- **Merchant Webhooks**: Notifies creators of user data deletion +- **Comprehensive Audit Trail**: Immutable logging for compliance +- **Deep Integration Tests**: Verifies users cannot be identified post-scrub + +## Architecture + +### Components + +1. **PIIScrubbingService** (`src/services/piiScrubbingService.js`) + - Core scrubbing logic + - Cryptographic hashing + - Database operations + - Redis cache clearing + - Webhook notifications + +2. **Compliance API** (`routes/compliance.js`) + - POST `/api/v1/compliance/forget` - User-initiated deletion + - GET `/api/v1/compliance/forget/:walletAddress/status` - Verification + - POST `/api/v1/compliance/forget/batch` - Admin batch scrubbing + - GET `/api/v1/compliance/audit` - Audit log retrieval + - POST `/api/v1/compliance/export` - Data portability + +3. **Background Worker** (`workers/piiScrubbingWorker.js`) + - Automated scrubbing of inactive users + - Configurable retention period + - Dry-run support + +4. **Integration Tests** (`piiScrubbing.test.js`) + - Cryptographic hashing verification + - Database scrubbing validation + - Identification prevention tests + - Audit logging verification + +## Quick Start + +### 1. Configuration + +Add to your `.env` file: + +```bash +# PII Scrubbing Configuration +PII_SCRUBBING_SALT=your-secure-random-salt-min-32-bytes +INACTIVE_RETENTION_YEARS=3 +PII_SCRUBBING_ENABLED=true +PII_SCRUBBING_CRON_SCHEDULE=0 2 * * 0 +``` + +**Important**: Generate a secure salt: + +```bash +node -e "console.log(require('crypto').randomBytes(32).toString('hex'))" +``` + +### 2. Integrate API Routes + +Add to your main application (`index.js`): + +```javascript +const complianceRoutes = require('./routes/compliance'); +app.use('/api/v1/compliance', complianceRoutes); +``` + +### 3. Run Manual Scrubbing + +```bash +# Scrub inactive users (3+ years) +npm run pii-scrub + +# Dry run to count inactive users +npm run pii-scrub:dry-run + +# Custom retention period +node workers/piiScrubbingWorker.js 5 +``` + +### 4. Run Tests + +```bash +npm run test:pii +``` + +## API Endpoints + +### POST /api/v1/compliance/forget + +Initiates the Right to be Forgotten process for a user. + +**Request Body:** +```json +{ + "walletAddress": "GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ", + "reason": "user_request", + "requestedBy": "user" +} +``` + +**Response:** +```json +{ + "success": true, + "scrubId": "uuid-v4", + "walletAddress": "GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ", + "duration": 1234, + "dbResult": { + "success": true, + "tablesScrubbed": [...] + }, + "redisResult": { + "success": true, + "keysScrubbed": 5 + }, + "webhookResult": { + "success": true, + "webhooksSent": 2 + } +} +``` + +### GET /api/v1/compliance/forget/:walletAddress/status + +Check the scrubbing status of a user. + +**Response:** +```json +{ + "success": true, + "walletAddress": "GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ", + "anonymizedAddress": "GD5DQ6ZQZ_abc123...", + "tables": { + "subscriptions": { + "found": true, + "emailScrubbed": true, + "addressAnonymized": true + } + }, + "isScrubbed": true +} +``` + +### POST /api/v1/compliance/forget/batch + +Admin-only endpoint to batch scrub inactive users. + +**Request Body:** +```json +{ + "years": 3, + "dryRun": false +} +``` + +**Response:** +```json +{ + "success": true, + "batchId": "uuid-v4", + "totalUsers": 100, + "successful": 95, + "failed": 5, + "errors": [...] +} +``` + +### POST /api/v1/compliance/export + +Export a user's data for GDPR data portability. + +**Request Body:** +```json +{ + "walletAddress": "GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ" +} +``` + +**Response:** +```json +{ + "success": true, + "exportId": "uuid-v4", + "walletAddress": "GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ", + "exportedAt": "2026-04-26T14:00:00Z", + "subscriptions": [...], + "comments": [...], + "auditLogs": [...] +} +``` + +## Database Tables Scrubbed + +The following PII fields are scrubbed across these tables: + +| Table | Fields Scrubbed | Financial Data Preserved | +|-------|----------------|-------------------------| +| subscriptions | user_email, wallet_address | balance, daily_spend, creator_id | +| creator_audit_logs | ip_address | All other fields | +| api_key_audit_logs | ip_address | All other fields | +| data_export_tracking | requester_email | All other fields | +| privacy_preferences | share_email_with_merchants | All other fields | +| comments | user_address | content, timestamps | +| leaderboard_entries | fan_address | scores, timestamps | +| social_tokens | user_address | tokens, timestamps | + +## Cryptographic Security + +### Hashing Algorithm + +- **Algorithm**: SHA-256 HMAC +- **Salt**: 32-byte secure salt from environment variable +- **Format**: Hexadecimal string (64 characters) + +### Wallet Address Anonymization + +**Original**: `GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ` + +**Anonymized**: `GD5DQ6ZQZ_abc123def456...` + +- First 8 characters preserved for debugging +- Remaining characters replaced with hash +- Irreversible without salt + +### Email Anonymization + +**Original**: `user@example.com` + +**Anonymized**: `scrubbed_[hash]@anon.example.com` + +- Full email replaced with hash +- Domain standardized to anon.example.com +- Irreversible without salt + +## Redis Cache Scrubbing + +The following Redis key patterns are scrubbed: + +- `user:{walletAddress}:*` +- `profile:{walletAddress}:*` +- `subscription:{walletAddress}:*` +- `creator:{walletAddress}:*` +- `session:{walletAddress}:*` +- `cache:{walletAddress}:*` + +## Merchant Webhooks + +When a user invokes their right to be forgotten, affected merchants receive a webhook: + +**Webhook Payload:** +```json +{ + "event": "user.forget", + "timestamp": "2026-04-26T14:00:00Z", + "scrub_id": "uuid-v4", + "data": { + "anonymized_wallet_address": "GD5DQ6ZQZ_abc123...", + "reason": "user_request", + "scrubbed_at": "2026-04-26T14:00:00Z" + } +} +``` + +**Security**: Webhooks are signed with the merchant's webhook secret. + +## Automated Retention Policy + +### Configuration + +- **Default Retention**: 3 years of inactivity +- **Cron Schedule**: Weekly on Sunday at 2 AM +- **Dry Run**: Supported for testing + +### Cron Job Setup + +Add to your crontab: + +```bash +# Weekly PII scrubbing of inactive users +0 2 * * 0 cd /path/to/app && npm run pii-scrub +``` + +Or use a process manager like PM2: + +```javascript +// ecosystem.config.js +module.exports = { + apps: [{ + name: 'pii-scrubbing-worker', + script: './workers/piiScrubbingWorker.js', + args: '3', + cron_restart: '0 2 * * 0', + env: { + NODE_ENV: 'production' + } + }] +}; +``` + +## Audit Logging + +All scrubbing operations are logged to `creator_audit_logs`: + +**Audit Entry Structure:** +```json +{ + "id": "uuid-v4", + "action_type": "pii_scrub", + "entity_type": "user", + "entity_id": "GD5DQ6ZQZ_abc123...", + "timestamp": "2026-04-26T14:00:00Z", + "ip_address": "system", + "metadata_json": { + "scrubId": "uuid-v4", + "original_wallet_hash": "sha256-hash", + "reason": "user_request", + "requestedBy": "user", + "dbResult": {...}, + "redisResult": {...}, + "webhookResult": {...} + } +} +``` + +**Retention**: 5 years for compliance + +## Testing + +### Run Integration Tests + +```bash +npm run test:pii +``` + +### Test Coverage + +The test suite verifies: + +- **Cryptographic Security**: Hash consistency and uniqueness +- **Database Scrubbing**: All PII fields across all tables +- **Identification Prevention**: Users cannot be identified post-scrub +- **Financial Data Preservation**: Tax records remain intact +- **Audit Logging**: Complete audit trail creation +- **Idempotency**: Safe to run multiple times +- **Inactive User Detection**: Correct identification of stale accounts +- **Batch Processing**: Multiple users scrubbed correctly + +### Verification Endpoint + +After scrubbing, verify the operation: + +```bash +curl -X GET \ + http://localhost:3000/api/v1/compliance/forget/GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ/status +``` + +## Compliance + +### GDPR (General Data Protection Regulation) + +- **Article 17**: Right to erasure ("right to be forgotten") ✅ +- **Article 20**: Right to data portability ✅ +- **Recital 39**: Data minimization ✅ + +### CCPA (California Consumer Privacy Act) + +- **Right to delete** ✅ +- **Right to know** ✅ +- **Right to opt-out** ✅ + +### Other Regulations + +- **LGPD** (Brazil) - Right to deletion ✅ +- **POPIA** (South Africa) - Right to deletion ✅ + +## Security Considerations + +### Salt Management + +- **Storage**: Environment variable (never in code) +- **Generation**: Cryptographically random (32 bytes) +- **Rotation**: Rotate annually or on compromise +- **Backup**: Secure backup with access controls + +### Access Control + +- **API Endpoints**: Admin authentication for batch operations +- **Webhook Secrets**: Per-merchant secrets for verification +- **Audit Logs**: Admin-only access to audit trail + +### Data Recovery + +- **Irreversible**: One-way hashing prevents reversal +- **Backups**: Scrubbed data propagates to backups +- **Legal Holds**: Separate process for legal preservation + +## Troubleshooting + +### Scrubbing Fails + +**Symptoms**: API returns 500 error + +**Solutions**: +1. Check database connectivity +2. Verify salt is configured +3. Check Redis connection +4. Review audit logs for specific errors + +### Webhooks Not Sent + +**Symptoms**: Merchants not notified + +**Solutions**: +1. Verify webhook URLs in creators table +2. Check webhook secrets +3. Review webhook service logs +4. Test webhook endpoint manually + +### Redis Keys Not Scrubbed + +**Symptoms**: Old data in cache + +**Solutions**: +1. Verify Redis connection +2. Check key patterns match +3. Manually flush Redis if needed +4. Verify Redis client configuration + +### Audit Logs Missing + +**Symptoms**: No audit entries created + +**Solutions**: +1. Check audit_log_service configuration +2. Verify database write permissions +3. Check for database errors +4. Review service logs + +## Performance + +### Scrubbing Duration + +- **Single User**: ~1-2 seconds +- **100 Users**: ~30-60 seconds +- **1000 Users**: ~5-10 minutes + +### Optimization Tips + +- Use batch operations for large datasets +- Schedule during low-traffic periods +- Monitor database performance during scrubbing +- Use read replicas for audit queries + +## Monitoring + +### Key Metrics + +- Scrubbing operations per day +- Average scrubbing duration +- Failed scrubbing attempts +- Webhook delivery rate +- Redis keys scrubbed + +### Alerts + +Configure alerts for: +- High failure rate (>5%) +- Long scrubbing duration (>10 minutes) +- Webhook delivery failures +- Database errors during scrubbing + +## Documentation + +- [Privacy Policy - Data Retention](./docs/PRIVACY_POLICY_DATA_RETENTION.md) +- [GDPR Compliance Guide](./docs/GDPR_COMPLIANCE.md) +- [API Documentation](./docs/API_DOCUMENTATION.md) + +## Support + +For issues or questions: +- **Email**: privacy@substream.protocol +- **GitHub Issues**: [SubStream Protocol Backend](https://github.com/SubStream-Protocol/SubStream-Protocol-Backend/issues) +- **Documentation**: [Full Documentation](./docs) + +## License + +MIT + +## Changelog + +### v1.0.0 (April 26, 2026) +- Initial release +- Cryptographic PII scrubbing +- Automated retention policy +- Redis cache scrubbing +- Merchant webhook notifications +- Comprehensive audit logging +- Deep integration tests diff --git a/docs/BACKWARDS_COMPATIBLE_MIGRATIONS.md b/docs/BACKWARDS_COMPATIBLE_MIGRATIONS.md new file mode 100644 index 0000000..4d6d064 --- /dev/null +++ b/docs/BACKWARDS_COMPATIBLE_MIGRATIONS.md @@ -0,0 +1,534 @@ +# Backwards-Compatible Database Migrations + +This document provides guidelines and patterns for writing backwards-compatible database migrations that allow old and new application versions to coexist during deployments. + +## Overview + +In a Kubernetes environment with rolling updates, old pods continue running while new pods are starting. Migrations must be backwards-compatible to ensure: +- Old pods can still read/write data with the old schema +- New pods can read/write data with the new schema +- No data loss or corruption occurs during the transition + +## Core Principles + +1. **Never break existing contracts** +2. **Add before removing** +3. **Use nullable columns for new fields** +4. **Default values for new required fields** +5. **Two-phase deployment for breaking changes** + +--- + +## Migration Patterns + +### Pattern 1: Adding a New Column + +**❌ Wrong (Breaking):** +```javascript +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + table.string('email').notNullable(); // Old pods will fail + }); +}; +``` + +**✅ Correct (Backwards-Compatible):** +```javascript +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + table.string('email').nullable(); // Old pods ignore it + }); +}; + +exports.down = function(knex) { + return knex.schema.table('users', (table) => { + table.dropColumn('email'); + }); +}; +``` + +**Deployment Steps:** +1. Deploy migration (adds nullable column) +2. Deploy new application code (uses new column) +3. Backfill data for existing rows +4. Deploy migration to make column non-nullable (optional) + +### Pattern 2: Adding a New Table + +**✅ Correct:** +```javascript +exports.up = function(knex) { + return knex.schema.createTable('audit_logs', (table) => { + table.increments('id').primary(); + table.integer('user_id').unsigned(); + table.string('action'); + table.timestamp('created_at').defaultTo(knex.fn.now()); + + table.foreign('user_id').references('users.id'); + }); +}; + +exports.down = function(knex) { + return knex.schema.dropTable('audit_logs'); +}; +``` + +**Deployment Steps:** +1. Deploy migration (creates new table) +2. Deploy new application code (writes to new table) +3. Old pods ignore the new table (safe) + +### Pattern 3: Renaming a Column + +**❌ Wrong (Breaking):** +```javascript +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + table.renameColumn('username', 'display_name'); // Old pods will fail + }); +}; +``` + +**✅ Correct (Two-Phase):** + +**Phase 1: Add new column** +```javascript +// Migration 001_add_display_name.js +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + table.string('display_name').nullable(); + }); +}; + +exports.down = function(knex) { + return knex.schema.table('users', (table) => { + table.dropColumn('display_name'); + }); +}; +``` + +**Phase 2: Backfill data** +```javascript +// Migration 002_backfill_display_name.js +exports.up = async function(knex) { + await knex('users') + .whereNull('display_name') + .update({ + display_name: knex.raw('username') + }); +}; + +exports.down = async function(knex) { + // No-op +}; +``` + +**Phase 3: Update application code** +- Deploy new version that reads from `display_name` +- Keep writing to both `username` and `display_name` + +**Phase 4: Remove old column** +```javascript +// Migration 003_remove_username.js +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + table.dropColumn('username'); + }); +}; + +exports.down = function(knex) { + return knex.schema.table('users', (table) => { + table.string('username').nullable(); + }); +}; +``` + +### Pattern 4: Changing Column Type + +**❌ Wrong (Breaking):** +```javascript +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + table.integer('age').alter(); // Old pods expect string + }); +}; +``` + +**✅ Correct (Two-Phase):** + +**Phase 1: Add new column** +```javascript +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + table.integer('age_new').nullable(); + }); +}; +``` + +**Phase 2: Backfill and convert** +```javascript +exports.up = async function(knex) { + await knex('users') + .whereNull('age_new') + .update({ + age_new: knex.raw('CAST(age AS INTEGER)') + }); +}; +``` + +**Phase 3: Update application code** +- Deploy new version that reads from `age_new` +- Keep writing to both columns + +**Phase 4: Remove old column** +```javascript +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + table.dropColumn('age'); + }); + + return knex.schema.table('users', (table) => { + table.renameColumn('age_new', 'age'); + }); +}; +``` + +### Pattern 5: Adding a Foreign Key + +**✅ Correct:** +```javascript +exports.up = function(knex) { + return knex.schema.table('posts', (table) => { + table.integer('author_id').unsigned().nullable(); + table + .foreign('author_id') + .references('users.id') + .onDelete('SET NULL'); // Safe deletion + }); +}; + +exports.down = function(knex) { + return knex.schema.table('posts', (table) => { + table.dropForeign(['author_id']); + table.dropColumn('author_id'); + }); +}; +``` + +**Deployment Steps:** +1. Deploy migration (adds nullable foreign key) +2. Deploy new application code (uses foreign key) +3. Backfill data +4. Make column non-nullable (optional) + +### Pattern 6: Adding an Index + +**✅ Correct (Non-Blocking):** +```javascript +exports.up = function(knex) { + // For PostgreSQL, use CONCURRENTLY to avoid locking + return knex.raw(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_users_email + ON users(email) + `); +}; + +exports.down = function(knex) { + return knex.raw('DROP INDEX CONCURRENTLY IF EXISTS idx_users_email'); +}; +``` + +**For SQLite (no CONCURRENTLY):** +```javascript +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + table.index('email'); + }); +}; +``` + +**Note:** SQLite index creation is fast and typically doesn't require special handling. + +### Pattern 7: Changing Default Values + +**✅ Correct:** +```javascript +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + // New rows get the new default + table.timestamp('created_at').defaultTo(knex.fn.now()).alter(); + }); +}; +``` + +**Note:** This only affects new rows. Old rows are unaffected. + +### Pattern 8: Adding a Constraint + +**❌ Wrong (Breaking):** +```javascript +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + table.unique('email'); // Old data might violate this + }); +}; +``` + +**✅ Correct (Two-Phase):** + +**Phase 1: Clean up data** +```javascript +exports.up = async function(knex) { + // Remove duplicates + await knex.raw(` + DELETE FROM users u1 + WHERE id NOT IN ( + SELECT MIN(id) + FROM users u2 + GROUP BY email + ) + `); +}; +``` + +**Phase 2: Add constraint** +```javascript +exports.up = function(knex) { + return knex.schema.table('users', (table) => { + table.unique('email'); + }); +}; +``` + +--- + +## Data Migration Strategies + +### Strategy 1: Backfill in Migration + +```javascript +exports.up = async function(knex) { + // Add column + await knex.schema.table('users', (table) => { + table.string('full_name').nullable(); + }); + + // Backfill data + await knex('users') + .whereNull('full_name') + .update({ + full_name: knex.raw('CONCAT(first_name, " ", last_name)') + }); +}; +``` + +### Strategy 2: Backfill in Application Code + +For large datasets, backfill in the application: + +```javascript +// In the application +async function backfillUserNames() { + const batchSize = 1000; + let offset = 0; + + while (true) { + const users = await knex('users') + .whereNull('full_name') + .limit(batchSize) + .offset(offset); + + if (users.length === 0) break; + + for (const user of users) { + await knex('users') + .where('id', user.id) + .update({ + full_name: `${user.first_name} ${user.last_name}` + }); + } + + offset += batchSize; + } +} +``` + +### Strategy 3: Backfill via Background Job + +```javascript +// Create a job to backfill data +exports.up = async function(knex) { + await knex.schema.table('users', (table) => { + table.string('full_name').nullable(); + }); + + // Queue background job + await knex('background_jobs').insert({ + type: 'backfill_user_names', + status: 'pending', + created_at: new Date() + }); +}; +``` + +--- + +## Deployment Checklist + +Before deploying a migration, verify: + +- [ ] Migration is idempotent (can run multiple times safely) +- [ ] Migration has a rollback script +- [ ] New columns are nullable or have default values +- [ ] No columns are dropped without a two-phase process +- [ ] No constraints are added without data cleanup +- [ ] Indexes use CONCURRENTLY (PostgreSQL) or are fast (SQLite) +- [ ] Foreign keys use ON DELETE SET NULL or similar safe policies +- [ ] Migration tested in staging environment +- [ ] Application code updated to handle both old and new schema +- [ ] Rollback procedure documented + +--- + +## Example: Complete Migration Workflow + +### Scenario: Add `status` column to `subscriptions` table + +**Step 1: Migration 1 - Add nullable column** +```javascript +// migrations/knex/015_add_subscription_status.js +exports.up = function(knex) { + return knex.schema.table('subscriptions', (table) => { + table.string('status').defaultTo('active').nullable(); + }); +}; + +exports.down = function(knex) { + return knex.schema.table('subscriptions', (table) => { + table.dropColumn('status'); + }); +}; +``` + +**Step 2: Deploy migration** +```bash +helm upgrade substream-backend ./helm/substream-backend \ + --namespace production \ + --set migration.enabled=true +``` + +**Step 3: Update application code** +- Update application to read `status` column +- Update application to write `status` column +- Deploy new version + +**Step 4: Verify** +- Check that both old and new pods work +- Verify data integrity +- Monitor for errors + +**Step 5: (Optional) Make column non-nullable** +```javascript +// migrations/knex/016_make_status_required.js +exports.up = async function(knex) { + // Ensure all rows have a value + await knex('subscriptions') + .whereNull('status') + .update({ status: 'active' }); + + // Make column required + return knex.schema.table('subscriptions', (table) => { + table.string('status').defaultTo('active').alter(); + }); +}; + +exports.down = function(knex) { + return knex.schema.table('subscriptions', (table) => { + table.string('status').nullable().alter(); + }); +}; +``` + +--- + +## Testing Backwards Compatibility + +### Unit Tests + +```javascript +describe('Migration backwards compatibility', () => { + it('should work with old schema', async () => { + // Test that old application code works with new schema + const oldApp = require('./old-app'); + await oldApp.createUser({ name: 'John' }); + }); + + it('should work with new schema', async () => { + // Test that new application code works with new schema + const newApp = require('./new-app'); + await newApp.createUser({ name: 'John', email: 'john@example.com' }); + }); +}); +``` + +### Integration Tests + +```javascript +describe('Deployment compatibility', () => { + it('should handle mixed pod versions', async () => { + // Simulate old and new pods running simultaneously + const oldPod = await startOldPod(); + const newPod = await startNewPod(); + + // Both should work + await oldPod.writeData({ field: 'value' }); + await newPod.readData(); + + await oldPod.stop(); + await newPod.stop(); + }); +}); +``` + +--- + +## Common Pitfalls + +### Pitfall 1: Assuming Migration Completes Instantly + +**Problem:** Application code deployed before migration completes. + +**Solution:** Use initContainer or Helm hooks to ensure migration completes before application starts. + +### Pitfall 2: Not Testing with Production Data + +**Problem:** Migration works with test data but fails with production data scale. + +**Solution:** Test migrations with production-like data volume in staging. + +### Pitfall 3: Forgetting Rollback Scripts + +**Problem:** Migration fails and cannot be rolled back. + +**Solution:** Always write rollback scripts and test them. + +### Pitfall 4: Changing Data Semantics + +**Problem:** Migration changes the meaning of existing data. + +**Solution:** Add new columns instead of modifying existing ones. + +### Pitfall 5: Long-Running Migrations + +**Problem:** Migration takes too long and times out. + +**Solution:** Use batch processing, increase timeout, or use online schema change tools. + +--- + +## References + +- [Knex.js Migration Documentation](https://knexjs.org/#Migrations) +- [PostgreSQL ALTER TABLE Documentation](https://www.postgresql.org/docs/current/sql-altertable.html) +- [Migration Failure Runbook](./MIGRATION_FAILURE_RUNBOOK.md) +- [Vault Migration Setup](./VAULT_MIGRATION_SETUP.md) diff --git a/docs/LONG_RUNNING_MIGRATIONS.md b/docs/LONG_RUNNING_MIGRATIONS.md new file mode 100644 index 0000000..13012d5 --- /dev/null +++ b/docs/LONG_RUNNING_MIGRATIONS.md @@ -0,0 +1,613 @@ +# Long-Running Migration Strategies + +This document provides strategies for handling long-running database migrations that may exceed default deployment timeouts. + +## Overview + +Some database operations can take significant time, especially on large datasets: +- Adding indexes to tables with millions of rows +- Backfilling data for new columns +- Converting column types +- Large data transformations + +These operations can trigger deployment timeouts if not properly managed. + +## Configuration + +### Default Timeouts + +The migration system has configurable timeouts: + +```yaml +migration: + timeout: 1800 # 30 minutes in seconds + lockTimeout: 300 # 5 minutes in seconds +``` + +### Increasing Timeouts + +For known long-running migrations, increase the timeout: + +```yaml +migration: + timeout: 7200 # 2 hours + lockTimeout: 600 # 10 minutes +``` + +**Apply via Helm:** +```bash +helm upgrade substream-backend ./helm/substream-backend \ + --namespace production \ + --set migration.timeout=7200 \ + --set migration.lockTimeout=600 +``` + +--- + +## Strategy 1: Batch Processing + +Break large operations into smaller batches to avoid long transactions and reduce lock contention. + +### Example: Backfilling Data + +**❌ Wrong (Single Transaction):** +```javascript +exports.up = async function(knex) { + // This could take hours on a large table + await knex('users') + .whereNull('email_verified') + .update({ email_verified: true }); +}; +``` + +**✅ Correct (Batch Processing):** +```javascript +exports.up = async function(knex) { + const batchSize = 10000; + let processed = 0; + let hasMore = true; + + console.log('Starting batch backfill...'); + + while (hasMore) { + const updated = await knex('users') + .whereNull('email_verified') + .limit(batchSize) + .update({ email_verified: true }); + + processed += updated; + hasMore = updated > 0; + + console.log(`Processed ${processed} records...`); + + // Small delay to reduce database load + if (hasMore) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + } + + console.log(`Backfill complete: ${processed} records updated`); +}; +``` + +### Example: Index Creation with Batch Data + +```javascript +exports.up = async function(knex) { + // Add column first + await knex.schema.table('orders', (table) => { + table.integer('customer_id').nullable(); + }); + + // Backfill in batches + const batchSize = 5000; + let offset = 0; + let total = 0; + + while (true) { + const orders = await knex('orders') + .whereNull('customer_id') + .limit(batchSize) + .offset(offset); + + if (orders.length === 0) break; + + for (const order of orders) { + // Derive customer_id from existing data + const customerId = await deriveCustomerId(order); + await knex('orders') + .where('id', order.id) + .update({ customer_id: customerId }); + } + + total += orders.length; + offset += batchSize; + console.log(`Backfilled ${total} orders`); + } + + // Create index after backfill + await knex.schema.table('orders', (table) => { + table.index('customer_id'); + }); +}; +``` + +--- + +## Strategy 2: Online Schema Changes + +For PostgreSQL, use online schema change tools to avoid locking tables. + +### PostgreSQL CONCURRENTLY + +```javascript +exports.up = function(knex) { + // Create index without locking the table + return knex.raw(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_users_email + ON users(email) + `); +}; + +exports.down = function(knex) { + return knex.raw('DROP INDEX CONCURRENTLY IF EXISTS idx_users_email'); +}; +``` + +**Important Notes:** +- CONCURRENTLY cannot be used in a transaction +- Index creation is still atomic +- If the operation fails, the index may be invalid +- Check for invalid indexes after migration + +### Check for Invalid Indexes + +```javascript +// After migration +await knex.raw(` + SELECT indexname, indexdef + FROM pg_indexes + WHERE indisvalid = false +`); +``` + +--- + +## Strategy 3: Two-Phase Migration + +Deploy migrations in multiple phases to spread the work across deployments. + +### Phase 1: Prepare Schema + +```javascript +// Migration 001_add_column.js +exports.up = function(knex) { + return knex.schema.table('subscriptions', (table) => { + table.string('status').defaultTo('active').nullable(); + }); +}; +``` + +### Phase 2: Backfill Data (Optional) + +```javascript +// Migration 002_backfill_status.js +exports.up = async function(knex) { + const batchSize = 10000; + let processed = 0; + + while (true) { + const updated = await knex('subscriptions') + .whereNull('status') + .limit(batchSize) + .update({ status: 'active' }); + + if (updated === 0) break; + + processed += updated; + console.log(`Backfilled ${processed} subscriptions`); + } +}; +``` + +### Phase 3: Update Application + +Deploy new application code that uses the new column. + +### Phase 4: Finalize Schema (Optional) + +```javascript +// Migration 003_finalize_column.js +exports.up = async function(knex) { + // Ensure all rows have a value + await knex('subscriptions') + .whereNull('status') + .update({ status: 'active' }); + + // Make column non-nullable + return knex.schema.table('subscriptions', (table) => { + table.string('status').defaultTo('active').alter(); + }); +}; +``` + +--- + +## Strategy 4: Background Job Migration + +For very large operations, use a background job instead of blocking deployment. + +### Create Migration Job + +```javascript +// Migration 001_queue_backfill.js +exports.up = async function(knex) { + // Add column + await knex.schema.table('users', (table) => { + table.string('full_name').nullable(); + }); + + // Queue background job + await knex('background_jobs').insert({ + type: 'backfill_user_names', + status: 'pending', + priority: 1, + created_at: new Date(), + metadata: JSON.stringify({ + batch_size: 10000, + estimated_total: 1000000 + }) + }); + + console.log('Backfill job queued'); +}; +``` + +### Process Background Job + +Create a worker to process the job: + +```javascript +// workers/backfillWorker.js +async function processBackfillJob(job) { + const { batch_size, estimated_total } = job.metadata; + let processed = 0; + + while (true) { + const users = await knex('users') + .whereNull('full_name') + .limit(batch_size); + + if (users.length === 0) break; + + for (const user of users) { + await knex('users') + .where('id', user.id) + .update({ + full_name: `${user.first_name} ${user.last_name}` + }); + } + + processed += users.length; + console.log(`Backfill progress: ${processed}/${estimated_total}`); + + // Update job progress + await knex('background_jobs') + .where('id', job.id) + .update({ + status: 'in_progress', + progress: processed / estimated_total + }); + } + + // Mark job complete + await knex('background_jobs') + .where('id', job.id) + .update({ status: 'completed' }); +} +``` + +--- + +## Strategy 5: Copy and Swap + +For major schema changes, create a new table and swap it in. + +### Step 1: Create New Table + +```javascript +// Migration 001_create_new_table.js +exports.up = async function(knex) { + await knex.schema.createTable('users_v2', (table) => { + table.increments('id').primary(); + table.string('email').unique(); + table.string('full_name'); + table.timestamp('created_at').defaultTo(knex.fn.now()); + table.timestamp('updated_at').defaultTo(knex.fn.now()); + }); +}; +``` + +### Step 2: Copy Data in Batches + +```javascript +// Migration 002_copy_data.js +exports.up = async function(knex) { + const batchSize = 10000; + let offset = 0; + let total = 0; + + while (true) { + const users = await knex('users') + .limit(batchSize) + .offset(offset); + + if (users.length === 0) break; + + await knex('users_v2').insert( + users.map(u => ({ + email: u.email, + full_name: `${u.first_name} ${u.last_name}`, + created_at: u.created_at, + updated_at: u.updated_at + })) + ); + + total += users.length; + offset += batchSize; + console.log(`Copied ${total} users`); + } +}; +``` + +### Step 3: Update Application + +Deploy new application code that reads from `users_v2`. + +### Step 4: Swap Tables + +```javascript +// Migration 003_swap_tables.js +exports.up = async function(knex) { + // Rename old table + await knex.schema.renameTable('users', 'users_old'); + + // Rename new table + await knex.schema.renameTable('users_v2', 'users'); +}; + +exports.down = async function(knex) { + // Rollback swap + await knex.schema.renameTable('users', 'users_v2'); + await knex.schema.renameTable('users_old', 'users'); +}; +``` + +### Step 5: Clean Up (Later) + +```javascript +// Migration 004_cleanup.js +exports.up = async function(knex) { + // Drop old table after verification + await knex.schema.dropTableIfExists('users_old'); +}; +``` + +--- + +## Monitoring Long-Running Migrations + +### Progress Logging + +Add progress logging to migrations: + +```javascript +exports.up = async function(knex) { + const total = await knex('large_table').count('* as count').first(); + const batchSize = 10000; + let processed = 0; + + console.log(`Total records to process: ${total.count}`); + + while (true) { + const updated = await knex('large_table') + .whereNull('new_column') + .limit(batchSize) + .update({ new_column: 'default_value' }); + + if (updated === 0) break; + + processed += updated; + const progress = ((processed / total.count) * 100).toFixed(2); + console.log(`Progress: ${progress}% (${processed}/${total.count})`); + } +}; +``` + +### Kubernetes Monitoring + +Monitor the migration job: + +```bash +# Watch job progress +kubectl logs -f job/substream-backend-migration -n substream + +# Check job status +kubectl describe job/substream-backend-migration -n substream + +# Check pod resource usage +kubectl top pod -l job-name=substream-backend-migration -n substream +``` + +### Alerts + +Set up alerts for: +- Migration job running longer than expected +- Migration job failure +- High database CPU during migration +- Lock acquisition timeouts + +--- + +## Testing Long-Running Migrations + +### Load Testing + +Test migrations with production-like data volumes: + +```bash +# Create test database with production data volume +pg_dump production_db | psql test_db + +# Run migration +node scripts/migrate-init-container.js + +# Monitor performance +``` + +### Performance Profiling + +Profile migration performance: + +```javascript +exports.up = async function(knex) { + const startTime = Date.now(); + + // Migration logic + + const duration = Date.now() - startTime; + console.log(`Migration duration: ${duration}ms`); + + // Log to monitoring system + await logMetric('migration_duration', duration); +}; +``` + +--- + +## Best Practices + +1. **Estimate Duration**: Test migrations with production-like data to estimate duration +2. **Add Buffers**: Set timeout 2-3x the estimated duration +3. **Monitor Progress**: Log progress for long operations +4. **Use Batches**: Break large operations into smaller batches +5. **Avoid Peak Hours**: Schedule long migrations during low-traffic periods +6. **Have Rollback Plan**: Always have a rollback strategy +7. **Communicate**: Notify stakeholders about expected downtime or performance impact +8. **Test Thoroughly**: Test in staging with production data volumes + +--- + +## Troubleshooting + +### Migration Times Out + +**Symptoms:** +- Job fails with "deadline exceeded" +- InitContainer exits with timeout error + +**Solutions:** +1. Increase `migration.timeout` in values.yaml +2. Optimize the migration (use batches, indexes, etc.) +3. Split migration into multiple phases +4. Use background job strategy + +### Migration Causes High Database Load + +**Symptoms:** +- Database CPU spikes to 100% +- Other queries slow down +- Application performance degrades + +**Solutions:** +1. Reduce batch size +2. Add delays between batches +3. Use online schema changes (CONCURRENTLY) +4. Schedule during maintenance window +5. Increase database resources temporarily + +### Migration Gets Stuck + +**Symptoms:** +- Migration runs indefinitely +- No progress in logs +- Lock held for too long + +**Solutions:** +1. Check for database locks +2. Kill stuck queries +3. Release migration lock manually +4. Restart migration job + +--- + +## Example: Complete Long-Running Migration + +### Scenario: Add index to 100M row table + +**Step 1: Estimate Duration** +```bash +# Test on sample data +CREATE INDEX CONCURRENTLY idx_test ON test_table(column); +# Monitor time: ~30 minutes for 10M rows +# Estimate: 5 hours for 100M rows +``` + +**Step 2: Configure Timeout** +```yaml +migration: + timeout: 21600 # 6 hours + lockTimeout: 600 # 10 minutes +``` + +**Step 3: Write Migration** +```javascript +// migrations/knex/015_add_large_index.js +exports.up = async function(knex) { + console.log('Starting index creation...'); + const startTime = Date.now(); + + await knex.raw(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_large_table_column + ON large_table(column) + `); + + const duration = Date.now() - startTime; + console.log(`Index created in ${duration}ms`); +}; + +exports.down = async function(knex) { + await knex.raw('DROP INDEX CONCURRENTLY IF EXISTS idx_large_table_column'); +}; +``` + +**Step 4: Deploy** +```bash +helm upgrade substream-backend ./helm/substream-backend \ + --namespace production \ + --set migration.timeout=21600 \ + --set migration.lockTimeout=600 +``` + +**Step 5: Monitor** +```bash +kubectl logs -f job/substream-backend-migration -n substream +``` + +**Step 6: Verify** +```bash +# Check index exists +kubectl exec -it -n substream -- node -e " +const knex = require('knex'); +const db = knex({ client: 'better-sqlite3', connection: { filename: '/app/data/substream.db' } }); +db.raw('PRAGMA index_list(large_table)').then(console.log).finally(() => db.destroy()); +" +``` + +--- + +## References + +- [PostgreSQL Index Creation](https://www.postgresql.org/docs/current/sql-createindex.html) +- [Backwards-Compatible Migrations](./BACKWARDS_COMPATIBLE_MIGRATIONS.md) +- [Migration Failure Runbook](./MIGRATION_FAILURE_RUNBOOK.md) diff --git a/docs/MIGRATION_FAILURE_RUNBOOK.md b/docs/MIGRATION_FAILURE_RUNBOOK.md new file mode 100644 index 0000000..9175de4 --- /dev/null +++ b/docs/MIGRATION_FAILURE_RUNBOOK.md @@ -0,0 +1,457 @@ +# Database Migration Failure Runbook + +This runbook provides step-by-step procedures for handling failed database migrations and performing safe rollbacks in the SubStream Protocol Backend. + +## Table of Contents + +1. [Immediate Response](#immediate-response) +2. [Diagnosis](#diagnosis) +3. [Rollback Procedures](#rollback-procedures) +4. [Post-Incident Actions](#post-incident-actions) +5. [Prevention Strategies](#prevention-strategies) + +--- + +## Immediate Response + +### Step 1: Stop the Deployment + +When a migration fails, the Kubernetes deployment will automatically stop. However, verify that no pods are running with the new version: + +```bash +# Check deployment status +kubectl get deployment substream-backend -n substream + +# Check pod status +kubectl get pods -n substream -l app=substream-backend + +# If any new pods are running, scale to 0 +kubectl scale deployment substream-backend --replicas=0 -n substream +``` + +### Step 2: Preserve the Current State + +Before making any changes, gather diagnostic information: + +```bash +# Get migration job logs +kubectl logs job/substream-backend-migration -n substream > migration-failure.log + +# Get initContainer logs (if using initContainer strategy) +kubectl logs -c migration -n substream > migration-init-failure.log + +# Get database lock status +kubectl exec -it -n substream -- node -e " +const knex = require('knex'); +const db = knex({ client: 'better-sqlite3', connection: { filename: '/app/data/substream.db' } }); +db('_migration_locks').select('*').then(console.log).finally(() => db.destroy()); +" + +# List current migration status +kubectl exec -it -n substream -- npx knex migrate:list +``` + +### Step 3: Notify Stakeholders + +Notify the following teams immediately: +- Engineering team +- Database team (if separate) +- DevOps/SRE team +- Product team (if user-facing impact is expected) + +--- + +## Diagnosis + +### Step 1: Analyze the Failure + +Review the migration logs to identify the root cause: + +```bash +# Check for common error patterns +grep -i "error" migration-failure.log +grep -i "timeout" migration-failure.log +grep -i "lock" migration-failure.log +grep -i "permission" migration-failure.log +``` + +### Step 2: Categorize the Failure + +#### Category A: Lock Acquisition Failure +**Symptoms:** +- "Lock already held" error +- "Timeout waiting for migration lock" + +**Diagnosis:** +- Another migration is in progress +- Previous migration crashed without releasing lock +- Lock timeout is too short + +**Resolution:** +```bash +# Manually release the lock (emergency only) +kubectl exec -it -n substream -- node -e " +const knex = require('knex'); +const db = knex({ client: 'better-sqlite3', connection: { filename: '/app/data/substream.db' } }); +db('_migration_locks').where('lock_key', 'schema_migration').del().then(() => console.log('Lock released')).finally(() => db.destroy()); +" +``` + +#### Category B: Schema Conflict +**Symptoms:** +- "Table already exists" +- "Column already exists" +- "Duplicate key error" + +**Diagnosis:** +- Migration was partially applied +- Manual schema changes were made +- Migration files are out of order + +**Resolution:** +```bash +# Check which migrations have been applied +kubectl exec -it -n substream -- npx knex migrate:list + +# If migration is partially applied, manually complete or rollback +kubectl exec -it -n substream -- npx knex migrate:rollback +``` + +#### Category C: Data Integrity Error +**Symptoms:** +- "Foreign key constraint" +- "Check constraint violation" +- "Cannot truncate table" + +**Diagnosis:** +- Data conflicts with new schema +- Referential integrity issues +- Data type mismatches + +**Resolution:** +1. Fix data issues manually +2. Create a data migration script +3. Re-run the schema migration + +#### Category D: Timeout Error +**Symptoms:** +- "Migration timeout after Xms" +- "Query execution timeout" + +**Diagnosis:** +- Long-running migration (e.g., adding index to large table) +- Insufficient resources +- Database performance issues + +**Resolution:** +1. Increase migration timeout in values.yaml +2. Add more resources to migration job +3. Optimize the migration (see [Long-Running Migrations](#long-running-migrations)) + +#### Category E: Permission Error +**Symptoms:** +- "Permission denied" +- "Access denied" + +**Diagnosis:** +- Insufficient database privileges +- Vault role misconfiguration +- Service account lacks permissions + +**Resolution:** +1. Verify Vault policy +2. Check database role permissions +3. Update service account annotations + +--- + +## Rollback Procedures + +### Option 1: Automatic Rollback (Recommended) + +The migration script includes automatic rollback on failure. If this didn't trigger, manually rollback: + +```bash +# Rollback last migration batch +kubectl exec -it -n substream -- npx knex migrate:rollback + +# Verify rollback +kubectl exec -it -n substream -- npx knex migrate:list +``` + +### Option 2: Manual Rollback + +If automatic rollback fails, perform manual rollback: + +#### Step 1: Identify the Failed Migration + +```bash +# Get migration list +kubectl exec -it -n substream -- npx knex migrate:list + +# Note the migration name that failed +``` + +#### Step 2: Create Rollback Script + +Create a manual rollback script based on the migration file: + +```javascript +// scripts/manual-rollback.js +const knex = require('knex'); +const knexConfig = require('../knexfile'); + +async function rollback() { + const db = knex(knexConfig); + + try { + // Reverse the changes from the failed migration + await db.schema.dropTableIfExists('new_table'); + await db.schema.table('existing_table', (table) => { + table.dropColumn('new_column'); + }); + + console.log('Manual rollback completed'); + } catch (error) { + console.error('Rollback failed:', error); + throw error; + } finally { + await db.destroy(); + } +} + +rollback(); +``` + +#### Step 3: Execute Rollback + +```bash +kubectl exec -it -n substream -- node scripts/manual-rollback.js +``` + +### Option 3: Database Restore (Last Resort) + +If rollback is not possible, restore from backup: + +```bash +# Identify the last successful backup +# This depends on your backup solution (e.g., Velero, pg_dump, etc.) + +# For SQLite, restore from backup +kubectl cp ./substream.db.backup :/app/data/substream.db -n substream + +# Restart the application +kubectl rollout restart deployment substream-backend -n substream +``` + +--- + +## Post-Incident Actions + +### Step 1: Verify System Health + +After rollback, verify the system is healthy: + +```bash +# Check pod status +kubectl get pods -n substream + +# Check application health +kubectl exec -it -n substream -- curl http://localhost:3000/health + +# Check database connectivity +kubectl exec -it -n substream -- node -e " +const knex = require('knex'); +const db = knex({ client: 'better-sqlite3', connection: { filename: '/app/data/substream.db' } }); +db.raw('SELECT 1').then(() => console.log('Database OK')).catch(console.error).finally(() => db.destroy()); +" +``` + +### Step 2: Document the Incident + +Create an incident report with: +- Timestamp of failure +- Error messages and logs +- Root cause analysis +- Actions taken +- Resolution steps +- Preventive measures + +### Step 3: Update Monitoring + +Add alerts for common migration failures: +- Migration job failures +- Lock acquisition timeouts +- Long-running migrations +- Database permission errors + +### Step 4: Post-Mortem + +Conduct a post-mortem meeting with stakeholders to: +- Review the incident timeline +- Identify process improvements +- Update documentation +- Assign action items + +--- + +## Prevention Strategies + +### 1. Pre-Deployment Testing + +Always test migrations in a staging environment: + +```bash +# Run migrations in staging +helm upgrade substream-backend ./helm/substream-backend \ + --namespace staging \ + --values values-staging.yaml \ + --set migration.enabled=true + +# Verify application works +# Run smoke tests +# Run integration tests +``` + +### 2. Migration Best Practices + +Follow these guidelines when writing migrations: + +- **Always write rollback scripts** +- **Use idempotent operations** (IF NOT EXISTS, IF EXISTS) +- **Avoid data loss operations** (DROP without backup) +- **Test with production-like data** +- **Document breaking changes** + +### 3. Staged Rollout + +Use canary deployments for critical migrations: + +```yaml +# Initial canary (10% traffic) +helm upgrade substream-backend ./helm/substream-backend \ + --namespace production \ + --set replicaCount=1 \ + --set migration.enabled=true + +# Monitor for 30 minutes +# If successful, scale up +helm upgrade substream-backend ./helm/substream-backend \ + --namespace production \ + --set replicaCount=3 +``` + +### 4. Backup Before Migration + +Automate database backups before migrations: + +```yaml +# Add pre-migration backup hook +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "substream-backend.fullname" . }}-backup + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "-10" +spec: + template: + spec: + containers: + - name: backup + image: appropriate/backup-tool + command: ["backup-database"] +``` + +### 5. Health Checks + +Implement comprehensive health checks: + +```javascript +// Add to migration script +async function postMigrationHealthCheck() { + // Check table counts + // Check data integrity + // Test critical queries + // Verify indexes +} +``` + +--- + +## Long-Running Migrations + +For migrations that take longer than the default timeout: + +### Strategy 1: Increase Timeout + +```yaml +migration: + timeout: 3600 # 1 hour +``` + +### Strategy 2: Batch Processing + +Break large operations into smaller batches: + +```javascript +// Instead of: UPDATE users SET status = 'active' +// Use: +async function batchUpdate() { + const batchSize = 10000; + let offset = 0; + + while (true) { + const users = await db('users') + .limit(batchSize) + .offset(offset); + + if (users.length === 0) break; + + await db('users') + .whereIn('id', users.map(u => u.id)) + .update({ status: 'active' }); + + offset += batchSize; + console.log(`Processed ${offset} users`); + } +} +``` + +### Strategy 3: Online Schema Change + +For PostgreSQL, use online schema change tools: +- `pg_repack` +- `pg_squeeze` +- `ALTER TABLE ... CONCURRENTLY` + +### Strategy 4: Two-Phase Migration + +Deploy in two phases: + +**Phase 1 (Non-breaking):** +- Add new columns (nullable) +- Create new tables +- Add new indexes (CONCURRENTLY) + +**Phase 2 (Breaking):** +- Backfill data +- Update application code +- Remove old columns/tables + +--- + +## Emergency Contacts + +- **On-Call Engineer**: [Phone/Slack] +- **Database Team**: [Phone/Slack] +- **DevOps Team**: [Phone/Slack] +- **Engineering Manager**: [Phone/Slack] + +--- + +## Related Documentation + +- [Vault Migration Setup](./VAULT_MIGRATION_SETUP.md) +- [Backwards-Compatible Migrations](./BACKWARDS_COMPATIBLE_MIGRATIONS.md) +- [Helm Chart Documentation](../../helm/substream-backend/README.md) diff --git a/docs/PRIVACY_POLICY_DATA_RETENTION.md b/docs/PRIVACY_POLICY_DATA_RETENTION.md new file mode 100644 index 0000000..7125b69 --- /dev/null +++ b/docs/PRIVACY_POLICY_DATA_RETENTION.md @@ -0,0 +1,303 @@ +# Privacy Policy - Data Retention and Right to be Forgotten + +## Overview + +This document outlines SubStream Protocol's data retention policies and procedures for handling user data deletion requests in compliance with GDPR (General Data Protection Regulation), CCPA (California Consumer Privacy Act), and other global privacy regulations. + +## Data Retention Timeline + +### Active User Data + +**Retention Period:** Indefinite while account remains active + +**Data Categories Retained:** +- Wallet address (public key) +- Subscription records +- Payment/billing events +- User preferences +- Activity logs +- Comments and engagement data + +**Purpose:** Service delivery, payment processing, content delivery, analytics + +### Inactive User Data + +**Retention Period:** 3 years from last activity + +**Definition of Inactivity:** No subscription activity, no logins, no content interactions for 3 consecutive years + +**Automated Action:** PII is automatically scrubbed after 3 years of inactivity + +### Financial Data + +**Retention Period:** 7 years (tax compliance requirement) + +**Data Categories Retained:** +- Billing events +- Transaction records +- Payment amounts +- Subscription duration + +**Anonymization:** User identity is cryptographically hashed, but financial data is preserved for tax accounting + +**Purpose:** Tax compliance, financial auditing, fraud prevention + +### Audit Logs + +**Retention Period:** 5 years + +**Data Categories Retained:** +- System access logs +- API request logs +- Security events +- Compliance actions + +**Purpose:** Security monitoring, compliance auditing, incident response + +## Right to be Forgotten (Data Deletion) + +### User-Initiated Deletion + +**Eligibility:** Any user may request deletion of their personal data at any time + +**Process:** +1. User submits deletion request via API endpoint or support ticket +2. System verifies user identity (wallet signature or authentication) +3. PII scrubbing process is initiated within 24 hours +4. User receives confirmation of completion +5. Affected merchants receive webhook notification + +**Data Deleted:** +- Email addresses +- IP addresses +- User names +- Profile information +- Device fingerprints +- Any other directly identifying information + +**Data Preserved (Anonymized):** +- Financial records (with anonymized user identity) +- Audit logs (with anonymized user identity) +- System records required for compliance + +### Automated Retention Policy + +**Trigger:** 3 years of account inactivity + +**Process:** +1. Automated job runs weekly to identify inactive users +2. PII scrubbing is performed automatically +3. Audit log entry is created +4. No notification is sent (user is inactive) + +**Scope:** All PII across database, Redis cache, and analytics warehouse + +## Data Categories and Retention + +### Personal Identifiable Information (PII) + +| Data Category | Active Retention | Inactive Retention | Deletion Method | +|---------------|------------------|-------------------|-----------------| +| Email Address | Until account deletion | 3 years | Cryptographic hash | +| IP Address | 1 year | 3 years | Cryptographic hash | +| Device Fingerprint | 1 year | 3 years | Deletion | +| User Name | Until account deletion | 3 years | Deletion | +| Profile Bio | Until account deletion | 3 years | Deletion | +| Avatar Image | Until account deletion | 3 years | Deletion | + +### Financial Data + +| Data Category | Retention Period | Deletion Method | +|---------------|------------------|-----------------| +| Billing Events | 7 years | Anonymized (wallet hashed) | +| Transaction Records | 7 years | Anonymized (wallet hashed) | +| Subscription Records | 7 years | Anonymized (wallet hashed) | +| Payment Amounts | 7 years | Preserved (tax compliance) | + +### Content Data + +| Data Category | Retention Period | Deletion Method | +|---------------|------------------|-----------------| +| User Comments | Until account deletion | 3 years (anonymized) | +| User Likes | Until account deletion | 3 years (anonymized) | +| User Content | Until account deletion | 3 years (anonymized) | + +### System Data + +| Data Category | Retention Period | Deletion Method | +|---------------|------------------|-----------------| +| Audit Logs | 5 years | Preserved (compliance) | +| API Logs | 1 year | Deletion | +| Error Logs | 1 year | Deletion | +| Performance Metrics | 1 year | Deletion | + +## Data Deletion Process + +### Technical Implementation + +**Cryptographic Hashing:** +- Algorithm: SHA-256 with HMAC +- Salt: Environment-specific secure salt (32 bytes) +- Format: `prefix_hash` for debugging, irreversible for security + +**Wallet Address Anonymization:** +- Original: `GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ` +- Anonymized: `GD5DQ6ZQZ_abc123def456...` (first 8 chars + hash) + +**Email Anonymization:** +- Original: `user@example.com` +- Anonymized: `scrubbed_[hash]@anon.example.com` + +**IP Address Anonymization:** +- Original: `192.168.1.100` +- Anonymized: `scrubbed_[hash]` + +### Database Tables Scrubbed + +1. **subscriptions** - user_email, wallet_address +2. **creator_audit_logs** - ip_address +3. **api_key_audit_logs** - ip_address +4. **data_export_tracking** - requester_email +5. **privacy_preferences** - share_email_with_merchants +6. **comments** - user_address +7. **leaderboard_entries** - fan_address +8. **social_tokens** - user_address +9. **activitypub_engagements** - fan_address + +### Cache Scrubbing + +**Redis Keys Scrubbed:** +- `user:{walletAddress}:*` +- `profile:{walletAddress}:*` +- `subscription:{walletAddress}:*` +- `creator:{walletAddress}:*` +- `session:{walletAddress}:*` +- `cache:{walletAddress}:*` + +### Analytics Warehouse + +**Data Anonymized:** +- User identifiers replaced with hashed values +- IP addresses removed or hashed +- Email addresses removed or hashed +- Device fingerprints removed + +## Merchant Notifications + +### Webhook Payload + +When a user invokes their right to be forgotten, affected merchants receive a webhook notification: + +```json +{ + "event": "user.forget", + "timestamp": "2026-04-26T14:00:00Z", + "scrub_id": "uuid-v4", + "data": { + "anonymized_wallet_address": "GD5DQ6ZQZ_abc123...", + "reason": "user_request", + "scrubbed_at": "2026-04-26T14:00:00Z" + } +} +``` + +**Purpose:** Inform merchants that user data has been deleted so they can update their own records + +**Timing:** Within 24 hours of deletion request + +**Security:** Webhooks signed with merchant's secret + +## Audit Trail + +All deletion operations are logged with: + +- Scrub operation ID (UUID) +- Original wallet address (hashed) +- Anonymized wallet address +- Reason for deletion +- Timestamp +- Tables affected +- Rows modified +- Operator (user/admin/system) +- Success/failure status + +**Retention:** Audit logs retained for 5 years for compliance + +## Verification + +Users can verify their data has been deleted by: + +1. Calling the verification endpoint with their wallet address +2. Receiving confirmation of scrubbing status +3. Reviewing which tables were affected + +**Endpoint:** `GET /api/v1/compliance/forget/:walletAddress/status` + +## Data Export (Right to Data Portability) + +Users may request a copy of their data before deletion: + +**Endpoint:** `POST /api/v1/compliance/export` + +**Data Included:** +- Subscriptions +- Comments +- Audit logs (last 100 entries) +- Privacy preferences + +**Format:** JSON + +**Delivery:** Secure download link (expires in 7 days) + +## Exceptions and Limitations + +### Data That Cannot Be Deleted + +1. **Financial Records** - Required for tax compliance (7 years) +2. **Audit Logs** - Required for security and compliance (5 years) +3. **Blockchain Transactions** - Immutable public ledger +4. **Legal Holds** - Data subject to legal preservation orders + +### Data That Is Anonymized Not Deleted + +1. **Subscription Records** - Financial data preserved, identity hashed +2. **Billing Events** - Payment data preserved, identity hashed +3. **Content Engagement** - Content preserved, user identity hashed + +## Compliance Certifications + +This data retention policy is designed to comply with: + +- **GDPR** (EU General Data Protection Regulation) + - Article 17: Right to erasure ("right to be forgotten") + - Article 20: Right to data portability + - Recital 39: Data minimization + +- **CCPA** (California Consumer Privacy Act) + - Right to delete + - Right to know + - Right to opt-out + +- **LGPD** (Brazilian General Data Protection Law) + - Right to deletion + - Right to data portability + +- **POPIA** (South African Protection of Personal Information Act) + - Right to deletion + - Data minimization + +## Contact + +For data deletion requests or privacy-related inquiries: + +- **Email:** privacy@substream.protocol +- **API:** POST /api/v1/compliance/forget +- **Support:** https://support.substream.protocol + +## Last Updated + +April 26, 2026 + +## Version History + +- **v1.0** (April 26, 2026) - Initial policy for automated PII scrubbing diff --git a/docs/VAULT_MIGRATION_SETUP.md b/docs/VAULT_MIGRATION_SETUP.md new file mode 100644 index 0000000..70b7266 --- /dev/null +++ b/docs/VAULT_MIGRATION_SETUP.md @@ -0,0 +1,158 @@ +# Vault Integration for Database Migration Credentials + +This document outlines the setup for HashiCorp Vault integration to securely manage database migration credentials. + +## Overview + +The migration initContainer requires elevated database privileges to execute schema changes. These credentials should never be hardcoded in the Docker image or Helm values. Instead, we use Vault to dynamically provide credentials with the necessary permissions. + +## Prerequisites + +- HashiCorp Vault deployed in your Kubernetes cluster +- Vault Kubernetes authentication configured +- Database secrets engine enabled + +## Vault Policy for Migration + +Create a Vault policy with the necessary permissions for database migrations: + +```hcl +# File: vault-policy-migration.hcl +path "database/creds/migration" { + capabilities = ["read"] +} + +path "sys/leases/renew" { + capabilities = ["update"] +} + +path "sys/leases/lookup" { + capabilities = ["read"] +} +``` + +## Database Role Configuration + +Configure the database secrets engine with a migration role that has elevated privileges: + +```bash +# Enable database secrets engine (if not already enabled) +vault secrets enable database + +# Configure PostgreSQL database connection +vault write database/config/substream-prod \ + plugin_name=postgresql-database-plugin \ + connection_url="postgresql://{{username}}:{{password}}@postgres-prod:5432/substream" \ + allowed_roles="migration,app" + +# Create migration role with elevated privileges +vault write database/roles/migration \ + db_name=substream-prod \ + creation_statements="CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; \ + GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO \"{{name}}\"; \ + GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO \"{{name}}\"; \ + GRANT CREATE ON SCHEMA public TO \"{{name}}\"; \ + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO \"{{name}}\";" \ + default_ttl="1h" \ + max_ttl="24h" +``` + +## Kubernetes Authentication Setup + +Configure Vault to authenticate Kubernetes service accounts: + +```bash +# Enable Kubernetes authentication +vault auth enable kubernetes + +# Configure Kubernetes authentication +vault write auth/kubernetes/config \ + kubernetes_host="https://kubernetes.default.svc:443" \ + kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + +# Create role for migration service account +vault write auth/kubernetes/role/substream-migration \ + bound_service_account_names=substream-backend-migration \ + bound_service_account_namespaces=substream \ + policies=migration \ + ttl=1h +``` + +## Helm Chart Configuration + +Update your Helm values to enable Vault integration: + +```yaml +migration: + enabled: true + strategy: "initContainer" + vault: + enabled: true + role: "substream-migration" + secretPath: "database/creds/migration" + address: "http://vault:8200" +``` + +## Service Account Annotation + +Annotate the service account to allow Vault authentication: + +```yaml +serviceAccount: + create: true + annotations: + eks.amazonaws.com/role-arn: "arn:aws:iam::ACCOUNT_ID:role/vault-auth-role" + name: "" +``` + +## Environment Variables in InitContainer + +The initContainer will receive the following environment variables when Vault is enabled: + +- `VAULT_ADDR`: Vault server address +- `VAULT_ROLE`: Kubernetes authentication role +- `VAULT_SECRET_PATH`: Path to database credentials +- `DATABASE_URL`: Dynamically fetched from Vault + +## Security Considerations + +1. **Least Privilege**: The migration role should only have permissions needed for schema changes +2. **Short TTL**: Credentials should have a short TTL (1 hour default) +3. **Rotation**: Enable automatic credential rotation +4. **Audit Logging**: Enable Vault audit logging for all credential access +5. **Namespace Isolation**: Use separate Vault namespaces for different environments + +## Testing Vault Integration + +Test the Vault integration locally: + +```bash +# Set Vault address +export VAULT_ADDR="http://vault:8200" + +# Login with Kubernetes auth +vault write auth/kubernetes/login role=substream-migration jwt=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + +# Retrieve database credentials +vault read database/creds/migration +``` + +## Troubleshooting + +### Migration fails with "permission denied" + +- Verify the Vault policy has `read` access to `database/creds/migration` +- Check that the database role has sufficient privileges (CREATE, ALTER, etc.) +- Ensure the service account has the correct Vault role annotation + +### Credentials expire during migration + +- Increase the `default_ttl` and `max_ttl` for the database role +- Implement credential renewal in the migration script +- Break long-running migrations into smaller batches + +### Vault authentication fails + +- Verify the Kubernetes auth method is properly configured +- Check that the service account name matches the Vault role binding +- Ensure the service account token is mounted in the pod diff --git a/helm/substream-backend/templates/deployment.yaml b/helm/substream-backend/templates/deployment.yaml index faa6a78..2f1a4bb 100644 --- a/helm/substream-backend/templates/deployment.yaml +++ b/helm/substream-backend/templates/deployment.yaml @@ -26,6 +26,52 @@ spec: serviceAccountName: {{ include "substream-backend.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if and .Values.migration.enabled (eq .Values.migration.strategy "initContainer") }} + initContainers: + - name: migration + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - node + - scripts/migrate-init-container.js + env: + - name: NODE_ENV + value: {{ .Values.env.NODE_ENV }} + - name: DATABASE_FILENAME + value: {{ .Values.database.filename }} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MIGRATION_TIMEOUT + value: {{ .Values.migration.timeout | quote }} + - name: MIGRATION_LOCK_TIMEOUT + value: {{ .Values.migration.lockTimeout | quote }} + {{- if .Values.redis.existingSecret }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.redis.existingSecret }} + key: {{ .Values.redis.existingPasswordKey }} + {{- end }} + resources: + {{- toYaml .Values.migration.resources | nindent 12 }} + volumeMounts: + - name: data-volume + mountPath: /app/data + - name: tmp-volume + mountPath: /tmp + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: diff --git a/helm/substream-backend/templates/migration-job.yaml b/helm/substream-backend/templates/migration-job.yaml new file mode 100644 index 0000000..98861e7 --- /dev/null +++ b/helm/substream-backend/templates/migration-job.yaml @@ -0,0 +1,85 @@ +{{- if and .Values.migration.enabled (eq .Values.migration.strategy "job") }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "substream-backend.fullname" . }}-migration + namespace: {{ .Values.namespace }} + labels: + {{- include "substream-backend.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + backoffLimit: 3 + activeDeadlineSeconds: {{ .Values.migration.timeout }} + template: + metadata: + labels: + {{- include "substream-backend.selectorLabels" . | nindent 8 }} + spec: + restartPolicy: OnFailure + serviceAccountName: {{ include "substream-backend.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: migration + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - node + - scripts/migrate-init-container.js + env: + - name: NODE_ENV + value: {{ .Values.env.NODE_ENV }} + - name: DATABASE_FILENAME + value: {{ .Values.database.filename }} + - name: POD_NAME + value: {{ include "substream-backend.fullname" . }}-migration + - name: NAMESPACE + value: {{ .Values.namespace }} + - name: HOSTNAME + value: migration-job + - name: MIGRATION_TIMEOUT + value: {{ .Values.migration.timeout | quote }} + - name: MIGRATION_LOCK_TIMEOUT + value: {{ .Values.migration.lockTimeout | quote }} + {{- if .Values.redis.existingSecret }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.redis.existingSecret }} + key: {{ .Values.redis.existingPasswordKey }} + {{- end }} + resources: + {{- toYaml .Values.migration.resources | nindent 12 }} + volumeMounts: + - name: data-volume + mountPath: /app/data + - name: tmp-volume + mountPath: /tmp + volumes: + - name: data-volume + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "substream-backend.fullname" . }}-data + {{- else }} + emptyDir: {} + {{- end }} + - name: tmp-volume + emptyDir: {} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/substream-backend/values.yaml b/helm/substream-backend/values.yaml index 89ad3f6..34266ba 100644 --- a/helm/substream-backend/values.yaml +++ b/helm/substream-backend/values.yaml @@ -101,6 +101,26 @@ database: filename: "/app/data/substream.db" maxConnections: 20 +# Migration configuration +migration: + enabled: true + strategy: "initContainer" # Options: "initContainer" or "job" + timeout: 1800 # 30 minutes in seconds + lockTimeout: 300 # 5 minutes in seconds + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" + # Vault configuration for migration credentials + vault: + enabled: false + role: "substream-migration" + secretPath: "database/creds/migration" + address: "http://vault:8200" + persistence: enabled: true storageClass: "fast-ssd" diff --git a/infrastructure/aws-waf-rules.json b/infrastructure/aws-waf-rules.json new file mode 100644 index 0000000..e8d7953 --- /dev/null +++ b/infrastructure/aws-waf-rules.json @@ -0,0 +1,287 @@ +{ + "AWSCloudFormationVersion": "2010-09-09", + "Description": "AWS WAF Rules for SubStream Protocol API Gateway", + "Resources": { + "WebACL": { + "Type": "AWS::WAFv2::WebACL", + "Properties": { + "Name": "SubStreamAPIWebACL", + "Scope": "REGIONAL", + "DefaultAction": { + "Allow": {} + }, + "Description": "WAF rules to protect API Gateway from DDoS and application attacks", + "Rules": [ + { + "Name": "BlockKnownMaliciousIPs", + "Priority": 1, + "Statement": { + "IPSetReferenceStatement": { + "ARN": { + "Ref": "MaliciousIPSet" + } + } + }, + "Action": { + "Block": {} + }, + "VisibilityConfig": { + "SampledRequestsEnabled": true, + "CloudWatchMetricsEnabled": true, + "MetricName": "BlockKnownMaliciousIPs" + } + }, + { + "Name": "BlockTorExitNodes", + "Priority": 2, + "Statement": { + "IPSetReferenceStatement": { + "ARN": { + "Ref": "TorExitNodeIPSet" + } + } + }, + "Action": { + "Block": {} + }, + "VisibilityConfig": { + "SampledRequestsEnabled": true, + "CloudWatchMetricsEnabled": true, + "MetricName": "BlockTorExitNodes" + } + }, + { + "Name": "RateLimitPerIP", + "Priority": 3, + "Statement": { + "RateBasedStatement": { + "Limit": 2000, + "AggregateKeyType": "IP" + } + }, + "Action": { + "Block": {} + }, + "VisibilityConfig": { + "SampledRequestsEnabled": true, + "CloudWatchMetricsEnabled": true, + "MetricName": "RateLimitPerIP" + } + }, + { + "Name": "BlockSQLInjection", + "Priority": 4, + "Statement": { + "ManagedRuleGroupStatement": { + "VendorName": "AWS", + "Name": "AWSManagedRulesSQLiRuleSet", + "ExcludedRules": [] + } + }, + "OverrideAction": { + "None": {} + }, + "VisibilityConfig": { + "SampledRequestsEnabled": true, + "CloudWatchMetricsEnabled": true, + "MetricName": "BlockSQLInjection" + } + }, + { + "Name": "BlockXSSAttacks", + "Priority": 5, + "Statement": { + "ManagedRuleGroupStatement": { + "VendorName": "AWS", + "Name": "AWSManagedRulesCommonRuleSet", + "ExcludedRules": [] + } + }, + "OverrideAction": { + "None": {} + }, + "VisibilityConfig": { + "SampledRequestsEnabled": true, + "CloudWatchMetricsEnabled": true, + "MetricName": "BlockXSSAttacks" + } + }, + { + "Name": "BlockHeadlessBrowsers", + "Priority": 6, + "Statement": { + "RegexMatchStatement": { + "FieldToMatch": { + "SingleHeader": { + "Name": "user-agent" + } + }, + "TextTransformations": [ + { + "Type": "LOWERCASE", + "Priority": 0 + } + ], + "RegexString": "(headlesschrome|phantomjs|slimerjs|htmlunit|jsoup|python-requests|curl|wget|axios|node-fetch|go-http-client|java|apache-httpclient)" + } + }, + "Action": { + "Block": {} + }, + "VisibilityConfig": { + "SampledRequestsEnabled": true, + "CloudWatchMetricsEnabled": true, + "MetricName": "BlockHeadlessBrowsers" + } + }, + { + "Name": "AllowStripeWebhooks", + "Priority": 7, + "Statement": { + "IPSetReferenceStatement": { + "ARN": { + "Ref": "StripeWebhookIPSet" + } + } + }, + "Action": { + "Allow": {} + }, + "VisibilityConfig": { + "SampledRequestsEnabled": true, + "CloudWatchMetricsEnabled": true, + "MetricName": "AllowStripeWebhooks" + } + }, + { + "Name": "AllowPayPalWebhooks", + "Priority": 8, + "Statement": { + "IPSetReferenceStatement": { + "ARN": { + "Ref": "PayPalWebhookIPSet" + } + } + }, + "Action": { + "Allow": {} + }, + "VisibilityConfig": { + "SampledRequestsEnabled": true, + "CloudWatchMetricsEnabled": true, + "MetricName": "AllowPayPalWebhooks" + } + }, + { + "Name": "SizeRestriction", + "Priority": 9, + "Statement": { + "SizeConstraintStatement": { + "ComparisonOperator": "GT", + "FieldToMatch": { + "Body": {} + }, + "Size": 10485760, + "TextTransformations": [] + } + }, + "Action": { + "Block": {} + }, + "VisibilityConfig": { + "SampledRequestsEnabled": true, + "CloudWatchMetricsEnabled": true, + "MetricName": "SizeRestriction" + } + }, + { + "Name": "BlockBadUserAgents", + "Priority": 10, + "Statement": { + "ManagedRuleGroupStatement": { + "VendorName": "AWS", + "Name": "AWSManagedRulesKnownBadInputsRuleSet", + "ExcludedRules": [] + } + }, + "OverrideAction": { + "None": {} + }, + "VisibilityConfig": { + "SampledRequestsEnabled": true, + "CloudWatchMetricsEnabled": true, + "MetricName": "BlockBadUserAgents" + } + } + ], + "VisibilityConfig": { + "SampledRequestsEnabled": true, + "CloudWatchMetricsEnabled": true, + "MetricName": "SubStreamAPIWebACL" + } + } + }, + "MaliciousIPSet": { + "Type": "AWS::WAFv2::IPSet", + "Properties": { + "Name": "MaliciousIPSet", + "Description": "IP addresses known to be malicious", + "IPAddressVersion": "IPV4", + "Addresses": [] + } + }, + "TorExitNodeIPSet": { + "Type": "AWS::WAFv2::IPSet", + "Properties": { + "Name": "TorExitNodeIPSet", + "Description": "Tor exit node IP addresses", + "IPAddressVersion": "IPV4", + "Addresses": [] + } + }, + "StripeWebhookIPSet": { + "Type": "AWS::WAFv2::IPSet", + "Properties": { + "Name": "StripeWebhookIPSet", + "Description": "Stripe webhook IP addresses (whitelisted)", + "IPAddressVersion": "IPV4", + "Addresses": [ + "3.18.12.63/32", + "3.130.192.231/32", + "13.248.212.57/32", + "13.234.10.11/32", + "52.74.223.119/32", + "54.187.174.169/32", + "54.187.205.192/32", + "54.187.216.72/32", + "54.241.31.173/32", + "54.241.32.135/32" + ] + } + }, + "PayPalWebhookIPSet": { + "Type": "AWS::WAFv2::IPSet", + "Properties": { + "Name": "PayPalWebhookIPSet", + "Description": "PayPal webhook IP addresses (whitelisted)", + "IPAddressVersion": "IPV4", + "Addresses": [ + "66.211.170.66/32", + "173.0.84.0/24", + "173.0.88.0/24", + "173.0.90.0/24", + "173.0.92.0/24", + "173.0.94.0/24" + ] + } + } + }, + "Outputs": { + "WebACLId": { + "Description": "Web ACL ID", + "Value": { + "Ref": "WebACL" + } + } + } +} diff --git a/middleware/rateLimit.js b/middleware/rateLimit.js new file mode 100644 index 0000000..6815343 --- /dev/null +++ b/middleware/rateLimit.js @@ -0,0 +1,383 @@ +/** + * Rate Limiting Middleware + * + * Express middleware that enforces rate limits using the token bucket algorithm. + * Returns HTTP 429 with Retry-After header when limits are exceeded. + * Differentiates between authenticated and anonymous traffic. + */ + +const RateLimitService = require('../services/rateLimitService'); +const logger = require('../utils/logger'); + +class RateLimitMiddleware { + constructor(options = {}) { + this.rateLimitService = options.rateLimitService || new RateLimitService(); + + // Middleware configuration + this.skipSuccessfulRequests = options.skipSuccessfulRequests || false; + this.skipFailedRequests = options.skipFailedRequests || false; + this.enableWebhookWhitelist = options.enableWebhookWhitelist !== false; + this.enableIPBlacklist = options.enableIPBlacklist !== false; + this.enableTorBlocking = options.enableTorBlocking !== false; + + // Headers to add to responses + this.addHeaders = options.addHeaders !== false; + } + + /** + * Extract client IP from request + * @param {object} req - Express request + * @returns {string} IP address + */ + getClientIP(req) { + return req.ip || + req.connection?.remoteAddress || + req.socket?.remoteAddress || + (req.headers['x-forwarded-for'] || '').split(',')[0].trim() || + '0.0.0.0'; + } + + /** + * Extract tenant ID from request + * @param {object} req - Express request + * @returns {string|null} Tenant ID + */ + getTenantId(req) { + return req.user?.tenantId || + req.tenantId || + req.headers['x-tenant-id'] || + null; + } + + /** + * Extract API key from request + * @param {object} req - Express request + * @returns {string|null} API key + */ + getAPIKey(req) { + return req.headers['x-api-key'] || + req.query.api_key || + req.user?.apiKey || + null; + } + + /** + * Check if request is from a whitelisted webhook + * @param {object} req - Express request + * @returns {boolean} + */ + isWebhookRequest(req) { + const hostname = req.hostname || req.headers['host'] || ''; + const userAgent = req.headers['user-agent'] || ''; + const path = req.path || ''; + + // Check hostname whitelist + if (this.enableWebhookWhitelist && this.rateLimitService.isWebhookWhitelisted(hostname)) { + return true; + } + + // Check for common webhook paths + const webhookPaths = ['/webhook', '/webhooks', '/hooks', '/callback']; + if (webhookPaths.some(wp => path.startsWith(wp))) { + return true; + } + + // Check for known webhook user agents + const webhookUserAgents = ['Stripe', 'PayPal', 'GitHub', 'webhook']; + if (webhookUserAgents.some(ua => userAgent.includes(ua))) { + return true; + } + + return false; + } + + /** + * Rate limiting middleware for general API requests + */ + apiRateLimit() { + return async (req, res, next) => { + try { + const ip = this.getClientIP(req); + const tenantId = this.getTenantId(req); + const apiKey = this.getAPIKey(req); + + // Skip webhook requests if whitelist is enabled + if (this.enableWebhookWhitelist && this.isWebhookRequest(req)) { + logger.debug('[RateLimit] Skipping whitelisted webhook request', { + ip, + path: req.path + }); + return next(); + } + + // Check IP blacklist + if (this.enableIPBlacklist && this.rateLimitService.isIPBlacklisted(ip)) { + logger.warn('[RateLimit] Blocked blacklisted IP', { + ip, + path: req.path, + userAgent: req.headers['user-agent'] + }); + + return res.status(403).json({ + error: 'Forbidden', + message: 'Your IP has been blocked due to suspicious activity', + code: 'IP_BLOCKED' + }); + } + + // Check Tor exit nodes + if (this.enableTorBlocking && this.rateLimitService.isTorExitNode(ip)) { + logger.warn('[RateLimit] Blocked Tor exit node', { + ip, + path: req.path + }); + + return res.status(403).json({ + error: 'Forbidden', + message: 'Access from Tor exit nodes is not allowed', + code: 'TOR_BLOCKED' + }); + } + + // Check rate limit + const result = await this.rateLimitService.checkAPIRateLimit(ip, tenantId, apiKey); + + // Add rate limit headers to response + if (this.addHeaders) { + res.setHeader('X-RateLimit-Limit', result.limit); + res.setHeader('X-RateLimit-Remaining', result.remaining); + res.setHeader('X-RateLimit-Reset', result.reset); + } + + if (!result.allowed) { + logger.warn('[RateLimit] Rate limit exceeded', { + ip, + tenantId, + path: req.path, + limit: result.limit, + remaining: result.remaining, + retryAfter: result.retryAfter + }); + + // Return 429 with Retry-After header + res.setHeader('Retry-After', result.retryAfter); + return res.status(429).json({ + error: 'Too Many Requests', + message: 'Rate limit exceeded. Please retry later.', + code: 'RATE_LIMIT_EXCEEDED', + retryAfter: result.retryAfter, + limit: result.limit, + reset: result.reset + }); + } + + // Add rate limit info to request for downstream use + req.rateLimit = { + limit: result.limit, + remaining: result.remaining, + reset: result.reset + }; + + next(); + } catch (error) { + logger.error('[RateLimit] Middleware error', { + error: error.message, + path: req.path + }); + + // Fail open - allow request if middleware fails + next(); + } + }; + } + + /** + * Rate limiting middleware for login attempts + */ + loginRateLimit() { + return async (req, res, next) => { + try { + const ip = this.getClientIP(req); + const tenantId = this.getTenantId(req); + + // Check rate limit + const result = await this.rateLimitService.checkLoginRateLimit(ip, tenantId); + + // Add rate limit headers + if (this.addHeaders) { + res.setHeader('X-RateLimit-Limit', result.limit); + res.setHeader('X-RateLimit-Remaining', result.remaining); + res.setHeader('X-RateLimit-Reset', result.reset); + } + + if (!result.allowed) { + logger.warn('[RateLimit] Login rate limit exceeded', { + ip, + tenantId, + retryAfter: result.retryAfter + }); + + res.setHeader('Retry-After', result.retryAfter); + return res.status(429).json({ + error: 'Too Many Login Attempts', + message: 'Too many login attempts. Please try again later.', + code: 'LOGIN_RATE_LIMIT_EXCEEDED', + retryAfter: result.retryAfter, + reset: result.reset + }); + } + + next(); + } catch (error) { + logger.error('[RateLimit] Login middleware error', { + error: error.message + }); + + // Fail open + next(); + } + }; + } + + /** + * Rate limiting middleware for specific endpoints with custom limits + * @param {object} customLimits - Custom rate limits + */ + customRateLimit(customLimits) { + return async (req, res, next) => { + try { + const ip = this.getClientIP(req); + const tenantId = this.getTenantId(req); + const apiKey = this.getAPIKey(req); + + // Skip webhook requests + if (this.enableWebhookWhitelist && this.isWebhookRequest(req)) { + return next(); + } + + // Check IP blacklist + if (this.enableIPBlacklist && this.rateLimitService.isIPBlacklisted(ip)) { + return res.status(403).json({ + error: 'Forbidden', + message: 'Your IP has been blocked', + code: 'IP_BLOCKED' + }); + } + + // Use custom limits + const result = await this.rateLimitService.checkRateLimit( + tenantId ? `custom:${tenantId}` : `custom:${ip}`, + customLimits + ); + + if (this.addHeaders) { + res.setHeader('X-RateLimit-Limit', result.limit); + res.setHeader('X-RateLimit-Remaining', result.remaining); + res.setHeader('X-RateLimit-Reset', result.reset); + } + + if (!result.allowed) { + res.setHeader('Retry-After', result.retryAfter); + return res.status(429).json({ + error: 'Too Many Requests', + message: 'Rate limit exceeded for this endpoint', + code: 'CUSTOM_RATE_LIMIT_EXCEEDED', + retryAfter: result.retryAfter, + reset: result.reset + }); + } + + next(); + } catch (error) { + logger.error('[RateLimit] Custom middleware error', { + error: error.message + }); + + next(); + } + }; + } + + /** + * Middleware to block headless browsers + */ + blockHeadlessBrowsers() { + return (req, res, next) => { + const userAgent = req.headers['user-agent'] || ''; + + // Common headless browser signatures + const headlessSignatures = [ + 'HeadlessChrome', + 'PhantomJS', + 'SlimerJS', + 'HtmlUnit', + 'JSoup', + 'python-requests', + 'curl', + 'wget', + 'axios', + 'node-fetch', + 'Go-http-client', + 'Java', + 'Apache-HttpClient' + ]; + + const isHeadless = headlessSignatures.some(signature => + userAgent.includes(signature) + ); + + if (isHeadless && !this.isWebhookRequest(req)) { + logger.warn('[RateLimit] Blocked headless browser', { + ip: this.getClientIP(req), + userAgent, + path: req.path + }); + + return res.status(403).json({ + error: 'Forbidden', + message: 'Automated browsers are not allowed', + code: 'HEADLESS_BROWSER_BLOCKED' + }); + } + + next(); + }; + } + + /** + * Middleware to require API key authentication + */ + requireAPIKey() { + return (req, res, next) => { + const apiKey = this.getAPIKey(req); + + if (!apiKey) { + return res.status(401).json({ + error: 'Unauthorized', + message: 'API key is required', + code: 'API_KEY_REQUIRED' + }); + } + + // Add API key to request for downstream use + req.apiKey = apiKey; + next(); + }; + } +} + +// Factory function for creating middleware instances +function createRateLimitMiddleware(options = {}) { + const middleware = new RateLimitMiddleware(options); + + return { + apiRateLimit: middleware.apiRateLimit.bind(middleware), + loginRateLimit: middleware.loginRateLimit.bind(middleware), + customRateLimit: middleware.customRateLimit.bind(middleware), + blockHeadlessBrowsers: middleware.blockHeadlessBrowsers.bind(middleware), + requireAPIKey: middleware.requireAPIKey.bind(middleware), + getService: () => middleware.rateLimitService + }; +} + +module.exports = createRateLimitMiddleware; diff --git a/package.json b/package.json index f3fa5f0..2c09f64 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,11 @@ "soroban": "node worker.js --soroban", "soroban:dev": "nodemon worker.js --soroban", "soroban:health": "node worker.js --soroban --health", + "pii-scrub": "node workers/piiScrubbingWorker.js", + "pii-scrub:dry-run": "node workers/piiScrubbingWorker.js 3 --dry-run", "test": "jest", "test:soroban": "jest --testPathPattern=soroban", + "test:pii": "jest piiScrubbing.test.js", "migrate": "node migrations/runMigrations.js", "migrate:rollback": "knex migrate:rollback", "migrate:make": "knex migrate:make", @@ -92,4 +95,4 @@ "ts-node": "^10.9.2", "tsconfig-paths": "^4.2.0" } -} +} \ No newline at end of file diff --git a/piiScrubbing.test.js b/piiScrubbing.test.js new file mode 100644 index 0000000..5cf11c0 --- /dev/null +++ b/piiScrubbing.test.js @@ -0,0 +1,641 @@ +/** + * PII Scrubbing Integration Tests + * + * Deep integration tests verifying that scrubbed users cannot be identified + * through any database join or external query, ensuring GDPR/CCPA compliance. + */ + +const PIIScrubbingService = require('./src/services/piiScrubbingService'); +const Database = require('better-sqlite3'); +const path = require('path'); +const fs = require('fs'); + +describe('PII Scrubbing Integration Tests', () => { + let db; + let piiService; + let testDbPath; + const testWalletAddress = 'GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ'; + const testCreatorId = 'GABCD1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ123456789'; + + beforeAll(() => { + // Create in-memory test database + testDbPath = path.join(__dirname, 'test-pii-scrubbing.db'); + if (fs.existsSync(testDbPath)) { + fs.unlinkSync(testDbPath); + } + + db = new Database(testDbPath); + + // Initialize schema + db.exec(` + PRAGMA foreign_keys = ON; + + CREATE TABLE IF NOT EXISTS creators ( + id TEXT PRIMARY KEY, + created_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS subscriptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + creator_id TEXT NOT NULL, + wallet_address TEXT NOT NULL, + active INTEGER DEFAULT 1, + subscribed_at TEXT NOT NULL, + user_email TEXT, + balance REAL, + daily_spend REAL, + risk_status TEXT, + estimated_run_out_at TEXT, + migrated_from_stripe INTEGER DEFAULT 0, + stripe_plan_id TEXT + ); + + CREATE TABLE IF NOT EXISTS creator_audit_logs ( + id TEXT PRIMARY KEY, + creator_id TEXT, + action_type TEXT, + entity_type TEXT, + entity_id TEXT, + timestamp TEXT, + ip_address TEXT, + metadata_json TEXT, + created_at TEXT + ); + + CREATE TABLE IF NOT EXISTS data_export_tracking ( + id TEXT PRIMARY KEY, + wallet_address TEXT, + requester_email TEXT, + export_type TEXT, + status TEXT, + created_at TEXT + ); + + CREATE TABLE IF NOT EXISTS privacy_preferences ( + wallet_address TEXT PRIMARY KEY, + share_email_with_merchants INTEGER DEFAULT 1 + ); + + CREATE TABLE IF NOT EXISTS comments ( + id TEXT PRIMARY KEY, + post_id TEXT, + user_address TEXT, + creator_id TEXT, + content TEXT, + created_at TEXT, + updated_at TEXT + ); + + CREATE TABLE IF NOT EXISTS leaderboard_entries ( + id TEXT PRIMARY KEY, + creator_address TEXT, + fan_address TEXT, + score INTEGER, + created_at TEXT + ); + + CREATE TABLE IF NOT EXISTS social_tokens ( + id TEXT PRIMARY KEY, + creator_address TEXT, + user_address TEXT, + token TEXT, + active INTEGER DEFAULT 1, + created_at TEXT + ); + `); + + // Insert test data + db.prepare('INSERT INTO creators (id, created_at) VALUES (?, ?)').run(testCreatorId, new Date().toISOString()); + + db.prepare(` + INSERT INTO subscriptions (creator_id, wallet_address, active, subscribed_at, user_email, balance, daily_spend, risk_status) + VALUES (?, ?, 1, ?, ?, 100.0, 10.0, 'active') + `).run(testCreatorId, testWalletAddress, new Date().toISOString(), 'test@example.com', 100.0, 10.0, 'active'); + + db.prepare(` + INSERT INTO creator_audit_logs (id, creator_id, action_type, entity_type, entity_id, timestamp, ip_address, metadata_json, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + require('crypto').randomUUID(), + testCreatorId, + 'subscription_created', + 'subscription', + '1', + new Date().toISOString(), + '192.168.1.100', + JSON.stringify({ wallet_address: testWalletAddress }), + new Date().toISOString() + ); + + db.prepare(` + INSERT INTO data_export_tracking (id, wallet_address, requester_email, export_type, status, created_at) + VALUES (?, ?, ?, ?, ?, ?) + `).run( + require('crypto').randomUUID(), + testWalletAddress, + 'test@example.com', + 'data_export', + 'completed', + new Date().toISOString() + ); + + db.prepare(` + INSERT INTO privacy_preferences (wallet_address, share_email_with_merchants) + VALUES (?, 1) + `).run(testWalletAddress); + + db.prepare(` + INSERT INTO comments (id, post_id, user_address, creator_id, content, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + `).run( + require('crypto').randomUUID(), + 'post-1', + testWalletAddress, + testCreatorId, + 'Test comment', + new Date().toISOString(), + new Date().toISOString() + ); + + db.prepare(` + INSERT INTO leaderboard_entries (id, creator_address, fan_address, score, created_at) + VALUES (?, ?, ?, ?, ?) + `).run( + require('crypto').randomUUID(), + testCreatorId, + testWalletAddress, + 100, + new Date().toISOString() + ); + + db.prepare(` + INSERT INTO social_tokens (id, creator_address, user_address, token, active, created_at) + VALUES (?, ?, ?, ?, 1, ?) + `).run( + require('crypto').randomUUID(), + testCreatorId, + testWalletAddress, + 'test-token', + new Date().toISOString() + ); + + // Initialize PII service + piiService = new PIIScrubbingService({ + database: { db } + }); + }); + + afterAll(() => { + if (db) { + db.close(); + } + if (fs.existsSync(testDbPath)) { + fs.unlinkSync(testDbPath); + } + }); + + describe('Cryptographic Hashing', () => { + test('should produce consistent hashes for the same input', () => { + const value = 'test@example.com'; + const hash1 = piiService.hashValue(value); + const hash2 = piiService.hashValue(value); + + expect(hash1).toBe(hash2); + expect(hash1).toMatch(/^[a-f0-9]{64}$/); // SHA-256 produces 64 hex chars + }); + + test('should produce different hashes for different inputs', () => { + const hash1 = piiService.hashValue('test@example.com'); + const hash2 = piiService.hashValue('other@example.com'); + + expect(hash1).not.toBe(hash2); + }); + + test('should anonymize wallet address with prefix and hash', () => { + const anonymized = piiService.anonymizeWalletAddress(testWalletAddress); + + expect(anonymized).toContain(testWalletAddress.substring(0, 8)); + expect(anonymized).toContain('_'); + expect(anonymized).not.toBe(testWalletAddress); + }); + }); + + describe('Database PII Scrubbing', () => { + test('should scrub PII from subscriptions table', async () => { + await piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }); + + const subscription = db.prepare(` + SELECT user_email, wallet_address, risk_status + FROM subscriptions + WHERE wallet_address = ? OR wallet_address LIKE ? + `).get(testWalletAddress, `${testWalletAddress.substring(0, 8)}%`); + + expect(subscription).toBeDefined(); + expect(subscription.user_email).toContain('scrubbed_'); + expect(subscription.user_email).toContain('@anon.example.com'); + expect(subscription.wallet_address).not.toBe(testWalletAddress); + expect(subscription.risk_status).toBe('scrubbed'); + }); + + test('should preserve financial data in subscriptions', async () => { + await piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }); + + const subscription = db.prepare(` + SELECT balance, daily_spend, creator_id + FROM subscriptions + WHERE wallet_address LIKE ? + `).get(`${testWalletAddress.substring(0, 8)}%`); + + expect(subscription).toBeDefined(); + expect(subscription.balance).toBe(100.0); + expect(subscription.daily_spend).toBe(10.0); + expect(subscription.creator_id).toBe(testCreatorId); + }); + + test('should scrub IP addresses from audit logs', async () => { + await piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }); + + const auditLog = db.prepare(` + SELECT ip_address + FROM creator_audit_logs + WHERE metadata_json LIKE ? + `).get(`%${testWalletAddress}%`); + + expect(auditLog).toBeDefined(); + expect(auditLog.ip_address).toContain('scrubbed_'); + expect(auditLog.ip_address).not.toBe('192.168.1.100'); + }); + + test('should scrub requester email from data export tracking', async () => { + await piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }); + + const exportTracking = db.prepare(` + SELECT requester_email + FROM data_export_tracking + WHERE wallet_address = ? + `).get(testWalletAddress); + + expect(exportTracking).toBeDefined(); + expect(exportTracking.requester_email).toContain('scrubbed_'); + expect(exportTracking.requester_email).toContain('@anon.example.com'); + }); + + test('should update privacy preferences', async () => { + await piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }); + + const privacyPref = db.prepare(` + SELECT share_email_with_merchants + FROM privacy_preferences + WHERE wallet_address = ? + `).get(testWalletAddress); + + expect(privacyPref).toBeDefined(); + expect(privacyPref.share_email_with_merchants).toBe(0); + }); + + test('should scrub user address from comments', async () => { + await piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }); + + const comment = db.prepare(` + SELECT user_address + FROM comments + WHERE user_address = ? OR user_address LIKE ? + `).get(testWalletAddress, `${testWalletAddress.substring(0, 8)}%`); + + expect(comment).toBeDefined(); + expect(comment.user_address).not.toBe(testWalletAddress); + expect(comment.user_address).toContain(testWalletAddress.substring(0, 8)); + }); + + test('should scrub fan address from leaderboard entries', async () => { + await piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }); + + const leaderboard = db.prepare(` + SELECT fan_address + FROM leaderboard_entries + WHERE fan_address = ? OR fan_address LIKE ? + `).get(testWalletAddress, `${testWalletAddress.substring(0, 8)}%`); + + expect(leaderboard).toBeDefined(); + expect(leaderboard.fan_address).not.toBe(testWalletAddress); + expect(leaderboard.fan_address).toContain(testWalletAddress.substring(0, 8)); + }); + + test('should scrub user address from social tokens', async () => { + await piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }); + + const socialToken = db.prepare(` + SELECT user_address + FROM social_tokens + WHERE user_address = ? OR user_address LIKE ? + `).get(testWalletAddress, `${testWalletAddress.substring(0, 8)}%`); + + expect(socialToken).toBeDefined(); + expect(socialToken.user_address).not.toBe(testWalletAddress); + expect(socialToken.user_address).toContain(testWalletAddress.substring(0, 8)); + }); + }); + + describe('Identification Prevention', () => { + beforeEach(async () => { + // Scrub the user before each identification test + await piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }); + }); + + test('cannot identify user by original wallet address in subscriptions', () => { + const result = db.prepare(` + SELECT * FROM subscriptions WHERE wallet_address = ? + `).get(testWalletAddress); + + expect(result).toBeUndefined(); + }); + + test('cannot identify user by email in subscriptions', () => { + const result = db.prepare(` + SELECT * FROM subscriptions WHERE user_email = ? + `).get('test@example.com'); + + expect(result).toBeUndefined(); + }); + + test('cannot identify user by IP address in audit logs', () => { + const result = db.prepare(` + SELECT * FROM creator_audit_logs WHERE ip_address = ? + `).get('192.168.1.100'); + + expect(result).toBeUndefined(); + }); + + test('cannot identify user by email in data export tracking', () => { + const result = db.prepare(` + SELECT * FROM data_export_tracking WHERE requester_email = ? + `).get('test@example.com'); + + expect(result).toBeUndefined(); + }); + + test('cannot identify user by address in comments', () => { + const result = db.prepare(` + SELECT * FROM comments WHERE user_address = ? + `).get(testWalletAddress); + + expect(result).toBeUndefined(); + }); + + test('cannot identify user by address in leaderboard entries', () => { + const result = db.prepare(` + SELECT * FROM leaderboard_entries WHERE fan_address = ? + `).get(testWalletAddress); + + expect(result).toBeUndefined(); + }); + + test('cannot identify user by address in social tokens', () => { + const result = db.prepare(` + SELECT * FROM social_tokens WHERE user_address = ? + `).get(testWalletAddress); + + expect(result).toBeUndefined(); + }); + + test('cannot identify user through database joins', () => { + // Try to join across tables to find the user + const result = db.prepare(` + SELECT s.wallet_address, s.user_email, c.user_address, l.fan_address, st.user_address + FROM subscriptions s + LEFT JOIN comments c ON s.wallet_address = c.user_address + LEFT JOIN leaderboard_entries l ON s.wallet_address = l.fan_address + LEFT JOIN social_tokens st ON s.wallet_address = st.user_address + WHERE s.wallet_address = ? + OR c.user_address = ? + OR l.fan_address = ? + OR st.user_address = ? + `).get(testWalletAddress, testWalletAddress, testWalletAddress, testWalletAddress); + + expect(result).toBeUndefined(); + }); + + test('cannot identify user through pattern matching', () => { + // Try to find user by email pattern + const result = db.prepare(` + SELECT * FROM subscriptions WHERE user_email LIKE '%test@example.com%' + `).get(); + + expect(result).toBeUndefined(); + }); + + test('financial data remains accessible but anonymized', () => { + const result = db.prepare(` + SELECT balance, daily_spend, creator_id, wallet_address + FROM subscriptions + WHERE wallet_address LIKE ? + `).get(`${testWalletAddress.substring(0, 8)}%`); + + expect(result).toBeDefined(); + expect(result.balance).toBe(100.0); + expect(result.daily_spend).toBe(10.0); + expect(result.creator_id).toBe(testCreatorId); + expect(result.wallet_address).not.toBe(testWalletAddress); + }); + }); + + describe('Audit Logging', () => { + test('should create audit log entry for scrubbing operation', async () => { + await piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }); + + const auditLog = db.prepare(` + SELECT * FROM creator_audit_logs + WHERE action_type = 'pii_scrub' + ORDER BY created_at DESC + LIMIT 1 + `).get(); + + expect(auditLog).toBeDefined(); + expect(auditLog.action_type).toBe('pii_scrub'); + expect(auditLog.entity_type).toBe('user'); + + const metadata = JSON.parse(auditLog.metadata_json); + expect(metadata.scrubId).toBeDefined(); + expect(metadata.reason).toBe('test'); + expect(metadata.original_wallet_hash).toBeDefined(); + }); + + test('should log failed scrubbing attempts', async () => { + // This test would require mocking a failure scenario + // For now, we verify the audit log structure + const auditLog = db.prepare(` + SELECT * FROM creator_audit_logs + WHERE action_type = 'pii_scrub' + ORDER BY created_at DESC + LIMIT 1 + `).get(); + + expect(auditLog).toBeDefined(); + expect(auditLog.ip_address).toBe('system'); + }); + }); + + describe('Verification', () => { + test('should verify scrubbing status correctly', async () => { + await piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }); + + const verification = piiService.verifyScrubbing(testWalletAddress); + + expect(verification).toBeDefined(); + expect(verification.walletAddress).toBe(testWalletAddress); + expect(verification.anonymizedAddress).toContain(testWalletAddress.substring(0, 8)); + expect(verification.tables).toBeDefined(); + expect(verification.isScrubbed).toBe(true); + }); + + test('should identify unscrubbed users', () => { + // Insert a new unscrubbed user + const newWallet = 'GABCD1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ123456789'; + db.prepare(` + INSERT INTO subscriptions (creator_id, wallet_address, active, subscribed_at, user_email) + VALUES (?, ?, 1, ?, ?) + `).run(testCreatorId, newWallet, new Date().toISOString(), 'new@example.com'); + + const verification = piiService.verifyScrubbing(newWallet); + + expect(verification.isScrubbed).toBe(false); + }); + }); + + describe('Inactive User Detection', () => { + test('should find inactive users based on retention policy', () => { + // Insert an old inactive subscription + const oldDate = new Date(); + oldDate.setFullYear(oldDate.getFullYear() - 4); + + const oldWallet = 'GOLD1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ123456789'; + db.prepare(` + INSERT INTO subscriptions (creator_id, wallet_address, active, subscribed_at, user_email) + VALUES (?, ?, 0, ?, ?) + `).run(testCreatorId, oldWallet, oldDate.toISOString(), 'old@example.com'); + + const inactiveUsers = piiService.findInactiveUsers(3); + + expect(inactiveUsers).toContain(oldWallet); + }); + + test('should not find active users as inactive', () => { + const activeUsers = piiService.findInactiveUsers(3); + + expect(activeUsers).not.toContain(testWalletAddress); + }); + }); + + describe('Batch Scrubbing', () => { + test('should scrub multiple inactive users', async () => { + // Insert multiple old inactive subscriptions + const oldDate = new Date(); + oldDate.setFullYear(oldDate.getFullYear() - 4); + + const oldWallets = [ + 'GOLD111111111111111111111111111111111111111111111', + 'GOLD222222222222222222222222222222222222222222222', + 'GOLD333333333333333333333333333333333333333333333' + ]; + + for (const wallet of oldWallets) { + db.prepare(` + INSERT INTO subscriptions (creator_id, wallet_address, active, subscribed_at, user_email) + VALUES (?, ?, 0, ?, ?) + `).run(testCreatorId, wallet, oldDate.toISOString(), `${wallet}@example.com`); + } + + const result = await piiService.scrubInactiveUsers(3); + + expect(result.successful).toBeGreaterThan(0); + expect(result.failed).toBe(0); + expect(result.totalUsers).toBe(oldWallets.length); + }); + }); + + describe('Idempotency', () => { + test('should be safe to run scrubbing multiple times', async () => { + // First scrub + await piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }); + + // Second scrub (should not fail) + await expect( + piiService.scrubUserPII(testWalletAddress, { + scrubRedis: false, + sendWebhooks: false, + reason: 'test' + }) + ).resolves.toBeDefined(); + }); + }); + + describe('Security', () => { + test('should use secure salt for hashing', () => { + const service1 = new PIIScrubbingService({ database: { db } }); + const service2 = new PIIScrubbingService({ database: { db } }); + + // Both services should use the same salt from environment + const hash1 = service1.hashValue('test@example.com'); + const hash2 = service2.hashValue('test@example.com'); + + expect(hash1).toBe(hash2); + }); + + test('should prevent dictionary attacks with salt', () => { + const commonEmail = 'common@example.com'; + const hash = piiService.hashValue(commonEmail); + + // Hash should not be predictable without the salt + expect(hash).not.toBe(commonEmail); + expect(hash).not.toContain(commonEmail); + }); + }); +}); diff --git a/routes/compliance.js b/routes/compliance.js new file mode 100644 index 0000000..598e98e --- /dev/null +++ b/routes/compliance.js @@ -0,0 +1,440 @@ +/** + * Compliance API Routes + * + * Provides endpoints for GDPR/CCPA compliance including: + * - Right to be Forgotten (data deletion) + * - Data export requests + * - Privacy preferences management + */ + +const express = require('express'); +const router = express.Router(); +const PIIScrubbingService = require('../services/piiScrubbingService'); +const logger = require('../utils/logger'); + +/** + * POST /api/v1/compliance/forget + * + * Initiates the Right to be Forgotten process for a user. + * This permanently obfuscates PII while preserving financial data for tax compliance. + * + * Request Body: + * { + * "walletAddress": "GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ", + * "reason": "user_request" | "inactive_retention" | "legal_requirement", + * "requestedBy": "user" | "admin" | "system" + * } + * + * Response: + * { + * "success": true, + * "scrubId": "uuid", + * "walletAddress": "GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ", + * "duration": 1234, + * "dbResult": { ... }, + * "redisResult": { ... }, + * "webhookResult": { ... } + * } + */ +router.post('/forget', async (req, res) => { + const { walletAddress, reason = 'user_request', requestedBy = 'user' } = req.body; + + // Validate input + if (!walletAddress) { + return res.status(400).json({ + success: false, + error: 'walletAddress is required' + }); + } + + // Validate wallet address format (basic Stellar address validation) + if (!walletAddress.match(/^G[A-Z0-9]{55}$/)) { + return res.status(400).json({ + success: false, + error: 'Invalid wallet address format' + }); + } + + // Validate reason + const validReasons = ['user_request', 'inactive_retention', 'legal_requirement']; + if (!validReasons.includes(reason)) { + return res.status(400).json({ + success: false, + error: `Invalid reason. Must be one of: ${validReasons.join(', ')}` + }); + } + + logger.info('[Compliance] Forget request received', { + walletAddress, + reason, + requestedBy, + ip: req.ip + }); + + try { + // Initialize PII scrubbing service + const piiService = new PIIScrubbingService({ + database: req.database, + redisClient: req.redisClient, + webhookService: req.webhookService, + auditLogService: req.auditLogService + }); + + // Execute PII scrubbing + const result = await piiService.scrubUserPII(walletAddress, { + scrubRedis: true, + sendWebhooks: true, + reason, + requestedBy + }); + + res.status(200).json({ + success: true, + message: 'PII scrubbing completed successfully', + ...result + }); + } catch (error) { + logger.error('[Compliance] Forget request failed', { + walletAddress, + error: error.message, + stack: error.stack + }); + + res.status(500).json({ + success: false, + error: 'Failed to process forget request', + message: error.message + }); + } +}); + +/** + * GET /api/v1/compliance/forget/:walletAddress/status + * + * Check the scrubbing status of a user's PII. + * + * Response: + * { + * "walletAddress": "GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ", + * "anonymizedAddress": "GD5DQ6ZQZ_abc123...", + * "tables": { + * "subscriptions": { ... }, + * "audit_logs": { ... } + * }, + * "isScrubbed": true + * } + */ +router.get('/forget/:walletAddress/status', async (req, res) => { + const { walletAddress } = req.params; + + // Validate wallet address format + if (!walletAddress.match(/^G[A-Z0-9]{55}$/)) { + return res.status(400).json({ + success: false, + error: 'Invalid wallet address format' + }); + } + + try { + const piiService = new PIIScrubbingService({ + database: req.database + }); + + const verification = piiService.verifyScrubbing(walletAddress); + + res.status(200).json({ + success: true, + ...verification + }); + } catch (error) { + logger.error('[Compliance] Status check failed', { + walletAddress, + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to check scrubbing status', + message: error.message + }); + } +}); + +/** + * POST /api/v1/compliance/forget/batch + * + * Admin-only endpoint to batch scrub inactive users. + * Requires admin authentication. + * + * Request Body: + * { + * "years": 3, + * "dryRun": false + * } + * + * Response: + * { + * "success": true, + * "batchId": "uuid", + * "totalUsers": 100, + * "successful": 95, + * "failed": 5, + * "errors": [ ... ] + * } + */ +router.post('/forget/batch', async (req, res) => { + // Admin authentication check (implement based on your auth system) + if (!req.user || !req.user.isAdmin) { + return res.status(403).json({ + success: false, + error: 'Admin access required' + }); + } + + const { years = 3, dryRun = false } = req.body; + + if (years < 1 || years > 10) { + return res.status(400).json({ + success: false, + error: 'Years must be between 1 and 10' + }); + } + + logger.info('[Compliance] Batch forget request', { + years, + dryRun, + requestedBy: req.user.id + }); + + try { + const piiService = new PIIScrubbingService({ + database: req.database, + redisClient: req.redisClient, + webhookService: req.webhookService, + auditLogService: req.auditLogService + }); + + if (dryRun) { + // Just count inactive users without scrubbing + const inactiveUsers = piiService.findInactiveUsers(years); + + return res.status(200).json({ + success: true, + dryRun: true, + totalUsers: inactiveUsers.length, + message: 'Dry run completed. No data was scrubbed.' + }); + } + + const result = await piiService.scrubInactiveUsers(years); + + res.status(200).json({ + success: true, + message: 'Batch scrubbing completed', + ...result + }); + } catch (error) { + logger.error('[Compliance] Batch forget failed', { + years, + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to process batch forget request', + message: error.message + }); + } +}); + +/** + * GET /api/v1/compliance/audit + * + * Retrieve audit logs for PII scrubbing operations. + * Requires admin authentication. + * + * Query Parameters: + * - limit: Number of records to return (default: 50) + * - offset: Offset for pagination (default: 0) + * + * Response: + * { + * "success": true, + * "logs": [ ... ], + * "total": 100 + * } + */ +router.get('/audit', async (req, res) => { + // Admin authentication check + if (!req.user || !req.user.isAdmin) { + return res.status(403).json({ + success: false, + error: 'Admin access required' + }); + } + + const limit = parseInt(req.query.limit) || 50; + const offset = parseInt(req.query.offset) || 0; + + if (limit > 1000) { + return res.status(400).json({ + success: false, + error: 'Limit cannot exceed 1000' + }); + } + + try { + const logs = req.database.db.prepare(` + SELECT id, creator_id, action_type, entity_type, entity_id, timestamp, ip_address, metadata_json, created_at + FROM creator_audit_logs + WHERE action_type = 'pii_scrub' + ORDER BY created_at DESC + LIMIT ? OFFSET ? + `).all(limit, offset); + + const total = req.database.db.prepare(` + SELECT COUNT(*) as count + FROM creator_audit_logs + WHERE action_type = 'pii_scrub' + `).get().count; + + res.status(200).json({ + success: true, + logs: logs.map(log => ({ + ...log, + metadata: JSON.parse(log.metadata_json) + })), + total, + limit, + offset + }); + } catch (error) { + logger.error('[Compliance] Audit log retrieval failed', { + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to retrieve audit logs', + message: error.message + }); + } +}); + +/** + * POST /api/v1/compliance/export + * + * Export a user's data for GDPR data portability requests. + * + * Request Body: + * { + * "walletAddress": "GD5DQ6ZQZKQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQZQ" + * } + * + * Response: + * { + * "success": true, + * "exportId": "uuid", + * "data": { ... }, + * "format": "json" + * } + */ +router.post('/export', async (req, res) => { + const { walletAddress } = req.body; + + if (!walletAddress) { + return res.status(400).json({ + success: false, + error: 'walletAddress is required' + }); + } + + // Validate wallet address format + if (!walletAddress.match(/^G[A-Z0-9]{55}$/)) { + return res.status(400).json({ + success: false, + error: 'Invalid wallet address format' + }); + } + + logger.info('[Compliance] Data export request', { + walletAddress, + ip: req.ip + }); + + try { + const exportId = require('crypto').randomUUID(); + const exportData = { + exportId, + walletAddress, + exportedAt: new Date().toISOString(), + subscriptions: [], + comments: [], + auditLogs: [] + }; + + // Export subscriptions + const subscriptions = req.database.db.prepare(` + SELECT creator_id, wallet_address, active, subscribed_at, balance, daily_spend, user_email, risk_status + FROM subscriptions + WHERE wallet_address = ? + `).all(walletAddress); + + exportData.subscriptions = subscriptions; + + // Export comments + const comments = req.database.db.prepare(` + SELECT id, post_id, user_address, creator_id, content, created_at, updated_at + FROM comments + WHERE user_address = ? + `).all(walletAddress); + + exportData.comments = comments; + + // Export audit logs (limited to last 100) + const auditLogs = req.database.db.prepare(` + SELECT id, action_type, entity_type, entity_id, timestamp, ip_address, metadata_json + FROM creator_audit_logs + WHERE metadata_json LIKE ? + ORDER BY created_at DESC + LIMIT 100 + `).all(`%${walletAddress}%`); + + exportData.auditLogs = auditLogs.map(log => ({ + ...log, + metadata: JSON.parse(log.metadata_json) + })); + + // Log the export request + req.database.db.prepare(` + INSERT INTO data_export_tracking (id, wallet_address, requester_email, export_type, status, created_at) + VALUES (?, ?, ?, ?, ?, ?) + `).run( + require('crypto').randomUUID(), + walletAddress, + 'user@anon.example.com', + 'pii_export', + 'completed', + new Date().toISOString() + ); + + res.status(200).json({ + success: true, + message: 'Data export completed', + ...exportData + }); + } catch (error) { + logger.error('[Compliance] Data export failed', { + walletAddress, + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to export data', + message: error.message + }); + } +}); + +module.exports = router; diff --git a/routes/rateLimit.js b/routes/rateLimit.js new file mode 100644 index 0000000..2f69a8f --- /dev/null +++ b/routes/rateLimit.js @@ -0,0 +1,526 @@ +/** + * Rate Limit Management API Routes + * + * Admin endpoints for managing rate limits, IP blacklists, and enterprise merchant configurations. + */ + +const express = require('express'); +const router = express.Router(); +const RateLimitService = require('../services/rateLimitService'); +const logger = require('../utils/logger'); + +/** + * GET /api/v1/rate-limit/stats + * + * Get rate limiting statistics + * + * Response: + * { + * "totalRedisKeys": 1000, + * "rateLimitKeys": 500, + * "blacklistedIPs": 10, + * "torExitNodes": 5000, + * "webhookWhitelisted": 8 + * } + */ +router.get('/stats', async (req, res) => { + // Admin authentication check + if (!req.user || !req.user.isAdmin) { + return res.status(403).json({ + success: false, + error: 'Admin access required' + }); + } + + try { + const rateLimitService = new RateLimitService(); + const stats = await rateLimitService.getStatistics(); + + res.status(200).json({ + success: true, + ...stats + }); + } catch (error) { + logger.error('[RateLimit] Stats retrieval failed', { + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to retrieve statistics', + message: error.message + }); + } +}); + +/** + * POST /api/v1/rate-limit/tenant/:tenantId/limits + * + * Update rate limits for a tenant (enterprise) + * + * Request Body: + * { + * "requestsPerMinute": 1000, + * "requestsPerHour": 50000, + * "loginAttemptsPerMinute": 50, + * "loginAttemptsPerHour": 200 + * } + * + * Response: + * { + * "success": true, + * "message": "Rate limits updated" + * } + */ +router.post('/tenant/:tenantId/limits', async (req, res) => { + // Admin authentication check + if (!req.user || !req.user.isAdmin) { + return res.status(403).json({ + success: false, + error: 'Admin access required' + }); + } + + const { tenantId } = req.params; + const { requestsPerMinute, requestsPerHour, loginAttemptsPerMinute, loginAttemptsPerHour } = req.body; + + // Validate input + if (!requestsPerMinute || !requestsPerHour) { + return res.status(400).json({ + success: false, + error: 'requestsPerMinute and requestsPerHour are required' + }); + } + + if (requestsPerMinute < 1 || requestsPerMinute > 10000) { + return res.status(400).json({ + success: false, + error: 'requestsPerMinute must be between 1 and 10000' + }); + } + + try { + const rateLimitService = new RateLimitService(); + const success = await rateLimitService.updateTenantRateLimits(tenantId, { + requestsPerMinute, + requestsPerHour, + loginAttemptsPerMinute: loginAttemptsPerMinute || 50, + loginAttemptsPerHour: loginAttemptsPerHour || 200 + }); + + if (success) { + res.status(200).json({ + success: true, + message: 'Rate limits updated successfully', + tenantId, + limits: { + requestsPerMinute, + requestsPerHour, + loginAttemptsPerMinute, + loginAttemptsPerHour + } + }); + } else { + res.status(500).json({ + success: false, + error: 'Failed to update rate limits' + }); + } + } catch (error) { + logger.error('[RateLimit] Tenant limit update failed', { + tenantId, + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to update rate limits', + message: error.message + }); + } +}); + +/** + * GET /api/v1/rate-limit/tenant/:tenantId/limits + * + * Get rate limits for a tenant + * + * Response: + * { + * "success": true, + * "tenantId": "123", + * "limits": { ... }, + * "isEnterprise": true + * } + */ +router.get('/tenant/:tenantId/limits', async (req, res) => { + const { tenantId } = req.params; + + // Allow tenant to view their own limits or admin to view any + if (!req.user || (req.user.tenantId !== tenantId && !req.user.isAdmin)) { + return res.status(403).json({ + success: false, + error: 'Access denied' + }); + } + + try { + const rateLimitService = new RateLimitService(); + const limits = await rateLimitService.getTenantRateLimits(tenantId, req.user.apiKey); + + res.status(200).json({ + success: true, + tenantId, + limits, + isEnterprise: limits.isEnterprise + }); + } catch (error) { + logger.error('[RateLimit] Tenant limit retrieval failed', { + tenantId, + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to retrieve rate limits', + message: error.message + }); + } +}); + +/** + * POST /api/v1/rate-limit/blacklist/ip + * + * Add IP to blacklist + * + * Request Body: + * { + * "ip": "1.2.3.4", + * "reason": "malicious_activity", + * "ttl": 3600 + * } + */ +router.post('/blacklist/ip', async (req, res) => { + // Admin authentication check + if (!req.user || !req.user.isAdmin) { + return res.status(403).json({ + success: false, + error: 'Admin access required' + }); + } + + const { ip, reason = 'manual', ttl = 3600 } = req.body; + + if (!ip) { + return res.status(400).json({ + success: false, + error: 'IP address is required' + }); + } + + try { + const rateLimitService = new RateLimitService(); + await rateLimitService.blacklistIP(ip, reason, ttl); + + res.status(200).json({ + success: true, + message: 'IP blacklisted successfully', + ip, + reason, + ttl + }); + } catch (error) { + logger.error('[RateLimit] IP blacklist failed', { + ip, + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to blacklist IP', + message: error.message + }); + } +}); + +/** + * DELETE /api/v1/rate-limit/blacklist/ip/:ip + * + * Remove IP from blacklist + */ +router.delete('/blacklist/ip/:ip', async (req, res) => { + // Admin authentication check + if (!req.user || !req.user.isAdmin) { + return res.status(403).json({ + success: false, + error: 'Admin access required' + }); + } + + const { ip } = req.params; + + try { + const rateLimitService = new RateLimitService(); + await rateLimitService.unblacklistIP(ip); + + res.status(200).json({ + success: true, + message: 'IP removed from blacklist', + ip + }); + } catch (error) { + logger.error('[RateLimit] IP unblacklist failed', { + ip, + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to remove IP from blacklist', + message: error.message + }); + } +}); + +/** + * GET /api/v1/rate-limit/blacklist/ip + * + * Get all blacklisted IPs + */ +router.get('/blacklist/ip', async (req, res) => { + // Admin authentication check + if (!req.user || !req.user.isAdmin) { + return res.status(403).json({ + success: false, + error: 'Admin access required' + }); + } + + try { + const rateLimitService = new RateLimitService(); + const blacklist = Array.from(rateLimitService.ipBlacklist); + + res.status(200).json({ + success: true, + count: blacklist.length, + ips: blacklist + }); + } catch (error) { + logger.error('[RateLimit] Blacklist retrieval failed', { + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to retrieve blacklist', + message: error.message + }); + } +}); + +/** + * POST /api/v1/rate-limit/tor/add + * + * Add Tor exit node + * + * Request Body: + * { + * "ip": "1.2.3.4" + * } + */ +router.post('/tor/add', async (req, res) => { + // Admin authentication check + if (!req.user || !req.user.isAdmin) { + return res.status(403).json({ + success: false, + error: 'Admin access required' + }); + } + + const { ip } = req.body; + + if (!ip) { + return res.status(400).json({ + success: false, + error: 'IP address is required' + }); + } + + try { + const rateLimitService = new RateLimitService(); + await rateLimitService.addTorExitNode(ip); + + res.status(200).json({ + success: true, + message: 'Tor exit node added', + ip + }); + } catch (error) { + logger.error('[RateLimit] Tor node add failed', { + ip, + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to add Tor exit node', + message: error.message + }); + } +}); + +/** + * DELETE /api/v1/rate-limit/tor/:ip + * + * Remove Tor exit node + */ +router.delete('/tor/:ip', async (req, res) => { + // Admin authentication check + if (!req.user || !req.user.isAdmin) { + return res.status(403).json({ + success: false, + error: 'Admin access required' + }); + } + + const { ip } = req.params; + + try { + const rateLimitService = new RateLimitService(); + await rateLimitService.removeTorExitNode(ip); + + res.status(200).json({ + success: true, + message: 'Tor exit node removed', + ip + }); + } catch (error) { + logger.error('[RateLimit] Tor node remove failed', { + ip, + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to remove Tor exit node', + message: error.message + }); + } +}); + +/** + * GET /api/v1/rate-limit/tor + * + * Get all Tor exit nodes + */ +router.get('/tor', async (req, res) => { + // Admin authentication check + if (!req.user || !req.user.isAdmin) { + return res.status(403).json({ + success: false, + error: 'Admin access required' + }); + } + + try { + const rateLimitService = new RateLimitService(); + const torNodes = Array.from(rateLimitService.torExitNodes); + + res.status(200).json({ + success: true, + count: torNodes.length, + nodes: torNodes + }); + } catch (error) { + logger.error('[RateLimit] Tor nodes retrieval failed', { + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to retrieve Tor exit nodes', + message: error.message + }); + } +}); + +/** + * POST /api/v1/rate-limit/reset/:key + * + * Reset rate limit for a specific key + * + * Request Body: + * { + * "key": "api:1.2.3.4" + * } + */ +router.post('/reset/:key', async (req, res) => { + // Admin authentication check + if (!req.user || !req.user.isAdmin) { + return res.status(403).json({ + success: false, + error: 'Admin access required' + }); + } + + const { key } = req.params; + + try { + const rateLimitService = new RateLimitService(); + await rateLimitService.resetRateLimit(key); + + res.status(200).json({ + success: true, + message: 'Rate limit reset', + key + }); + } catch (error) { + logger.error('[RateLimit] Rate limit reset failed', { + key, + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to reset rate limit', + message: error.message + }); + } +}); + +/** + * POST /api/v1/rate-limit/cleanup + * + * Trigger cleanup of expired rate limit entries + */ +router.post('/cleanup', async (req, res) => { + // Admin authentication check + if (!req.user || !req.user.isAdmin) { + return res.status(403).json({ + success: false, + error: 'Admin access required' + }); + } + + try { + const rateLimitService = new RateLimitService(); + await rateLimitService.cleanup(); + + res.status(200).json({ + success: true, + message: 'Cleanup completed' + }); + } catch (error) { + logger.error('[RateLimit] Cleanup failed', { + error: error.message + }); + + res.status(500).json({ + success: false, + error: 'Failed to perform cleanup', + message: error.message + }); + } +}); + +module.exports = router; diff --git a/scripts/migrate-init-container.js b/scripts/migrate-init-container.js new file mode 100644 index 0000000..9a9e493 --- /dev/null +++ b/scripts/migrate-init-container.js @@ -0,0 +1,332 @@ +#!/usr/bin/env node +/** + * Idempotent Migration Script for Kubernetes initContainer + * + * This script is designed to run in a Kubernetes initContainer with the following properties: + * - Completely idempotent: can be run multiple times safely + * - Distributed execution safe: uses database-level locking + * - Exit code 0 on success, 1 on failure (prevents pod startup) + * - Comprehensive logging for debugging + * - Vault integration for secure credential fetching + * + * Usage: node scripts/migrate-init-container.js + */ + +const knex = require('knex'); +const path = require('path'); + +// Configuration +const MIGRATION_LOCK_TIMEOUT = parseInt(process.env.MIGRATION_LOCK_TIMEOUT || '300000'); // 5 minutes +const MIGRATION_TIMEOUT = parseInt(process.env.MIGRATION_TIMEOUT || '1800000'); // 30 minutes for long-running migrations +const LOCK_TABLE = '_migration_locks'; +const LONG_RUNNING_THRESHOLD = 60000; // 1 minute - log progress for migrations taking longer + +class InitContainerMigrationRunner { + constructor() { + this.knex = null; + this.lockAcquired = false; + this.startTime = Date.now(); + } + + /** + * Main execution flow + */ + async run() { + try { + console.log('[InitContainer] Starting migration process...'); + console.log('[InitContainer] Timestamp:', new Date().toISOString()); + + // Initialize database connection + await this.initializeDatabase(); + + // Acquire distributed lock to prevent race conditions + await this.acquireLock(); + + // Run migrations with timeout protection + await this.runMigrationsWithTimeout(); + + // Release lock + await this.releaseLock(); + + // Cleanup + await this.cleanup(); + + const duration = Date.now() - this.startTime; + console.log(`[InitContainer] Migration completed successfully in ${duration}ms`); + console.log('[InitContainer] Exiting with code 0 (success)'); + process.exit(0); + } catch (error) { + console.error('[InitContainer] Migration failed:', error); + + // Attempt to release lock if acquired + if (this.lockAcquired) { + try { + await this.releaseLock(); + } catch (lockError) { + console.error('[InitContainer] Failed to release lock:', lockError); + } + } + + // Cleanup connection + try { + await this.cleanup(); + } catch (cleanupError) { + console.error('[InitContainer] Cleanup failed:', cleanupError); + } + + console.error('[InitContainer] Exiting with code 1 (failure - will prevent pod startup)'); + process.exit(1); + } + } + + /** + * Initialize database connection from environment variables + */ + async initializeDatabase() { + const dbConfig = { + client: 'better-sqlite3', + connection: { + filename: process.env.DATABASE_FILENAME || '/app/data/substream.db', + }, + useNullAsDefault: true, + migrations: { + directory: './migrations/knex', + extension: 'js', + loadExtensions: ['.js'], + }, + pool: { + min: 1, + max: 5, + acquireTimeoutMillis: 30000, + createTimeoutMillis: 5000, + destroyTimeoutMillis: 5000, + idleTimeoutMillis: 30000, + }, + }; + + console.log('[InitContainer] Initializing database connection...'); + console.log('[InitContainer] Database file:', dbConfig.connection.filename); + + this.knex = knex(dbConfig); + + // Test connection + try { + await this.knex.raw('SELECT 1'); + console.log('[InitContainer] Database connection successful'); + } catch (error) { + throw new Error(`Database connection failed: ${error.message}`); + } + + // Ensure lock table exists + await this.ensureLockTable(); + } + + /** + * Ensure the migration lock table exists + */ + async ensureLockTable() { + const hasTable = await this.knex.schema.hasTable(LOCK_TABLE); + + if (!hasTable) { + console.log('[InitContainer] Creating migration lock table...'); + await this.knex.schema.createTable(LOCK_TABLE, (table) => { + table.string('lock_key').primary(); + table.timestamp('acquired_at'); + table.timestamp('expires_at'); + table.string('node_id'); + table.string('pod_name'); + table.string('namespace'); + }); + console.log('[InitContainer] Lock table created'); + } + } + + /** + * Acquire distributed lock to prevent concurrent migrations + */ + async acquireLock() { + const lockKey = 'schema_migration'; + const nodeId = process.env.HOSTNAME || 'unknown'; + const podName = process.env.POD_NAME || 'unknown'; + const namespace = process.env.NAMESPACE || 'default'; + const now = new Date(); + const expiresAt = new Date(Date.now() + MIGRATION_LOCK_TIMEOUT); + + console.log('[InitContainer] Attempting to acquire migration lock...'); + console.log(`[InitContainer] Node ID: ${nodeId}, Pod: ${podName}, Namespace: ${namespace}`); + + // Check for existing lock + const existingLock = await this.knex(LOCK_TABLE) + .where('lock_key', lockKey) + .first(); + + if (existingLock) { + const expiresAtTime = new Date(existingLock.expires_at); + + // Check if lock is expired + if (expiresAtTime > now) { + const timeRemaining = Math.floor((expiresAtTime - now) / 1000); + console.warn(`[InitContainer] Lock already held by ${existingLock.node_id} (${existingLock.pod_name})`); + console.warn(`[InitContainer] Lock expires in ${timeRemaining} seconds`); + console.warn('[InitContainer] Waiting for lock to be released...'); + + // Wait for lock to be released (with timeout) + const maxWaitTime = MIGRATION_LOCK_TIMEOUT; + const checkInterval = 5000; + let waitedTime = 0; + + while (waitedTime < maxWaitTime) { + await new Promise(resolve => setTimeout(resolve, checkInterval)); + waitedTime += checkInterval; + + const currentLock = await this.knex(LOCK_TABLE) + .where('lock_key', lockKey) + .first(); + + if (!currentLock || new Date(currentLock.expires_at) <= new Date()) { + console.log('[InitContainer] Lock released, proceeding...'); + break; + } + + console.log(`[InitContainer] Still waiting for lock... (${waitedTime / 1000}s/${maxWaitTime / 1000}s)`); + } + + if (waitedTime >= maxWaitTime) { + throw new Error('Timeout waiting for migration lock'); + } + } else { + console.log('[InitContainer] Existing lock expired, cleaning up...'); + await this.knex(LOCK_TABLE).where('lock_key', lockKey).del(); + } + } + + // Acquire lock + await this.knex(LOCK_TABLE) + .insert({ + lock_key: lockKey, + acquired_at: now.toISOString(), + expires_at: expiresAt.toISOString(), + node_id: nodeId, + pod_name: podName, + namespace: namespace, + }); + + this.lockAcquired = true; + console.log('[InitContainer] Migration lock acquired successfully'); + } + + /** + * Release the migration lock + */ + async releaseLock() { + if (!this.lockAcquired) { + return; + } + + const lockKey = 'schema_migration'; + console.log('[InitContainer] Releasing migration lock...'); + + await this.knex(LOCK_TABLE) + .where('lock_key', lockKey) + .del(); + + this.lockAcquired = false; + console.log('[InitContainer] Migration lock released'); + } + + /** + * Run migrations with timeout protection + */ + async runMigrationsWithTimeout() { + console.log('[InitContainer] Running migrations...'); + + // Set up timeout for long-running migrations + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject(new Error(`Migration timeout after ${MIGRATION_TIMEOUT}ms`)); + }, MIGRATION_TIMEOUT); + }); + + // Run migrations + const migrationPromise = this.runMigrations(); + + // Race between migration and timeout + await Promise.race([migrationPromise, timeoutPromise]); + } + + /** + * Execute Knex migrations + */ + async runMigrations() { + try { + const migrationStartTime = Date.now(); + const [batchNo, log] = await this.knex.migrate.latest(); + + if (log.length === 0) { + console.log('[InitContainer] No new migrations to run (already up to date)'); + } else { + const migrationDuration = Date.now() - migrationStartTime; + console.log(`[InitContainer] Successfully ran ${log.length} migrations in batch ${batchNo}`); + log.forEach((migration) => { + console.log(`[InitContainer] - ${migration.name}`); + }); + console.log(`[InitContainer] Migration batch completed in ${migrationDuration}ms`); + + // Log warning if migration took longer than threshold + if (migrationDuration > LONG_RUNNING_THRESHOLD) { + console.warn(`[InitContainer] WARNING: Migration took ${migrationDuration}ms, which exceeds the ${LONG_RUNNING_THRESHOLD}ms threshold`); + console.warn('[InitContainer] Consider increasing MIGRATION_TIMEOUT or optimizing the migration'); + } + } + + // Verify migration status + await this.verifyMigrationStatus(); + } catch (error) { + throw new Error(`Migration execution failed: ${error.message}`); + } + } + + /** + * Verify that migrations were applied correctly + */ + async verifyMigrationStatus() { + console.log('[InitContainer] Verifying migration status...'); + + // Get current migration status + const completed = await this.knex.migrate.currentVersion(); + console.log(`[InitContainer] Current migration version: ${completed}`); + + // Check for any pending migrations + const allMigrations = await this.knex.migrate.list(); + const pending = allMigrations[1]; // Second element is pending migrations + + if (pending.length > 0) { + console.warn(`[InitContainer] WARNING: ${pending.length} migrations still pending`); + pending.forEach((migration) => { + console.warn(`[InitContainer] - ${migration.name}`); + }); + throw new Error('Not all migrations were applied successfully'); + } + + console.log('[InitContainer] Migration verification passed'); + } + + /** + * Cleanup database connections + */ + async cleanup() { + if (this.knex) { + console.log('[InitContainer] Closing database connection...'); + await this.knex.destroy(); + console.log('[InitContainer] Database connection closed'); + } + } +} + +// Execute if run directly +if (require.main === module) { + const runner = new InitContainerMigrationRunner(); + runner.run(); +} + +module.exports = { InitContainerMigrationRunner }; diff --git a/src/services/piiScrubbingService.js b/src/services/piiScrubbingService.js new file mode 100644 index 0000000..bb7dc71 --- /dev/null +++ b/src/services/piiScrubbingService.js @@ -0,0 +1,687 @@ +/** + * PII (Personally Identifiable Information) Scrubbing Service + * + * This service implements GDPR/CCPA compliant data deletion by: + * - Cryptographically hashing PII with secure salt + * - Preserving financial data for tax compliance while anonymizing user identity + * - Scrubbing Redis caches + * - Sending merchant webhooks + * - Maintaining immutable audit logs + * + * One-way hashing prevents reversal while allowing data correlation for accounting. + */ + +const crypto = require('crypto'); +const logger = require('../utils/logger'); + +class PIIScrubbingService { + constructor({ database, redisClient, webhookService, auditLogService } = {}) { + this.database = database; + this.redisClient = redisClient; + this.webhookService = webhookService; + this.auditLogService = auditLogService; + + // Secure salt for hashing (should be from environment variable in production) + this.salt = process.env.PII_SCRUBBING_SALT || crypto.randomBytes(32).toString('hex'); + + // Hash algorithm (SHA-256 is secure and fast) + this.hashAlgorithm = 'sha256'; + + // Retention period for inactive users (3 years) + this.inactiveRetentionYears = parseInt(process.env.INACTIVE_RETENTION_YEARS || '3'); + } + + /** + * Hash a value with salt using SHA-256 + * @param {string} value - Value to hash + * @returns {string} Hashed value + */ + hashValue(value) { + if (!value) return null; + + return crypto + .createHmac(this.hashAlgorithm, this.salt) + .update(value) + .digest('hex'); + } + + /** + * Anonymize a wallet address for financial records + * @param {string} walletAddress - Original wallet address + * @returns {string} Anonymized wallet address + */ + anonymizeWalletAddress(walletAddress) { + if (!walletAddress) return null; + + // Keep first 8 chars for debugging, hash the rest + const prefix = walletAddress.substring(0, 8); + const suffix = this.hashValue(walletAddress).substring(0, 40); + + return `${prefix}_${suffix}`; + } + + /** + * Scrub PII for a user by wallet address + * @param {string} walletAddress - User's wallet address + * @param {object} options - Scrubbing options + * @returns {object} Scrubbing result + */ + async scrubUserPII(walletAddress, options = {}) { + const { + scrubRedis = true, + sendWebhooks = true, + reason = 'user_request', + requestedBy = 'system' + } = options; + + const scrubId = crypto.randomUUID(); + const startTime = Date.now(); + + logger.info('[PIIScrubbing] Starting PII scrub', { + scrubId, + walletAddress, + reason, + requestedBy + }); + + try { + // Step 1: Scrub database PII + const dbResult = await this.scrubDatabasePII(walletAddress, scrubId); + + // Step 2: Scrub Redis caches + let redisResult = { success: true, keysScrubbed: 0 }; + if (scrubRedis && this.redisClient) { + redisResult = await this.scrubRedisCache(walletAddress, scrubId); + } + + // Step 3: Send merchant webhooks + let webhookResult = { success: true, webhooksSent: 0 }; + if (sendWebhooks && this.webhookService) { + webhookResult = await this.sendForgetWebhooks(walletAddress, scrubId, reason); + } + + // Step 4: Log audit trail + await this.logScrubbingAudit(walletAddress, scrubId, { + reason, + requestedBy, + dbResult, + redisResult, + webhookResult, + duration: Date.now() - startTime + }); + + const duration = Date.now() - startTime; + logger.info('[PIIScrubbing] PII scrub completed', { + scrubId, + walletAddress, + duration, + dbResult, + redisResult, + webhookResult + }); + + return { + success: true, + scrubId, + walletAddress, + duration, + dbResult, + redisResult, + webhookResult + }; + } catch (error) { + logger.error('[PIIScrubbing] PII scrub failed', { + scrubId, + walletAddress, + error: error.message, + stack: error.stack + }); + + // Log failure to audit + await this.logScrubbingAudit(walletAddress, scrubId, { + reason, + requestedBy, + error: error.message, + duration: Date.now() - startTime, + success: false + }); + + throw error; + } + } + + /** + * Scrub PII from database tables + * @param {string} walletAddress - User's wallet address + * @param {string} scrubId - Scrubbing operation ID + * @returns {object} Database scrubbing result + */ + async scrubDatabasePII(walletAddress, scrubId) { + const results = { + success: true, + tablesScrubbed: [], + errors: [] + }; + + const anonymizedAddress = this.anonymizeWalletAddress(walletAddress); + const hashedAddress = this.hashValue(walletAddress); + + // Table 1: subscriptions - Scrub email, preserve financial data + try { + const updateResult = this.database.db.prepare(` + UPDATE subscriptions + SET user_email = ?, + wallet_address = ?, + risk_status = 'scrubbed' + WHERE wallet_address = ? + `).run( + `scrubbed_${hashedAddress}@anon.example.com`, + anonymizedAddress, + walletAddress + ); + + if (updateResult.changes > 0) { + results.tablesScrubbed.push({ + table: 'subscriptions', + rowsUpdated: updateResult.changes, + fieldsScrubbed: ['user_email', 'wallet_address'] + }); + } + } catch (error) { + results.errors.push({ table: 'subscriptions', error: error.message }); + results.success = false; + } + + // Table 2: creator_audit_logs - Scrub IP addresses + try { + const updateResult = this.database.db.prepare(` + UPDATE creator_audit_logs + SET ip_address = ? + WHERE ip_address IN ( + SELECT ip_address FROM subscriptions WHERE wallet_address = ? + ) + `).run(`scrubbed_${hashedAddress}`); + + if (updateResult.changes > 0) { + results.tablesScrubbed.push({ + table: 'creator_audit_logs', + rowsUpdated: updateResult.changes, + fieldsScrubbed: ['ip_address'] + }); + } + } catch (error) { + results.errors.push({ table: 'creator_audit_logs', error: error.message }); + results.success = false; + } + + // Table 3: api_key_audit_logs - Scrub IP addresses + try { + // This table exists in PostgreSQL, handle accordingly + if (this.database.db && this.database.db.migrate) { + // PostgreSQL handling would go here + logger.info('[PIIScrubbing] Skipping api_key_audit_logs (PostgreSQL table)'); + } + } catch (error) { + logger.warn('[PIIScrubbing] api_key_audit_logs scrub error', { error: error.message }); + } + + // Table 4: data_export_tracking - Scrub requester email + try { + const updateResult = this.database.db.prepare(` + UPDATE data_export_tracking + SET requester_email = ? + WHERE requester_email IN ( + SELECT user_email FROM subscriptions WHERE wallet_address = ? + ) + `).run(`scrubbed_${hashedAddress}@anon.example.com`, walletAddress); + + if (updateResult.changes > 0) { + results.tablesScrubbed.push({ + table: 'data_export_tracking', + rowsUpdated: updateResult.changes, + fieldsScrubbed: ['requester_email'] + }); + } + } catch (error) { + results.errors.push({ table: 'data_export_tracking', error: error.message }); + results.success = false; + } + + // Table 5: privacy_preferences - Mark as scrubbed + try { + const updateResult = this.database.db.prepare(` + UPDATE privacy_preferences + SET share_email_with_merchants = 0 + WHERE wallet_address = ? + `).run(walletAddress); + + if (updateResult.changes > 0) { + results.tablesScrubbed.push({ + table: 'privacy_preferences', + rowsUpdated: updateResult.changes, + fieldsScrubbed: ['share_email_with_merchants'] + }); + } + } catch (error) { + results.errors.push({ table: 'privacy_preferences', error: error.message }); + results.success = false; + } + + // Table 6: comments - Scrub user_address in comments + try { + const updateResult = this.database.db.prepare(` + UPDATE comments + SET user_address = ? + WHERE user_address = ? + `).run(anonymizedAddress, walletAddress); + + if (updateResult.changes > 0) { + results.tablesScrubbed.push({ + table: 'comments', + rowsUpdated: updateResult.changes, + fieldsScrubbed: ['user_address'] + }); + } + } catch (error) { + results.errors.push({ table: 'comments', error: error.message }); + results.success = false; + } + + // Table 7: leaderboard tables - Scrub fan_address + try { + const tables = [ + 'leaderboard_entries', + 'leaderboard_content_engagement', + 'leaderboard_seasonal' + ]; + + for (const table of tables) { + try { + const updateResult = this.database.db.prepare(` + UPDATE ${table} + SET fan_address = ? + WHERE fan_address = ? + `).run(anonymizedAddress, walletAddress); + + if (updateResult.changes > 0) { + results.tablesScrubbed.push({ + table, + rowsUpdated: updateResult.changes, + fieldsScrubbed: ['fan_address'] + }); + } + } catch (error) { + // Table might not exist, continue + logger.debug(`[PIIScrubbing] Table ${table} not found or error`, { error: error.message }); + } + } + } catch (error) { + logger.warn('[PIIScrubbing] Leaderboard scrub error', { error: error.message }); + } + + // Table 8: social tokens - Scrub user_address + try { + const updateResult = this.database.db.prepare(` + UPDATE social_tokens + SET user_address = ? + WHERE user_address = ? + `).run(anonymizedAddress, walletAddress); + + if (updateResult.changes > 0) { + results.tablesScrubbed.push({ + table: 'social_tokens', + rowsUpdated: updateResult.changes, + fieldsScrubbed: ['user_address'] + }); + } + } catch (error) { + results.errors.push({ table: 'social_tokens', error: error.message }); + results.success = false; + } + + return results; + } + + /** + * Scrub Redis cache entries for a user + * @param {string} walletAddress - User's wallet address + * @param {string} scrubId - Scrubbing operation ID + * @returns {object} Redis scrubbing result + */ + async scrubRedisCache(walletAddress, scrubId) { + if (!this.redisClient) { + logger.warn('[PIIScrubbing] Redis client not available, skipping cache scrub'); + return { success: true, keysScrubbed: 0, skipped: true }; + } + + const result = { + success: true, + keysScrubbed: 0, + errors: [] + }; + + try { + // Common Redis key patterns for user data + const patterns = [ + `user:${walletAddress}:*`, + `profile:${walletAddress}:*`, + `subscription:${walletAddress}:*`, + `creator:${walletAddress}:*`, + `session:${walletAddress}:*`, + `cache:${walletAddress}:*` + ]; + + for (const pattern of patterns) { + try { + const keys = await this.redisClient.keys(pattern); + + if (keys.length > 0) { + await this.redisClient.del(keys); + result.keysScrubbed += keys.length; + logger.info('[PIIScrubbing] Scrubbed Redis keys', { + pattern, + count: keys.length + }); + } + } catch (error) { + result.errors.push({ pattern, error: error.message }); + logger.warn('[PIIScrubbing] Redis pattern scrub error', { + pattern, + error: error.message + }); + } + } + + logger.info('[PIIScrubbing] Redis cache scrub completed', { + scrubId, + walletAddress, + keysScrubbed: result.keysScrubbed + }); + } catch (error) { + result.success = false; + result.errors.push({ error: error.message }); + logger.error('[PIIScrubbing] Redis scrub failed', { + scrubId, + walletAddress, + error: error.message + }); + } + + return result; + } + + /** + * Send forget webhooks to affected merchants + * @param {string} walletAddress - User's wallet address + * @param {string} scrubId - Scrubbing operation ID + * @param {string} reason - Reason for scrubbing + * @returns {object} Webhook result + */ + async sendForgetWebhooks(walletAddress, scrubId, reason) { + const result = { + success: true, + webhooksSent: 0, + errors: [] + }; + + try { + // Find all creators this user subscribed to + const subscriptions = this.database.db.prepare(` + SELECT DISTINCT creator_id + FROM subscriptions + WHERE wallet_address = ? AND active = 1 + `).all(walletAddress); + + logger.info('[PIIScrubbing] Found subscriptions for webhook notification', { + scrubId, + walletAddress, + subscriptionCount: subscriptions.length + }); + + for (const subscription of subscriptions) { + try { + const creator = this.database.db.prepare(` + SELECT webhook_url, webhook_secret + FROM creators + WHERE id = ? + `).get(subscription.creator_id); + + if (creator && creator.webhook_url) { + const webhookPayload = { + event: 'user.forget', + timestamp: new Date().toISOString(), + scrub_id: scrubId, + data: { + anonymized_wallet_address: this.anonymizeWalletAddress(walletAddress), + reason, + scrubbed_at: new Date().toISOString() + } + }; + + // Send webhook (implementation depends on webhookService) + if (this.webhookService && typeof this.webhookService.send === 'function') { + await this.webhookService.send(creator.webhook_url, webhookPayload, { + secret: creator.webhook_secret + }); + result.webhooksSent++; + } else { + logger.warn('[PIIScrubbing] Webhook service not available', { + creatorId: subscription.creator_id + }); + } + } + } catch (error) { + result.errors.push({ + creatorId: subscription.creator_id, + error: error.message + }); + logger.warn('[PIIScrubbing] Webhook send failed', { + creatorId: subscription.creator_id, + error: error.message + }); + } + } + + logger.info('[PIIScrubbing] Webhook notifications completed', { + scrubId, + walletAddress, + webhooksSent: result.webhooksSent + }); + } catch (error) { + result.success = false; + result.errors.push({ error: error.message }); + logger.error('[PIIScrubbing] Webhook notification failed', { + scrubId, + walletAddress, + error: error.message + }); + } + + return result; + } + + /** + * Log scrubbing operation to audit trail + * @param {string} walletAddress - User's wallet address + * @param {string} scrubId - Scrubbing operation ID + * @param {object} metadata - Operation metadata + */ + async logScrubbingAudit(walletAddress, scrubId, metadata) { + try { + const auditEntry = { + id: crypto.randomUUID(), + action_type: 'pii_scrub', + entity_type: 'user', + entity_id: this.anonymizeWalletAddress(walletAddress), + timestamp: new Date().toISOString(), + ip_address: 'system', + metadata_json: JSON.stringify({ + scrubId, + original_wallet_hash: this.hashValue(walletAddress), + ...metadata + }), + created_at: new Date().toISOString() + }; + + this.database.db.prepare(` + INSERT INTO creator_audit_logs + (id, creator_id, action_type, entity_type, entity_id, timestamp, ip_address, metadata_json, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + auditEntry.id, + 'system', + auditEntry.action_type, + auditEntry.entity_type, + auditEntry.entity_id, + auditEntry.timestamp, + auditEntry.ip_address, + auditEntry.metadata_json, + auditEntry.created_at + ); + + logger.info('[PIIScrubbing] Audit log entry created', { + scrubId, + auditId: auditEntry.id + }); + } catch (error) { + logger.error('[PIIScrubbing] Failed to create audit log', { + scrubId, + walletAddress, + error: error.message + }); + } + } + + /** + * Find inactive users for automated scrubbing + * @param {number} years - Number of years of inactivity + * @returns {Array} Inactive user wallet addresses + */ + findInactiveUsers(years = 3) { + const cutoffDate = new Date(); + cutoffDate.setFullYear(cutoffDate.getFullYear() - years); + + const inactiveUsers = this.database.db.prepare(` + SELECT DISTINCT wallet_address + FROM subscriptions + WHERE subscribed_at < ? + AND active = 0 + `).all(cutoffDate.toISOString()); + + logger.info('[PIIScrubbing] Found inactive users', { + years, + count: inactiveUsers.length + }); + + return inactiveUsers.map(row => row.wallet_address); + } + + /** + * Automated scrubbing of inactive users + * @param {number} years - Years of inactivity threshold + * @returns {object} Batch scrubbing result + */ + async scrubInactiveUsers(years = 3) { + const batchId = crypto.randomUUID(); + const startTime = Date.now(); + + logger.info('[PIIScrubbing] Starting batch scrub of inactive users', { + batchId, + years + }); + + const inactiveUsers = this.findInactiveUsers(years); + const results = { + batchId, + totalUsers: inactiveUsers.length, + successful: 0, + failed: 0, + errors: [] + }; + + for (const walletAddress of inactiveUsers) { + try { + await this.scrubUserPII(walletAddress, { + scrubRedis: true, + sendWebhooks: true, + reason: 'inactive_retention_policy', + requestedBy: 'automated_cron' + }); + results.successful++; + } catch (error) { + results.failed++; + results.errors.push({ + walletAddress: this.anonymizeWalletAddress(walletAddress), + error: error.message + }); + logger.error('[PIIScrubbing] Failed to scrub inactive user', { + batchId, + walletAddress, + error: error.message + }); + } + } + + const duration = Date.now() - startTime; + logger.info('[PIIScrubbing] Batch scrub completed', { + batchId, + duration, + results + }); + + return results; + } + + /** + * Verify that a user's PII has been scrubbed + * @param {string} walletAddress - Original wallet address + * @returns {object} Verification result + */ + verifyScrubbing(walletAddress) { + const anonymizedAddress = this.anonymizeWalletAddress(walletAddress); + const hashedAddress = this.hashValue(walletAddress); + + const verification = { + walletAddress, + anonymizedAddress, + tables: {}, + isScrubbed: true + }; + + // Check subscriptions + const subscription = this.database.db.prepare(` + SELECT user_email, wallet_address, risk_status + FROM subscriptions + WHERE wallet_address = ? OR wallet_address = ? + LIMIT 1 + `).get(walletAddress, anonymizedAddress); + + if (subscription) { + verification.tables.subscriptions = { + found: true, + emailScrubbed: subscription.user_email?.includes('scrubbed_'), + addressAnonymized: subscription.wallet_address === anonymizedAddress, + riskStatus: subscription.risk_status + }; + verification.isScrubbed = verification.isScrubbed && subscription.user_email?.includes('scrubbed_'); + } else { + verification.tables.subscriptions = { found: false }; + } + + // Check audit logs + const auditLog = this.database.db.prepare(` + SELECT * FROM creator_audit_logs + WHERE metadata_json LIKE ? + LIMIT 1 + `).get(`%${hashedAddress}%`); + + verification.tables.audit_logs = { + found: !!auditLog, + hasScrubbingRecord: !!auditLog + }; + + return verification; + } +} + +module.exports = PIIScrubbingService; diff --git a/src/services/rateLimitService.js b/src/services/rateLimitService.js new file mode 100644 index 0000000..38a8f37 --- /dev/null +++ b/src/services/rateLimitService.js @@ -0,0 +1,480 @@ +/** + * Redis-Backed Token Bucket Rate Limiter + * + * Implements a distributed token bucket algorithm using Redis for: + * - Application-layer rate limiting + * - Differentiated limits for authenticated vs anonymous traffic + * - Dynamic rate limit updates for enterprise merchants + * - Precise boundary enforcement with HTTP 429 responses + */ + +const Redis = require('ioredis'); +const logger = require('../utils/logger'); + +class RateLimitService { + constructor({ redisUrl, redisOptions } = {}) { + this.redis = new Redis(redisUrl || process.env.REDIS_URL || 'redis://localhost:6379', redisOptions); + + // Default rate limits + this.defaultLimits = { + anonymous: { + requestsPerMinute: 100, + requestsPerHour: 1000, + loginAttemptsPerMinute: 5, + loginAttemptsPerHour: 20 + }, + authenticated: { + requestsPerMinute: 300, + requestsPerHour: 5000, + loginAttemptsPerMinute: 10, + loginAttemptsPerHour: 50 + }, + enterprise: { + requestsPerMinute: 1000, + requestsPerHour: 50000, + loginAttemptsPerMinute: 50, + loginAttemptsPerHour: 200 + } + }; + + // Token bucket configuration + this.bucketCapacity = { + anonymous: 100, + authenticated: 300, + enterprise: 1000 + }; + + this.refillRate = { + anonymous: 100 / 60, // 100 tokens per minute + authenticated: 300 / 60, // 300 tokens per minute + enterprise: 1000 / 60 // 1000 tokens per minute + }; + + // IP blacklist cache + this.ipBlacklist = new Set(); + this.torExitNodes = new Set(); + + // Webhook whitelist + this.webhookWhitelist = new Set([ + 'stripe.com', + 'api.stripe.com', + 'hooks.stripe.com', + 'webhooks.stripe.com', + 'paypal.com', + 'api.paypal.com', + 'github.com', + 'api.github.com' + ]); + + // Initialize + this.initialize(); + } + + /** + * Initialize the rate limiter + */ + async initialize() { + try { + // Load IP blacklist from Redis + const blacklist = await this.redis.smembers('rate_limit:ip_blacklist'); + this.ipBlacklist = new Set(blacklist); + + // Load Tor exit nodes from Redis + const torNodes = await this.redis.smembers('rate_limit:tor_exit_nodes'); + this.torExitNodes = new Set(torNodes); + + logger.info('[RateLimitService] Initialized', { + blacklistSize: this.ipBlacklist.size, + torNodesSize: this.torExitNodes.size + }); + } catch (error) { + logger.error('[RateLimitService] Initialization failed', { + error: error.message + }); + } + } + + /** + * Check if an IP is blacklisted + * @param {string} ip - IP address + * @returns {boolean} + */ + isIPBlacklisted(ip) { + return this.ipBlacklist.has(ip); + } + + /** + * Check if an IP is a Tor exit node + * @param {string} ip - IP address + * @returns {boolean} + */ + isTorExitNode(ip) { + return this.torExitNodes.has(ip); + } + + /** + * Check if a request is from a whitelisted webhook + * @param {string} hostname - Request hostname + * @returns {boolean} + */ + isWebhookWhitelisted(hostname) { + return this.webhookWhitelist.has(hostname) || + Array.from(this.webhookWhitelist).some(whitelisted => + hostname.endsWith(whitelisted) + ); + } + + /** + * Get rate limits for a tenant + * @param {string} tenantId - Tenant ID + * @param {string} apiKey - API key (optional) + * @returns {object} Rate limits + */ + async getTenantRateLimits(tenantId, apiKey = null) { + try { + // Check if tenant has custom rate limits (enterprise) + const customLimits = await this.redis.hgetall(`rate_limit:tenant:${tenantId}`); + + if (customLimits && Object.keys(customLimits).length > 0) { + return { + requestsPerMinute: parseInt(customLimits.requestsPerMinute) || this.defaultLimits.enterprise.requestsPerMinute, + requestsPerHour: parseInt(customLimits.requestsPerHour) || this.defaultLimits.enterprise.requestsPerHour, + loginAttemptsPerMinute: parseInt(customLimits.loginAttemptsPerMinute) || this.defaultLimits.enterprise.loginAttemptsPerMinute, + loginAttemptsPerHour: parseInt(customLimits.loginAttemptsPerHour) || this.defaultLimits.enterprise.loginAttemptsPerHour, + isEnterprise: true + }; + } + + // Check if authenticated + if (apiKey) { + return { + ...this.defaultLimits.authenticated, + isEnterprise: false + }; + } + + // Default to anonymous limits + return { + ...this.defaultLimits.anonymous, + isEnterprise: false + }; + } catch (error) { + logger.error('[RateLimitService] Failed to get tenant rate limits', { + tenantId, + error: error.message + }); + return this.defaultLimits.anonymous; + } + } + + /** + * Update rate limits for a tenant (enterprise) + * @param {string} tenantId - Tenant ID + * @param {object} limits - New rate limits + * @returns {boolean} + */ + async updateTenantRateLimits(tenantId, limits) { + try { + await this.redis.hset(`rate_limit:tenant:${tenantId}`, { + requestsPerMinute: limits.requestsPerMinute, + requestsPerHour: limits.requestsPerHour, + loginAttemptsPerMinute: limits.loginAttemptsPerMinute, + loginAttemptsPerHour: limits.loginAttemptsPerHour, + updatedAt: new Date().toISOString() + }); + + logger.info('[RateLimitService] Updated tenant rate limits', { + tenantId, + limits + }); + + return true; + } catch (error) { + logger.error('[RateLimitService] Failed to update tenant rate limits', { + tenantId, + error: error.message + }); + return false; + } + } + + /** + * Check rate limit using token bucket algorithm + * @param {string} key - Rate limit key (e.g., "ip:1.2.3.4" or "tenant:123") + * @param {object} limits - Rate limits + * @param {number} tokensRequired - Tokens required for this request + * @returns {object} Rate limit check result + */ + async checkRateLimit(key, limits, tokensRequired = 1) { + try { + const now = Date.now(); + const bucketKey = `rate_limit:bucket:${key}`; + + // Get current bucket state + const bucketData = await this.redis.hgetall(bucketKey); + + let tokens; + let lastRefill; + + if (Object.keys(bucketData).length > 0) { + tokens = parseFloat(bucketData.tokens); + lastRefill = parseInt(bucketData.lastRefill); + } else { + // Initialize bucket + tokens = limits.requestsPerMinute; + lastRefill = now; + } + + // Calculate time elapsed since last refill + const timeElapsed = (now - lastRefill) / 1000; // in seconds + + // Refill tokens based on elapsed time + const refillRate = limits.requestsPerMinute / 60; // tokens per second + const tokensToAdd = Math.min(timeElapsed * refillRate, limits.requestsPerMinute); + tokens = Math.min(tokens + tokensToAdd, limits.requestsPerMinute); + + // Check if we have enough tokens + if (tokens >= tokensRequired) { + // Consume tokens + tokens -= tokensRequired; + + // Update bucket state + await this.redis.hset(bucketKey, { + tokens: tokens.toString(), + lastRefill: now.toString() + }); + + // Set expiration (2x the refill period to allow for burst capacity) + await this.redis.expire(bucketKey, 120); + + return { + allowed: true, + remaining: Math.floor(tokens), + reset: new Date(now + 60000).toISOString(), + limit: limits.requestsPerMinute + }; + } else { + // Rate limit exceeded + const retryAfter = Math.ceil((tokensRequired - tokens) / refillRate); + + return { + allowed: false, + remaining: 0, + reset: new Date(now + retryAfter * 1000).toISOString(), + limit: limits.requestsPerMinute, + retryAfter: Math.max(retryAfter, 1) + }; + } + } catch (error) { + logger.error('[RateLimitService] Rate limit check failed', { + key, + error: error.message + }); + + // Fail open - allow request if Redis is down + return { + allowed: true, + remaining: limits.requestsPerMinute, + reset: new Date(Date.now() + 60000).toISOString(), + limit: limits.requestsPerMinute, + error: true + }; + } + } + + /** + * Check login attempt rate limit + * @param {string} ip - IP address + * @param {string} tenantId - Tenant ID (optional) + * @returns {object} Rate limit check result + */ + async checkLoginRateLimit(ip, tenantId = null) { + const key = tenantId ? `login:${tenantId}` : `login:${ip}`; + const limits = tenantId ? + this.defaultLimits.authenticated : + this.defaultLimits.anonymous; + + return await this.checkRateLimit(key, { + requestsPerMinute: limits.loginAttemptsPerMinute, + requestsPerHour: limits.loginAttemptsPerHour + }); + } + + /** + * Check general API rate limit + * @param {string} ip - IP address + * @param {string} tenantId - Tenant ID (optional) + * @param {string} apiKey - API key (optional) + * @returns {object} Rate limit check result + */ + async checkAPIRateLimit(ip, tenantId = null, apiKey = null) { + // Use tenant ID if authenticated, otherwise use IP + const key = tenantId ? `api:${tenantId}` : `api:${ip}`; + + const limits = await this.getTenantRateLimits(tenantId, apiKey); + + return await this.checkRateLimit(key, limits); + } + + /** + * Add IP to blacklist + * @param {string} ip - IP address + * @param {string} reason - Reason for blacklisting + * @param {number} ttl - Time to live in seconds (default: 1 hour) + */ + async blacklistIP(ip, reason = 'manual', ttl = 3600) { + try { + await this.redis.sadd('rate_limit:ip_blacklist', ip); + await this.redis.expire('rate_limit:ip_blacklist', ttl); + this.ipBlacklist.add(ip); + + logger.warn('[RateLimitService] IP blacklisted', { + ip, + reason, + ttl + }); + + // Log to audit + await this.redis.lpush('rate_limit:audit', JSON.stringify({ + action: 'blacklist', + ip, + reason, + timestamp: new Date().toISOString() + })); + } catch (error) { + logger.error('[RateLimitService] Failed to blacklist IP', { + ip, + error: error.message + }); + } + } + + /** + * Remove IP from blacklist + * @param {string} ip - IP address + */ + async unblacklistIP(ip) { + try { + await this.redis.srem('rate_limit:ip_blacklist', ip); + this.ipBlacklist.delete(ip); + + logger.info('[RateLimitService] IP unblacklisted', { ip }); + } catch (error) { + logger.error('[RateLimitService] Failed to unblacklist IP', { + ip, + error: error.message + }); + } + } + + /** + * Add Tor exit node to list + * @param {string} ip - IP address + */ + async addTorExitNode(ip) { + try { + await this.redis.sadd('rate_limit:tor_exit_nodes', ip); + this.torExitNodes.add(ip); + + logger.info('[RateLimitService] Tor exit node added', { ip }); + } catch (error) { + logger.error('[RateLimitService] Failed to add Tor exit node', { + ip, + error: error.message + }); + } + } + + /** + * Remove Tor exit node from list + * @param {string} ip - IP address + */ + async removeTorExitNode(ip) { + try { + await this.redis.srem('rate_limit:tor_exit_nodes', ip); + this.torExitNodes.delete(ip); + + logger.info('[RateLimitService] Tor exit node removed', { ip }); + } catch (error) { + logger.error('[RateLimitService] Failed to remove Tor exit node', { + ip, + error: error.message + }); + } + } + + /** + * Get rate limit statistics + * @returns {object} Statistics + */ + async getStatistics() { + try { + const totalKeys = await this.redis.dbsize(); + const rateLimitKeys = await this.redis.keys('rate_limit:*'); + + return { + totalRedisKeys: totalKeys, + rateLimitKeys: rateLimitKeys.length, + blacklistedIPs: this.ipBlacklist.size, + torExitNodes: this.torExitNodes.size, + webhookWhitelisted: this.webhookWhitelist.size + }; + } catch (error) { + logger.error('[RateLimitService] Failed to get statistics', { + error: error.message + }); + return null; + } + } + + /** + * Reset rate limit for a key + * @param {string} key - Rate limit key + */ + async resetRateLimit(key) { + try { + await this.redis.del(`rate_limit:bucket:${key}`); + logger.info('[RateLimitService] Rate limit reset', { key }); + } catch (error) { + logger.error('[RateLimitService] Failed to reset rate limit', { + key, + error: error.message + }); + } + } + + /** + * Cleanup expired rate limit entries + */ + async cleanup() { + try { + // Redis automatically expires keys, but we can force cleanup if needed + const keys = await this.redis.keys('rate_limit:bucket:*'); + let cleaned = 0; + + for (const key of keys) { + const ttl = await this.redis.ttl(key); + if (ttl === -1) { // No expiration set + await this.redis.expire(key, 120); + cleaned++; + } + } + + logger.info('[RateLimitService] Cleanup completed', { cleaned }); + } catch (error) { + logger.error('[RateLimitService] Cleanup failed', { + error: error.message + }); + } + } + + /** + * Close Redis connection + */ + async close() { + await this.redis.quit(); + logger.info('[RateLimitService] Redis connection closed'); + } +} + +module.exports = RateLimitService; diff --git a/workers/piiScrubbingWorker.js b/workers/piiScrubbingWorker.js new file mode 100644 index 0000000..c0121e4 --- /dev/null +++ b/workers/piiScrubbingWorker.js @@ -0,0 +1,148 @@ +/** + * PII Scrubbing Worker + * + * Background worker that automatically scrubs PII for inactive users + * based on retention policies (default: 3 years of inactivity). + * + * Runs as a cron job to ensure compliance with GDPR/CCPA requirements. + */ + +const PIIScrubbingService = require('../services/piiScrubbingService'); +const logger = require('../utils/logger'); +const Database = require('better-sqlite3'); + +class PIIScrubbingWorker { + constructor({ databasePath, redisClient, webhookService, auditLogService } = {}) { + this.databasePath = databasePath || process.env.DATABASE_FILENAME || './data/substream.db'; + this.redisClient = redisClient; + this.webhookService = webhookService; + this.auditLogService = auditLogService; + this.database = null; + this.piiService = null; + } + + /** + * Initialize the worker + */ + initialize() { + try { + this.database = new Database(this.databasePath); + + this.piiService = new PIIScrubbingService({ + database: { db: this.database }, + redisClient: this.redisClient, + webhookService: this.webhookService, + auditLogService: this.auditLogService + }); + + logger.info('[PIIScrubbingWorker] Worker initialized', { + databasePath: this.databasePath + }); + } catch (error) { + logger.error('[PIIScrubbingWorker] Initialization failed', { + error: error.message + }); + throw error; + } + } + + /** + * Run the scrubbing job + * @param {object} options - Job options + * @param {number} options.years - Years of inactivity threshold + * @param {boolean} options.dryRun - If true, only count without scrubbing + * @returns {object} Job result + */ + async run(options = {}) { + const { years = 3, dryRun = false } = options; + + if (!this.piiService) { + this.initialize(); + } + + const jobId = require('crypto').randomUUID(); + const startTime = Date.now(); + + logger.info('[PIIScrubbingWorker] Starting scrubbing job', { + jobId, + years, + dryRun + }); + + try { + let result; + + if (dryRun) { + // Dry run - just count inactive users + const inactiveUsers = this.piiService.findInactiveUsers(years); + + result = { + jobId, + dryRun: true, + years, + totalUsers: inactiveUsers.length, + duration: Date.now() - startTime, + message: 'Dry run completed. No data was scrubbed.' + }; + + logger.info('[PIIScrubbingWorker] Dry run completed', result); + } else { + // Actual scrubbing + result = await this.piiService.scrubInactiveUsers(years); + result.jobId = jobId; + result.years = years; + result.duration = Date.now() - startTime; + + logger.info('[PIIScrubbingWorker] Scrubbing job completed', result); + } + + return result; + } catch (error) { + const duration = Date.now() - startTime; + logger.error('[PIIScrubbingWorker] Scrubbing job failed', { + jobId, + years, + duration, + error: error.message, + stack: error.stack + }); + + throw error; + } + } + + /** + * Cleanup resources + */ + cleanup() { + if (this.database) { + this.database.close(); + logger.info('[PIIScrubbingWorker] Database connection closed'); + } + } +} + +// CLI execution +if (require.main === module) { + const args = process.argv.slice(2); + const options = { + years: parseInt(args[0]) || 3, + dryRun: args.includes('--dry-run') + }; + + const worker = new PIIScrubbingWorker(); + + worker.run(options) + .then((result) => { + console.log('Job completed successfully:', JSON.stringify(result, null, 2)); + worker.cleanup(); + process.exit(0); + }) + .catch((error) => { + console.error('Job failed:', error.message); + worker.cleanup(); + process.exit(1); + }); +} + +module.exports = PIIScrubbingWorker;