S3 MinIO Storage

S3 & MinIO Storage

Configure ChronoTrace to use cloud storage (AWS S3) or self-hosted object storage (MinIO) for scalable, reliable trace storage in production environments.

---

🌐 Storage Options Overview

Storage Type	Best For	Pros	Cons
AWS S3	Production, Multi-region	Highly reliable, Managed service, Global CDN	Cost, Vendor lock-in
MinIO	Self-hosted, Cost control	Self-hosted, S3-compatible, Cost effective	Maintenance overhead
Local	Development, Testing	Simple setup, No external deps	Not scalable, Single point of failure

---

🚀 AWS S3 Configuration

Basic S3 Setup

# .env configuration for S3
CHRONOTRACE_STORAGE=s3
CHRONOTRACE_S3_BUCKET=my-app-chronotrace
CHRONOTRACE_S3_REGION=us-east-1
CHRONOTRACE_S3_PREFIX=traces

# AWS credentials
AWS_ACCESS_KEY_ID=AKIA...
AWS_SECRET_ACCESS_KEY=...

Advanced S3 Configuration

// config/chronotrace.php
's3' => [
    'bucket' => env('CHRONOTRACE_S3_BUCKET', 'chronotrace'),
    'region' => env('CHRONOTRACE_S3_REGION', 'us-east-1'),
    'path_prefix' => env('CHRONOTRACE_S3_PREFIX', 'traces'),
    
    // Performance optimizations
    'use_path_style_endpoint' => false,
    'multipart_threshold' => 64 * 1024 * 1024, // 64MB
    'multipart_chunksize' => 16 * 1024 * 1024, // 16MB
    
    // Storage class optimization
    'storage_class' => env('CHRONOTRACE_S3_STORAGE_CLASS', 'STANDARD_IA'),
    
    // Server-side encryption
    'server_side_encryption' => 'AES256',
    'sse_kms_key_id' => env('CHRONOTRACE_S3_KMS_KEY'),
    
    // Lifecycle management
    'lifecycle_enabled' => true,
    'transition_to_ia_days' => 30,
    'transition_to_glacier_days' => 90,
    'expiration_days' => 365,
    
    // Cross-region replication
    'replication' => [
        'enabled' => env('CHRONOTRACE_S3_REPLICATION', false),
        'destination_bucket' => env('CHRONOTRACE_S3_BACKUP_BUCKET'),
        'destination_region' => env('CHRONOTRACE_S3_BACKUP_REGION'),
    ],
],

S3 Bucket Creation Script

#!/bin/bash
# create-chronotrace-s3-bucket.sh

BUCKET_NAME="my-app-chronotrace"
REGION="us-east-1"
ENVIRONMENT="production"

echo "Creating S3 bucket for ChronoTrace..."

# Create bucket
aws s3 mb s3://${BUCKET_NAME} --region ${REGION}

# Enable versioning
aws s3api put-bucket-versioning \
    --bucket ${BUCKET_NAME} \
    --versioning-configuration Status=Enabled

# Configure lifecycle policy
cat > lifecycle-policy.json << EOF
{
    "Rules": [
        {
            "ID": "ChronoTraceLifecycle",
            "Status": "Enabled",
            "Filter": {"Prefix": "traces/"},
            "Transitions": [
                {
                    "Days": 30,
                    "StorageClass": "STANDARD_IA"
                },
                {
                    "Days": 90,
                    "StorageClass": "GLACIER"
                }
            ],
            "Expiration": {
                "Days": 365
            }
        }
    ]
}
EOF

aws s3api put-bucket-lifecycle-configuration \
    --bucket ${BUCKET_NAME} \
    --lifecycle-configuration file://lifecycle-policy.json

# Configure bucket policy
cat > bucket-policy.json << EOF
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "ChronoTraceAccess",
            "Effect": "Allow",
            "Principal": {
                "AWS": "arn:aws:iam::ACCOUNT-ID:user/chronotrace-user"
            },
            "Action": [
                "s3:GetObject",
                "s3:PutObject",
                "s3:DeleteObject",
                "s3:ListBucket"
            ],
            "Resource": [
                "arn:aws:s3:::${BUCKET_NAME}",
                "arn:aws:s3:::${BUCKET_NAME}/*"
            ]
        }
    ]
}
EOF

aws s3api put-bucket-policy \
    --bucket ${BUCKET_NAME} \
    --policy file://bucket-policy.json

echo "S3 bucket ${BUCKET_NAME} created successfully!"

IAM Policy for ChronoTrace

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "ChronoTraceS3Access",
            "Effect": "Allow",
            "Action": [
                "s3:GetObject",
                "s3:PutObject",
                "s3:DeleteObject",
                "s3:ListBucket",
                "s3:GetBucketLocation"
            ],
            "Resource": [
                "arn:aws:s3:::my-app-chronotrace",
                "arn:aws:s3:::my-app-chronotrace/*"
            ]
        },
        {
            "Sid": "KMSAccess",
            "Effect": "Allow",
            "Action": [
                "kms:Decrypt",
                "kms:Encrypt",
                "kms:GenerateDataKey"
            ],
            "Resource": "arn:aws:kms:us-east-1:ACCOUNT-ID:key/KEY-ID"
        }
    ]
}

---

🏠 MinIO Configuration

MinIO Server Setup

# Install MinIO server
wget https://dl.min.io/server/minio/release/linux-amd64/minio
chmod +x minio
sudo mv minio /usr/local/bin/

# Create MinIO user and directories
sudo useradd -r minio-user -s /sbin/nologin
sudo mkdir -p /opt/minio/data
sudo chown minio-user:minio-user /opt/minio/data

# Create systemd service
sudo tee /etc/systemd/system/minio.service << EOF
[Unit]
Description=MinIO
Documentation=https://docs.min.io
Wants=network-online.target
After=network-online.target
AssertFileIsExecutable=/usr/local/bin/minio

[Service]
WorkingDirectory=/opt/minio
User=minio-user
Group=minio-user
EnvironmentFile=/etc/default/minio
ExecStartPre=/bin/bash -c "if [ -z \"\${MINIO_VOLUMES}\" ]; then echo \"Variable MINIO_VOLUMES not set in /etc/default/minio\"; exit 1; fi"
ExecStart=/usr/local/bin/minio server \$MINIO_OPTS \$MINIO_VOLUMES
Restart=always
LimitNOFILE=65536
TasksMax=infinity
TimeoutStopSec=infinity
SendSIGKILL=no

[Install]
WantedBy=multi-user.target
EOF

# Configure MinIO
sudo tee /etc/default/minio << EOF
MINIO_ROOT_USER=minioadmin
MINIO_ROOT_PASSWORD=minioadmin123
MINIO_VOLUMES="/opt/minio/data"
MINIO_OPTS="--console-address :9090"
EOF

# Start MinIO
sudo systemctl enable minio
sudo systemctl start minio

ChronoTrace MinIO Configuration

# .env configuration for MinIO
CHRONOTRACE_STORAGE=s3
CHRONOTRACE_S3_BUCKET=chronotrace
CHRONOTRACE_S3_REGION=us-east-1
CHRONOTRACE_S3_ENDPOINT=https://minio.yourserver.com
CHRONOTRACE_S3_USE_PATH_STYLE_ENDPOINT=true

# MinIO credentials
AWS_ACCESS_KEY_ID=your-minio-access-key
AWS_SECRET_ACCESS_KEY=your-minio-secret-key

MinIO Client Setup

# Install MinIO client
wget https://dl.min.io/client/mc/release/linux-amd64/mc
chmod +x mc
sudo mv mc /usr/local/bin/

# Configure MinIO client
mc alias set local http://localhost:9000 minioadmin minioadmin123

# Create bucket for ChronoTrace
mc mb local/chronotrace

# Set bucket policy
mc policy set public local/chronotrace

# Enable versioning
mc version enable local/chronotrace

# Configure lifecycle management
cat > lifecycle.json << EOF
{
    "Rules": [
        {
            "ID": "ChronoTraceCleanup",
            "Status": "Enabled",
            "Filter": {"Prefix": "traces/"},
            "Expiration": {
                "Days": 30
            }
        }
    ]
}
EOF

mc ilm import local/chronotrace < lifecycle.json

---

🔧 Advanced Storage Features

Multi-Region Setup

S3 Cross-Region Replication

// config/chronotrace.php
'storage_replication' => [
    'enabled' => env('CHRONOTRACE_REPLICATION_ENABLED', false),
    'primary_region' => env('CHRONOTRACE_PRIMARY_REGION', 'us-east-1'),
    'backup_regions' => [
        'us-west-2',
        'eu-west-1',
    ],
    'replication_strategy' => 'async', // async, sync
],

MinIO Distributed Setup

# Multi-node MinIO setup
# Node 1
minio server http://node{1...4}/opt/minio/data{1...4}

# Node 2-4 (same command on each node)
minio server http://node{1...4}/opt/minio/data{1...4}

# Load balancer configuration (nginx)
upstream minio {
    server node1:9000;
    server node2:9000;
    server node3:9000;
    server node4:9000;
}

server {
    listen 443 ssl;
    server_name minio.yourserver.com;
    
    location / {
        proxy_pass http://minio;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}

Intelligent Storage Tiering

// Implement intelligent storage tiering
class ChronoTraceStorageTiering
{
    public function __construct(
        private S3Client $s3Client,
        private string $bucket
    ) {}
    
    public function implementTiering()
    {
        // Move old traces to cheaper storage
        $this->moveToInfrequentAccess(30); // After 30 days
        $this->moveToGlacier(90);          // After 90 days
        $this->moveToDeepArchive(365);     // After 1 year
    }
    
    private function moveToInfrequentAccess($days)
    {
        $cutoffDate = now()->subDays($days);
        
        $objects = $this->s3Client->listObjects([
            'Bucket' => $this->bucket,
            'Prefix' => 'traces/',
        ]);
        
        foreach ($objects['Contents'] ?? [] as $object) {
            $lastModified = Carbon::parse($object['LastModified']);
            
            if ($lastModified->lt($cutoffDate) && $object['StorageClass'] !== 'STANDARD_IA') {
                $this->s3Client->copyObject([
                    'Bucket' => $this->bucket,
                    'Key' => $object['Key'],
                    'CopySource' => $this->bucket . '/' . $object['Key'],
                    'StorageClass' => 'STANDARD_IA',
                    'MetadataDirective' => 'REPLACE',
                ]);
                
                Log::info('Moved trace to Standard-IA', ['key' => $object['Key']]);
            }
        }
    }
}

Compression and Optimization

// Advanced compression for storage optimization
class ChronoTraceCompression
{
    public function compressTrace($traceData)
    {
        // Use different compression based on data type
        $compressed = [];
        
        foreach ($traceData as $section => $data) {
            switch ($section) {
                case 'database_events':
                    $compressed[$section] = $this->compressDatabase($data);
                    break;
                case 'http_events':
                    $compressed[$section] = $this->compressHttp($data);
                    break;
                case 'request_data':
                    $compressed[$section] = $this->compressRequest($data);
                    break;
                default:
                    $compressed[$section] = gzcompress(json_encode($data), 9);
            }
        }
        
        return $compressed;
    }
    
    private function compressDatabase($events)
    {
        // Remove redundant information for similar queries
        $compressed = [];
        $queryTemplates = [];
        
        foreach ($events as $event) {
            $template = $this->extractQueryTemplate($event['sql']);
            
            if (!isset($queryTemplates[$template])) {
                $queryTemplates[$template] = count($queryTemplates);
            }
            
            $compressed[] = [
                'template_id' => $queryTemplates[$template],
                'bindings' => $event['bindings'],
                'duration' => $event['duration'],
                'timestamp' => $event['timestamp'],
            ];
        }
        
        return [
            'templates' => array_flip($queryTemplates),
            'events' => $compressed,
        ];
    }
}

---

📊 Storage Monitoring

S3 Cost Monitoring

// Monitor S3 costs and usage
class S3CostMonitor
{
    public function getDailyCosts()
    {
        $costs = [];
        
        // Storage costs
        $storageSize = $this->getStorageSize();
        $costs['storage'] = $this->calculateStorageCost($storageSize);
        
        // Request costs
        $requests = $this->getRequestCount();
        $costs['requests'] = $this->calculateRequestCost($requests);
        
        // Data transfer costs
        $transfer = $this->getDataTransfer();
        $costs['transfer'] = $this->calculateTransferCost($transfer);
        
        return $costs;
    }
    
    private function calculateStorageCost($sizeGB)
    {
        // S3 Standard pricing (example rates)
        $rates = [
            'standard' => 0.023,      // per GB/month
            'standard_ia' => 0.0125,  // per GB/month
            'glacier' => 0.004,       // per GB/month
        ];
        
        $monthlyCost = 0;
        foreach ($this->getStorageByClass() as $class => $size) {
            $monthlyCost += $size * ($rates[$class] ?? $rates['standard']);
        }
        
        return $monthlyCost / 30; // Daily cost
    }
}

Storage Health Monitoring

# Monitor storage health and performance
php artisan chronotrace:storage-health

# Expected output:
┌─ STORAGE HEALTH REPORT ─────────────────────────────────────┐
│ Storage Type: AWS S3                                        │
│ Bucket: production-chronotrace                              │
│ Region: us-east-1                                           │
│                                                             │
│ 📊 Health Metrics:                                         │
│   Availability: 99.99%                                     │
│   Error Rate: 0.01%                                        │
│   Average Response Time: 145ms                             │
│   Last Backup: 2024-08-06 02:00 UTC                       │
│                                                             │
│ 💾 Storage Breakdown:                                      │
│   Standard: 1.2GB (456 traces)                            │
│   Standard-IA: 2.8GB (1,234 traces)                       │
│   Glacier: 5.6GB (3,456 traces)                           │
│                                                             │
│ 💰 Cost Analysis:                                          │
│   Daily Storage: $0.45                                     │
│   Daily Requests: $0.12                                    │
│   Monthly Estimate: $17.10                                 │
│                                                             │
│ ⚠️  Recommendations:                                       │
│   - Consider lifecycle policy optimization                  │
│   - Review retention policies                              │
└─────────────────────────────────────────────────────────────┘

---

🔒 Security Best Practices

Encryption at Rest

S3 Server-Side Encryption

// Configure encryption for S3
's3' => [
    'server_side_encryption' => 'aws:kms',
    'sse_kms_key_id' => env('CHRONOTRACE_S3_KMS_KEY'),
    'bucket_key_enabled' => true, // Reduce KMS costs
],

MinIO Encryption

# Enable MinIO encryption
export MINIO_KMS_SECRET_KEY="my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="

# Start MinIO with encryption
minio server --address :9000 /opt/minio/data

Access Control

// Implement fine-grained access control
class ChronoTraceStorageAccess
{
    public function generatePresignedUrl($traceId, $user)
    {
        // Verify user can access this trace
        if (!$this->canUserAccessTrace($user, $traceId)) {
            throw new UnauthorizedAccessException();
        }
        
        // Generate short-lived presigned URL
        return $this->s3Client->createPresignedRequest(
            $this->s3Client->getCommand('GetObject', [
                'Bucket' => $this->bucket,
                'Key' => "traces/{$traceId}.json",
            ]),
            '+15 minutes' // Short expiration
        )->getUri();
    }
    
    private function canUserAccessTrace($user, $traceId)
    {
        // Implement your access control logic
        $trace = ChronoTrace::find($traceId);
        
        return $user->hasRole('admin') || 
               $trace->user_id === $user->id ||
               $user->hasPermission('view_all_traces');
    }
}

---

🚀 Performance Optimization

Connection Pooling

// Optimize S3 connections
'filesystems' => [
    'disks' => [
        's3_chronotrace' => [
            'driver' => 's3',
            'key' => env('AWS_ACCESS_KEY_ID'),
            'secret' => env('AWS_SECRET_ACCESS_KEY'),
            'region' => env('AWS_DEFAULT_REGION'),
            'bucket' => env('CHRONOTRACE_S3_BUCKET'),
            'options' => [
                'http' => [
                    'connect_timeout' => 5,
                    'timeout' => 30,
                    'pool_size' => 50, // Connection pool
                ],
                'use_accelerate_endpoint' => env('CHRONOTRACE_S3_ACCELERATE', false),
                'use_dual_stack_endpoint' => true,
            ],
        ],
    ],
],

Async Operations

// Implement async storage operations
class AsyncStorageManager
{
    public function storeTraceAsync($traceData)
    {
        // Use Laravel's queue system for async storage
        StoreTraceJob::dispatch($traceData)
            ->onQueue('storage')
            ->delay(now()->addSeconds(1));
    }
    
    public function batchStoreTraces($traces)
    {
        // Batch multiple traces for efficiency
        $chunks = array_chunk($traces, 10);
        
        foreach ($chunks as $chunk) {
            BatchStoreTracesJob::dispatch($chunk)
                ->onQueue('storage');
        }
    }
}

class StoreTraceJob implements ShouldQueue
{
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
    
    public function handle()
    {
        try {
            // Compress trace data
            $compressed = $this->compressTrace($this->traceData);
            
            // Store to S3/MinIO
            Storage::disk('s3_chronotrace')->put(
                "traces/{$this->traceId}.json.gz",
                $compressed
            );
            
            Log::info('Trace stored successfully', ['trace_id' => $this->traceId]);
            
        } catch (Exception $e) {
            Log::error('Failed to store trace', [
                'trace_id' => $this->traceId,
                'error' => $e->getMessage()
            ]);
            
            throw $e; // Re-throw to trigger retry
        }
    }
}

---

📋 Storage Migration

Migrating from Local to S3

# Migration script: local-to-s3-migration.php
#!/usr/bin/env php
<?php

require 'vendor/autoload.php';

class ChronoTraceMigration
{
    public function migrateToS3()
    {
        $localPath = storage_path('chronotrace/traces');
        $s3Disk = Storage::disk('s3_chronotrace');
        
        $files = File::glob($localPath . '/*.json');
        $totalFiles = count($files);
        
        echo "Migrating {$totalFiles} traces to S3...\n";
        
        foreach ($files as $index => $file) {
            $filename = basename($file);
            $content = file_get_contents($file);
            
            // Compress before uploading
            $compressed = gzcompress($content, 6);
            
            // Upload to S3
            $s3Disk->put("traces/{$filename}.gz", $compressed);
            
            // Verify upload
            if ($s3Disk->exists("traces/{$filename}.gz")) {
                unlink($file); // Remove local file
                echo "✓ Migrated {$filename} (" . ($index + 1) . "/{$totalFiles})\n";
            } else {
                echo "✗ Failed to migrate {$filename}\n";
            }
        }
        
        echo "Migration completed!\n";
    }
}

$migration = new ChronoTraceMigration();
$migration->migrateToS3();

---

📚 Related Documentation

• Configuration - Storage configuration options
• Production Monitoring - Storage monitoring in production
• Security - Storage security best practices
• Troubleshooting - Storage-related troubleshooting

---

Storage configured! Your ChronoTrace traces are now stored reliably in the cloud with optimized costs and security.