Custom Storage

Custom Storage

Complete guide for configuring custom storage backends for Laravel ChronoTrace, including AWS S3, MinIO, and custom solutions.

AWS S3 Storage Setup

1. Create and Configure S3 Bucket

Create S3 Bucket:

# Using AWS CLI
aws s3 mb s3://my-chronotrace-bucket --region us-east-1

# Enable versioning
aws s3api put-bucket-versioning \
    --bucket my-chronotrace-bucket \
    --versioning-configuration Status=Enabled

# Configure lifecycle policy for cost optimization
cat > lifecycle-policy.json << EOF
{
    "Rules": [
        {
            "ID": "ChronoTraceLifecycle",
            "Status": "Enabled",
            "Filter": {"Prefix": "traces/"},
            "Transitions": [
                {
                    "Days": 30,
                    "StorageClass": "STANDARD_IA"
                },
                {
                    "Days": 90,
                    "StorageClass": "GLACIER"
                }
            ],
            "Expiration": {
                "Days": 365
            }
        }
    ]
}
EOF

aws s3api put-bucket-lifecycle-configuration \
    --bucket my-chronotrace-bucket \
    --lifecycle-configuration file://lifecycle-policy.json

2. IAM Configuration

Create IAM User and Policy:

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "ChronoTraceS3Access",
            "Effect": "Allow",
            "Action": [
                "s3:GetObject",
                "s3:PutObject",
                "s3:DeleteObject",
                "s3:ListBucket",
                "s3:GetBucketLocation"
            ],
            "Resource": [
                "arn:aws:s3:::my-chronotrace-bucket",
                "arn:aws:s3:::my-chronotrace-bucket/*"
            ]
        }
    ]
}

3. Laravel Configuration

// config/chronotrace.php
'storage' => 's3',
's3' => [
    'bucket' => env('CHRONOTRACE_S3_BUCKET', 'my-chronotrace-bucket'),
    'region' => env('CHRONOTRACE_S3_REGION', 'us-east-1'),
    'path_prefix' => env('CHRONOTRACE_S3_PREFIX', 'traces'),
    'storage_class' => 'STANDARD_IA',
    'server_side_encryption' => 'AES256',
],

Environment Variables:

# .env
CHRONOTRACE_STORAGE=s3
CHRONOTRACE_S3_BUCKET=my-chronotrace-bucket
CHRONOTRACE_S3_REGION=us-east-1
CHRONOTRACE_S3_PREFIX=production/traces
AWS_ACCESS_KEY_ID=AKIA...
AWS_SECRET_ACCESS_KEY=...

---

MinIO Storage Setup

1. Install MinIO Server

# Download and install MinIO
wget https://dl.min.io/server/minio/release/linux-amd64/minio
chmod +x minio
sudo mv minio /usr/local/bin/

# Create MinIO user and directories
sudo useradd -r minio-user -s /sbin/nologin
sudo mkdir -p /opt/minio/data
sudo chown minio-user:minio-user /opt/minio/data

2. Configure MinIO Service

# Create systemd service
sudo tee /etc/systemd/system/minio.service << EOF
[Unit]
Description=MinIO
Documentation=https://docs.min.io
Wants=network-online.target
After=network-online.target

[Service]
WorkingDirectory=/opt/minio
User=minio-user
Group=minio-user
EnvironmentFile=/etc/default/minio
ExecStart=/usr/local/bin/minio server \\$MINIO_OPTS \\$MINIO_VOLUMES
Restart=always
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

# Configure MinIO
sudo tee /etc/default/minio << EOF
MINIO_ROOT_USER=minioadmin
MINIO_ROOT_PASSWORD=minioadmin123
MINIO_VOLUMES="/opt/minio/data"
MINIO_OPTS="--console-address :9090"
EOF

# Start MinIO
sudo systemctl enable minio
sudo systemctl start minio

3. MinIO Client Setup

# Install MinIO client
wget https://dl.min.io/client/mc/release/linux-amd64/mc
chmod +x mc
sudo mv mc /usr/local/bin/

# Configure alias
mc alias set myminio http://localhost:9000 minioadmin minioadmin123

# Create bucket
mc mb myminio/chronotrace

# Set policy
mc policy set download myminio/chronotrace

4. Laravel MinIO Configuration

# .env
CHRONOTRACE_STORAGE=s3
CHRONOTRACE_S3_BUCKET=chronotrace
CHRONOTRACE_S3_ENDPOINT=http://localhost:9000
CHRONOTRACE_S3_USE_PATH_STYLE_ENDPOINT=true
AWS_ACCESS_KEY_ID=minioadmin
AWS_SECRET_ACCESS_KEY=minioadmin123

---

Database Storage

1. Create Migration

php artisan make:migration create_chronotrace_traces_table

// Migration file
public function up()
{
    Schema::create('chronotrace_traces', function (Blueprint $table) {
        $table->string('id')->primary();
        $table->string('method', 10);
        $table->string('url', 2048);
        $table->string('route')->nullable();
        $table->integer('status');
        $table->integer('duration_ms');
        $table->integer('memory_mb');
        $table->integer('user_id')->nullable();
        $table->json('request_data')->nullable();
        $table->json('response_data')->nullable();
        $table->json('events');
        $table->json('metadata')->nullable();
        $table->timestamp('created_at');
        
        $table->index(['status', 'created_at']);
        $table->index(['route', 'created_at']);
        $table->index(['user_id', 'created_at']);
        $table->index(['duration_ms', 'created_at']);
    });
}

2. Database Storage Driver

// app/Storage/DatabaseStorageDriver.php
use Grazulex\LaravelChronotrace\Contracts\StorageDriver;

class DatabaseStorageDriver implements StorageDriver
{
    public function store(string $traceId, array $data): bool
    {
        try {
            DB::table('chronotrace_traces')->insert([
                'id' => $traceId,
                'method' => $data['request']['method'],
                'url' => $data['request']['url'],
                'route' => $data['request']['route'] ?? null,
                'status' => $data['response']['status'],
                'duration_ms' => $data['performance']['total_duration_ms'],
                'memory_mb' => round($data['performance']['memory_peak_mb'], 2),
                'user_id' => $data['request']['user_id'] ?? null,
                'request_data' => json_encode($data['request']),
                'response_data' => json_encode($data['response']),
                'events' => json_encode($data['events']),
                'metadata' => json_encode($data['metadata'] ?? []),
                'created_at' => now(),
            ]);
            
            return true;
        } catch (Exception $e) {
            Log::error('Failed to store trace in database', [
                'trace_id' => $traceId,
                'error' => $e->getMessage()
            ]);
            return false;
        }
    }
    
    public function retrieve(string $traceId): ?array
    {
        $record = DB::table('chronotrace_traces')->where('id', $traceId)->first();
        
        if (!$record) {
            return null;
        }
        
        return [
            'trace_id' => $record->id,
            'request' => json_decode($record->request_data, true),
            'response' => json_decode($record->response_data, true),
            'events' => json_decode($record->events, true),
            'metadata' => json_decode($record->metadata, true),
        ];
    }
    
    public function delete(string $traceId): bool
    {
        return DB::table('chronotrace_traces')->where('id', $traceId)->delete() > 0;
    }
    
    public function exists(string $traceId): bool
    {
        return DB::table('chronotrace_traces')->where('id', $traceId)->exists();
    }
    
    public function list(array $filters = []): array
    {
        $query = DB::table('chronotrace_traces');
        
        if (isset($filters['status'])) {
            $query->where('status', $filters['status']);
        }
        
        if (isset($filters['route'])) {
            $query->where('route', 'like', $filters['route']);
        }
        
        if (isset($filters['since'])) {
            $query->where('created_at', '>=', $filters['since']);
        }
        
        return $query->orderBy('created_at', 'desc')
                    ->limit($filters['limit'] ?? 100)
                    ->get()
                    ->map(fn($record) => [
                        'id' => $record->id,
                        'method' => $record->method,
                        'route' => $record->route,
                        'status' => $record->status,
                        'duration' => $record->duration_ms,
                        'created_at' => $record->created_at,
                    ])
                    ->toArray();
    }
    
    public function purge(int $olderThanDays): int
    {
        return DB::table('chronotrace_traces')
                 ->where('created_at', '<', now()->subDays($olderThanDays))
                 ->delete();
    }
}

3. Register Database Driver

// config/chronotrace.php
'storage' => 'database',

// app/Providers/AppServiceProvider.php
public function boot()
{
    $this->app->afterResolving(StorageManager::class, function (StorageManager $manager) {
        $manager->extend('database', function ($app, $config) {
            return new DatabaseStorageDriver();
        });
    });
}

---

Redis Storage

1. Redis Storage Driver

// app/Storage/RedisStorageDriver.php
use Grazulex\LaravelChronotrace\Contracts\StorageDriver;

class RedisStorageDriver implements StorageDriver
{
    private $redis;
    private $keyPrefix;
    private $ttl;
    
    public function __construct(array $config = [])
    {
        $this->redis = Redis::connection($config['connection'] ?? 'default');
        $this->keyPrefix = $config['key_prefix'] ?? 'chronotrace:traces:';
        $this->ttl = $config['ttl'] ?? 86400 * 15; // 15 days
    }
    
    public function store(string $traceId, array $data): bool
    {
        try {
            $key = $this->keyPrefix . $traceId;
            $compressed = gzcompress(json_encode($data), 6);
            
            return $this->redis->setex($key, $this->ttl, $compressed);
        } catch (Exception $e) {
            Log::error('Failed to store trace in Redis', [
                'trace_id' => $traceId,
                'error' => $e->getMessage()
            ]);
            return false;
        }
    }
    
    public function retrieve(string $traceId): ?array
    {
        try {
            $key = $this->keyPrefix . $traceId;
            $compressed = $this->redis->get($key);
            
            if (!$compressed) {
                return null;
            }
            
            $data = gzuncompress($compressed);
            return json_decode($data, true);
        } catch (Exception $e) {
            Log::error('Failed to retrieve trace from Redis', [
                'trace_id' => $traceId,
                'error' => $e->getMessage()
            ]);
            return null;
        }
    }
    
    public function delete(string $traceId): bool
    {
        $key = $this->keyPrefix . $traceId;
        return $this->redis->del($key) > 0;
    }
    
    public function exists(string $traceId): bool
    {
        $key = $this->keyPrefix . $traceId;
        return $this->redis->exists($key) > 0;
    }
    
    public function list(array $filters = []): array
    {
        $pattern = $this->keyPrefix . '*';
        $keys = $this->redis->keys($pattern);
        
        $traces = [];
        foreach (array_slice($keys, 0, $filters['limit'] ?? 100) as $key) {
            $traceId = str_replace($this->keyPrefix, '', $key);
            $ttl = $this->redis->ttl($key);
            
            $traces[] = [
                'id' => $traceId,
                'expires_in' => $ttl,
            ];
        }
        
        return $traces;
    }
    
    public function purge(int $olderThanDays): int
    {
        // Redis handles TTL automatically, so we just return 0
        return 0;
    }
}

2. Configuration

// config/chronotrace.php
'storage' => 'redis',
'redis' => [
    'connection' => 'cache',
    'key_prefix' => 'chronotrace:traces:',
    'ttl' => 86400 * 15, // 15 days
],

---

Elasticsearch Storage

1. Elasticsearch Storage Driver

// app/Storage/ElasticsearchStorageDriver.php
use Elasticsearch\ClientBuilder;
use Grazulex\LaravelChronotrace\Contracts\StorageDriver;

class ElasticsearchStorageDriver implements StorageDriver
{
    private $client;
    private $index;
    
    public function __construct(array $config = [])
    {
        $this->client = ClientBuilder::create()
            ->setHosts($config['hosts'] ?? ['localhost:9200'])
            ->build();
            
        $this->index = $config['index'] ?? 'chronotrace-traces';
        
        $this->createIndexIfNotExists();
    }
    
    private function createIndexIfNotExists()
    {
        $params = [
            'index' => $this->index,
            'body' => [
                'mappings' => [
                    'properties' => [
                        'trace_id' => ['type' => 'keyword'],
                        'method' => ['type' => 'keyword'],
                        'route' => ['type' => 'keyword'],
                        'status' => ['type' => 'integer'],
                        'duration' => ['type' => 'integer'],
                        'timestamp' => ['type' => 'date'],
                        'events' => ['type' => 'nested'],
                        'request' => ['type' => 'object'],
                        'response' => ['type' => 'object'],
                    ]
                ]
            ]
        ];
        
        if (!$this->client->indices()->exists(['index' => $this->index])) {
            $this->client->indices()->create($params);
        }
    }
    
    public function store(string $traceId, array $data): bool
    {
        try {
            $params = [
                'index' => $this->index,
                'id' => $traceId,
                'body' => array_merge($data, [
                    'trace_id' => $traceId,
                    'timestamp' => now()->toISOString(),
                ])
            ];
            
            $response = $this->client->index($params);
            return isset($response['result']) && $response['result'] === 'created';
        } catch (Exception $e) {
            Log::error('Failed to store trace in Elasticsearch', [
                'trace_id' => $traceId,
                'error' => $e->getMessage()
            ]);
            return false;
        }
    }
    
    public function retrieve(string $traceId): ?array
    {
        try {
            $params = [
                'index' => $this->index,
                'id' => $traceId
            ];
            
            $response = $this->client->get($params);
            return $response['_source'] ?? null;
        } catch (Exception $e) {
            return null;
        }
    }
    
    public function delete(string $traceId): bool
    {
        try {
            $params = [
                'index' => $this->index,
                'id' => $traceId
            ];
            
            $response = $this->client->delete($params);
            return isset($response['result']) && $response['result'] === 'deleted';
        } catch (Exception $e) {
            return false;
        }
    }
    
    public function exists(string $traceId): bool
    {
        try {
            $params = [
                'index' => $this->index,
                'id' => $traceId
            ];
            
            return $this->client->exists($params);
        } catch (Exception $e) {
            return false;
        }
    }
    
    public function list(array $filters = []): array
    {
        $params = [
            'index' => $this->index,
            'body' => [
                'query' => [
                    'bool' => [
                        'must' => []
                    ]
                ],
                'sort' => [
                    'timestamp' => ['order' => 'desc']
                ],
                'size' => $filters['limit'] ?? 100
            ]
        ];
        
        // Add filters
        if (isset($filters['status'])) {
            $params['body']['query']['bool']['must'][] = [
                'term' => ['status' => $filters['status']]
            ];
        }
        
        if (isset($filters['route'])) {
            $params['body']['query']['bool']['must'][] = [
                'wildcard' => ['route' => $filters['route']]
            ];
        }
        
        try {
            $response = $this->client->search($params);
            return array_map(function($hit) {
                return $hit['_source'];
            }, $response['hits']['hits']);
        } catch (Exception $e) {
            return [];
        }
    }
    
    public function purge(int $olderThanDays): int
    {
        $params = [
            'index' => $this->index,
            'body' => [
                'query' => [
                    'range' => [
                        'timestamp' => [
                            'lt' => now()->subDays($olderThanDays)->toISOString()
                        ]
                    ]
                ]
            ]
        ];
        
        try {
            $response = $this->client->deleteByQuery($params);
            return $response['deleted'] ?? 0;
        } catch (Exception $e) {
            return 0;
        }
    }
}

---

Multi-Backend Storage

1. Multi-Backend Driver

// app/Storage/MultiBackendStorageDriver.php
use Grazulex\LaravelChronotrace\Contracts\StorageDriver;

class MultiBackendStorageDriver implements StorageDriver
{
    private $primary;
    private $backup;
    
    public function __construct(array $config = [])
    {
        $this->primary = app($config['primary']);
        $this->backup = app($config['backup']);
    }
    
    public function store(string $traceId, array $data): bool
    {
        $primaryResult = $this->primary->store($traceId, $data);
        
        // Always try backup storage
        $backupResult = $this->backup->store($traceId, $data);
        
        return $primaryResult || $backupResult;
    }
    
    public function retrieve(string $traceId): ?array
    {
        // Try primary first
        $data = $this->primary->retrieve($traceId);
        
        if ($data === null) {
            // Fallback to backup
            $data = $this->backup->retrieve($traceId);
        }
        
        return $data;
    }
    
    public function delete(string $traceId): bool
    {
        $primaryDeleted = $this->primary->delete($traceId);
        $backupDeleted = $this->backup->delete($traceId);
        
        return $primaryDeleted || $backupDeleted;
    }
    
    public function exists(string $traceId): bool
    {
        return $this->primary->exists($traceId) || 
               $this->backup->exists($traceId);
    }
    
    public function list(array $filters = []): array
    {
        // Merge results from both backends
        $primaryTraces = $this->primary->list($filters);
        $backupTraces = $this->backup->list($filters);
        
        $allTraces = array_merge($primaryTraces, $backupTraces);
        
        // Remove duplicates and sort
        $uniqueTraces = collect($allTraces)
            ->unique('id')
            ->sortByDesc('created_at')
            ->take($filters['limit'] ?? 100)
            ->values()
            ->toArray();
            
        return $uniqueTraces;
    }
    
    public function purge(int $olderThanDays): int
    {
        $primaryPurged = $this->primary->purge($olderThanDays);
        $backupPurged = $this->backup->purge($olderThanDays);
        
        return max($primaryPurged, $backupPurged);
    }
}

2. Configuration

// config/chronotrace.php
'storage' => 'multi',
'multi' => [
    'primary' => 'redis',
    'backup' => 's3',
],

---

Storage Testing

1. Storage Driver Tests

// tests/Unit/Storage/CustomStorageDriverTest.php
class CustomStorageDriverTest extends TestCase
{
    private $driver;
    
    protected function setUp(): void
    {
        parent::setUp();
        $this->driver = new CustomStorageDriver($config);
    }
    
    public function test_can_store_and_retrieve_trace()
    {
        $traceId = 'test_trace_' . uniqid();
        $data = [
            'request' => ['method' => 'GET', 'url' => '/test'],
            'response' => ['status' => 200],
            'events' => [],
        ];
        
        // Store trace
        $stored = $this->driver->store($traceId, $data);
        $this->assertTrue($stored);
        
        // Verify exists
        $this->assertTrue($this->driver->exists($traceId));
        
        // Retrieve trace
        $retrieved = $this->driver->retrieve($traceId);
        $this->assertNotNull($retrieved);
        $this->assertEquals($data['request']['method'], $retrieved['request']['method']);
        
        // Clean up
        $this->driver->delete($traceId);
    }
    
    public function test_can_list_traces()
    {
        // Store multiple traces
        for ($i = 0; $i < 5; $i++) {
            $traceId = 'test_trace_' . $i;
            $this->driver->store($traceId, $this->generateTraceData());
        }
        
        // List traces
        $traces = $this->driver->list(['limit' => 10]);
        $this->assertGreaterThanOrEqual(5, count($traces));
        
        // Clean up
        for ($i = 0; $i < 5; $i++) {
            $this->driver->delete('test_trace_' . $i);
        }
    }
    
    public function test_can_purge_old_traces()
    {
        // This test depends on your storage implementation
        $purged = $this->driver->purge(30);
        $this->assertIsInt($purged);
    }
}

2. Performance Testing

// tests/Performance/StoragePerformanceTest.php
class StoragePerformanceTest extends TestCase
{
    public function test_storage_performance_benchmark()
    {
        $driver = app(StorageDriver::class);
        $traceData = $this->generateLargeTraceData();
        
        // Test write performance
        $writeStart = microtime(true);
        for ($i = 0; $i < 100; $i++) {
            $traceId = 'perf_test_' . $i;
            $driver->store($traceId, $traceData);
        }
        $writeTime = microtime(true) - $writeStart;
        
        // Test read performance
        $readStart = microtime(true);
        for ($i = 0; $i < 100; $i++) {
            $traceId = 'perf_test_' . $i;
            $driver->retrieve($traceId);
        }
        $readTime = microtime(true) - $readStart;
        
        // Assert performance requirements
        $this->assertLessThan(10.0, $writeTime, 'Write performance too slow');
        $this->assertLessThan(5.0, $readTime, 'Read performance too slow');
        
        // Clean up
        for ($i = 0; $i < 100; $i++) {
            $driver->delete('perf_test_' . $i);
        }
    }
}

---

Monitoring and Maintenance

1. Storage Health Monitoring

// app/Services/StorageHealthMonitor.php
class StorageHealthMonitor
{
    public function checkHealth(): array
    {
        $driver = app(StorageDriver::class);
        
        return [
            'connectivity' => $this->testConnectivity($driver),
            'write_performance' => $this->testWritePerformance($driver),
            'read_performance' => $this->testReadPerformance($driver),
            'storage_usage' => $this->getStorageUsage($driver),
        ];
    }
    
    private function testConnectivity(StorageDriver $driver): bool
    {
        try {
            $testId = 'health_check_' . time();
            $testData = ['test' => true];
            
            $stored = $driver->store($testId, $testData);
            $exists = $driver->exists($testId);
            $retrieved = $driver->retrieve($testId);
            $deleted = $driver->delete($testId);
            
            return $stored && $exists && $retrieved && $deleted;
        } catch (Exception $e) {
            return false;
        }
    }
}

2. Automated Maintenance

# Create maintenance command
php artisan make:command ChronoTrace:Maintenance

// app/Console/Commands/ChronoTraceMaintenance.php
class ChronoTraceMaintenance extends Command
{
    protected $signature = 'chronotrace:maintenance {--dry-run}';
    
    public function handle()
    {
        $this->info('Starting ChronoTrace maintenance...');
        
        // Check storage health
        $this->checkStorageHealth();
        
        // Optimize storage
        $this->optimizeStorage();
        
        // Clean up old traces
        $this->cleanupOldTraces();
        
        $this->info('Maintenance completed!');
    }
    
    private function checkStorageHealth()
    {
        $monitor = app(StorageHealthMonitor::class);
        $health = $monitor->checkHealth();
        
        if (!$health['connectivity']) {
            $this->error('Storage connectivity issue detected!');
        }
        
        $this->info("Write performance: {$health['write_performance']}ms");
        $this->info("Read performance: {$health['read_performance']}ms");
    }
}

---

📚 Related Documentation

• S3 & MinIO Storage - Detailed cloud storage setup
• Configuration - Storage configuration options
• Production Monitoring - Storage monitoring in production
• API Reference - Storage driver API reference

---

Build the perfect storage solution for your needs! Choose from multiple backends or create custom drivers to match your infrastructure requirements.