Fix Redis support in manual development setup and consolidate environment configuration

.env.example

@@ -33,4 +33,54 @@ REDIS_HOST=localhost
 REDIS_PORT=6379
 REDIS_PASSWORD=
 REDIS_DATABASE=0
-REDIS_MAX_MEMORY=512mb
+REDIS_MAX_MEMORY=512mb
+REDIS_POOL_SIZE=10
+REDIS_MIN_IDLE_CONNECTIONS=5
+REDIS_MAX_RETRIES=3
+REDIS_DIAL_TIMEOUT=5s
+REDIS_READ_TIMEOUT=3s
+REDIS_WRITE_TIMEOUT=3s
+REDIS_IDLE_TIMEOUT=5m
+
+# Queue Configuration
+QUEUE_TASK_QUEUE_NAME=voidrunner:tasks
+QUEUE_DEAD_LETTER_QUEUE_NAME=voidrunner:tasks:dead
+QUEUE_RETRY_QUEUE_NAME=voidrunner:tasks:retry
+QUEUE_DEFAULT_PRIORITY=5
+QUEUE_MAX_RETRIES=3
+QUEUE_RETRY_DELAY=30s
+QUEUE_RETRY_BACKOFF_FACTOR=2.0
+QUEUE_MAX_RETRY_DELAY=15m
+QUEUE_VISIBILITY_TIMEOUT=30m
+QUEUE_MESSAGE_TTL=24h
+QUEUE_BATCH_SIZE=10
+
+# Worker Configuration
+WORKER_POOL_SIZE=5
+WORKER_MAX_CONCURRENT_TASKS=10
+WORKER_MAX_USER_CONCURRENT_TASKS=3
+WORKER_TASK_TIMEOUT=1h
+WORKER_HEARTBEAT_INTERVAL=30s
+WORKER_SHUTDOWN_TIMEOUT=30s
+WORKER_CLEANUP_INTERVAL=5m
+WORKER_STALE_TASK_THRESHOLD=2h
+WORKER_ID_PREFIX=voidrunner-worker
+
+# Embedded Workers (enable worker pool in API server process)
+EMBEDDED_WORKERS=true
+
+# Executor Configuration
+DOCKER_ENDPOINT=unix:///var/run/docker.sock
+EXECUTOR_DEFAULT_MEMORY_LIMIT_MB=128
+EXECUTOR_DEFAULT_CPU_QUOTA=50000
+EXECUTOR_DEFAULT_PIDS_LIMIT=128
+EXECUTOR_DEFAULT_TIMEOUT_SECONDS=300
+EXECUTOR_PYTHON_IMAGE=python:3.11-alpine
+EXECUTOR_BASH_IMAGE=alpine:latest
+EXECUTOR_JAVASCRIPT_IMAGE=node:18-alpine
+EXECUTOR_GO_IMAGE=golang:1.21-alpine
+EXECUTOR_ENABLE_SECCOMP=true
+EXECUTOR_SECCOMP_PROFILE_PATH=/opt/voidrunner/seccomp-profile.json
+EXECUTOR_ENABLE_APPARMOR=false
+EXECUTOR_APPARMOR_PROFILE=voidrunner-executor
+EXECUTOR_EXECUTION_USER=1000:1000

CLAUDE.md

@@ -734,10 +734,10 @@ make prod-status       # Show production environment status
 make env-status        # Show all environment status
 make docker-clean      # Clean Docker resources
 
-# Database management (Test DB only)
-make db-start          # Start test database (Docker)
-make db-stop           # Stop test database
-make db-reset          # Reset test database (clean slate)
+# Test services management (PostgreSQL + Redis)
+make services-start    # Start test services (Docker)
+make services-stop     # Stop test services
+make services-reset    # Reset test services (clean slate)
 
 # Database migrations
 make migrate-up        # Run database migrations up
@@ -754,10 +754,10 @@ make setup             # Setup complete development environment
 ### Database Operations
 
 ```bash
-# Database management (implemented)
-make db-start          # Start test database container
-make db-stop           # Stop test database container
-make db-reset          # Reset test database to clean state
+# Test services management (implemented)
+make services-start    # Start test services containers (PostgreSQL + Redis)
+make services-stop     # Stop test services containers
+make services-reset    # Reset test services to clean state
 
 # Migration management (implemented)
 make migrate-up        # Apply all pending migrations

Dockerfile

@@ -44,8 +44,8 @@ RUN addgroup -g 1001 -S voidrunner && \
 # Set working directory
 WORKDIR /app
 
-# Copy source code for development
-COPY --chown=voidrunner:voidrunner . .
+# Copy go mod files first for better caching
+COPY --chown=voidrunner:voidrunner go.mod go.sum ./
 
 # Create logs and tmp directories, and set up Go module cache permissions
 RUN mkdir -p logs tmp && \
@@ -56,11 +56,17 @@ RUN mkdir -p logs tmp && \
 # Switch to non-root user
 USER voidrunner
 
+# Download dependencies as non-root user
+RUN go mod download
+
+# Copy source code for development
+COPY --chown=voidrunner:voidrunner . .
+
 # Expose port
 EXPOSE 8080
 
 # Health check
-HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
+HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
     CMD curl -f http://localhost:8080/health || exit 1
 
 # Development entry point (use Go directly for now)

Makefile

@@ -1,7 +1,7 @@
 # VoidRunner Makefile
 # Provides standardized commands for building, testing, and running the application
 
-.PHONY: help test test-fast test-integration test-all build run dev clean coverage coverage-check docs docs-serve lint fmt vet security deps deps-update migrate-up migrate-down migrate-reset migration docker-build docker-run clean-docs install-tools setup all pre-commit bench db-start db-stop db-reset db-status dev-up dev-down dev-logs dev-restart dev-status prod-up prod-down prod-logs prod-restart prod-status docker-clean env-status
+.PHONY: help test test-fast test-integration test-all build run dev clean coverage coverage-check docs docs-serve lint fmt vet security deps deps-update migrate-up migrate-down migrate-reset migration docker-build docker-run clean-docs install-tools setup all pre-commit bench services-start services-stop services-reset services-status dev-up dev-down dev-logs dev-restart dev-status prod-up prod-down prod-logs prod-restart prod-status docker-clean env-status
 
 # Default target
 help: ## Show this help message
@@ -32,10 +32,10 @@ test-fast: ## Run unit tests in short mode (fast, excluding integration)
 	@go test -short ./cmd/... ./internal/... ./pkg/...
 
 
-test-integration: ## Run integration tests (requires database to be running)
+test-integration: ## Run integration tests (requires services to be running)
 	@echo "Running integration tests..."
-	@echo "Note: Database must be running before running integration tests"
-	@echo "Use 'make db-start' to start the test database"
+	@echo "Note: PostgreSQL and Redis must be running before running integration tests"
+	@echo "Use 'make services-start' to start the test services"
 	@INTEGRATION_TESTS=true \
 	 TEST_DB_HOST=$${TEST_DB_HOST:-localhost} \
 	 TEST_DB_PORT=$${TEST_DB_PORT:-5432} \
@@ -127,21 +127,21 @@ migration: ## Create a new migration file (usage: make migration name=migration_
 	@touch migrations/$(shell date +%Y%m%d%H%M%S)_$(name).down.sql
 	@echo "Migration files created in migrations/"
 
-# Database management targets
-db-start: ## Start test database (Docker)
-	@echo "Starting test database..."
-	@./scripts/start-test-db.sh
+# Test services management targets
+services-start: ## Start test services (PostgreSQL + Redis)
+	@echo "Starting test services..."
+	@./scripts/start-test-services.sh
 
-db-stop: ## Stop test database (Docker)
-	@echo "Stopping test database..."
-	@./scripts/stop-test-db.sh
+services-stop: ## Stop test services (PostgreSQL + Redis)
+	@echo "Stopping test services..."
+	@./scripts/stop-test-services.sh
 
-db-reset: ## Reset test database (clean slate)
-	@echo "Resetting test database..."
-	@./scripts/reset-test-db.sh
+services-reset: ## Reset test services (clean slate)
+	@echo "Resetting test services..."
+	@./scripts/reset-test-services.sh
 
-db-status: ## Show test database status
-	@echo "Test database status:"
+services-status: ## Show test services status
+	@echo "Test services status:"
 	@if command -v docker-compose &> /dev/null; then \
 		docker-compose -f docker-compose.test.yml ps; \
 	else \
@@ -196,12 +196,12 @@ env-status: ## Show all environment status
 	@echo "=== Production Environment ==="
 	@./scripts/prod-env.sh status || echo "Production environment not running"
 	@echo ""
-	@echo "=== Test Database ==="
+	@echo "=== Test Services ==="
 	@if command -v docker-compose &> /dev/null; then \
 		if docker-compose -f docker-compose.test.yml ps | grep -q "Up"; then \
-			echo "Test database: Running"; \
+			echo "Test services: Running"; \
 		else \
-			echo "Test database: Stopped"; \
+			echo "Test services: Stopped"; \
 		fi; \
 	else \
 		echo "docker-compose not found"; \
@@ -283,9 +283,9 @@ setup: deps install-tools ## Setup development environment
 	@echo "  - .env file: created (edit with your configuration)"
 	@echo ""
 	@echo "To run integration tests:"
-	@echo "  make db-start         # Start test database"
+	@echo "  make services-start   # Start test services (PostgreSQL + Redis)"
 	@echo "  make test-integration # Run integration tests"
-	@echo "  make db-stop          # Stop test database (optional)"
+	@echo "  make services-stop    # Stop test services (optional)"
 
 # Performance testing
 bench: ## Run benchmark tests

README.md

@@ -140,7 +140,7 @@ cp .env.example .env
 # Edit .env with your configuration
 
 # 4. Start dependencies and run migrations
-make db-start
+make services-start
 make migrate-up
 
 # 5. Start the server
@@ -175,7 +175,7 @@ The application uses environment variables for configuration. Copy `.env.example
 - **CORS**: Frontend domain configuration
 - **Logging**: Level and format settings
 
-**Note**: Redis configuration is required for task queuing and execution. The `.env.example` file includes database and JWT settings, but Redis configuration must be added manually.
+**Note**: Redis configuration is required for task queuing and execution. The `.env.example` file includes complete database, Redis, and JWT settings. For manual setup, use `make services-start` to start both PostgreSQL and Redis test services.
 
 ## Contributing
 

cmd/api/main.go

@@ -62,7 +62,7 @@ func main() {
 	log := logger.New(cfg.Logger.Level, cfg.Logger.Format)
 
 	// Initialize database connection
-	dbConn, err := database.NewConnection(&cfg.Database, log.Logger)
+	dbConn, err := database.NewConnectionWithRetry(&cfg.Database, log.Logger)
 	if err != nil {
 		log.Error("failed to initialize database connection", "error", err)
 		os.Exit(1)
@@ -116,8 +116,6 @@ func main() {
 		}
 	}()
 
-	log.Info("queue manager initialized successfully")
-
 	// Initialize JWT service
 	jwtService := auth.NewJWTService(&cfg.JWT)
 

config/README.md

@@ -110,8 +110,8 @@ SERVER_ENV=production ./bin/api
 
 ### 3. Testing Environment
 ```bash
-# Start test database and run integration tests
-make db-start
+# Start test services and run integration tests
+make services-start
 make test-integration
 
 # Or run all tests

docker-compose.dev.yml

@@ -53,33 +53,69 @@ services:
       REDIS_PORT: "6379"
       REDIS_PASSWORD: ""
       REDIS_DATABASE: "0"
+      REDIS_POOL_SIZE: "10"
+      REDIS_MIN_IDLE_CONNECTIONS: "5"
+      REDIS_MAX_RETRIES: "3"
+      REDIS_DIAL_TIMEOUT: "5s"
+      REDIS_READ_TIMEOUT: "3s"
+      REDIS_WRITE_TIMEOUT: "3s"
+      REDIS_IDLE_TIMEOUT: "5m"
 
       # Queue Configuration (Development)
       QUEUE_TASK_QUEUE_NAME: "voidrunner:tasks:dev"
-      QUEUE_RETRY_QUEUE_NAME: "voidrunner:tasks:retry:dev"
       QUEUE_DEAD_LETTER_QUEUE_NAME: "voidrunner:tasks:dead:dev"
+      QUEUE_RETRY_QUEUE_NAME: "voidrunner:tasks:retry:dev"
+      QUEUE_DEFAULT_PRIORITY: "5"
+      QUEUE_MAX_RETRIES: "3"
+      QUEUE_RETRY_DELAY: "30s"
+      QUEUE_RETRY_BACKOFF_FACTOR: "2.0"
+      QUEUE_MAX_RETRY_DELAY: "15m"
+      QUEUE_VISIBILITY_TIMEOUT: "30m"
+      QUEUE_MESSAGE_TTL: "24h"
+      QUEUE_BATCH_SIZE: "10"
 
       # Development Worker Settings
       WORKER_POOL_SIZE: "2"
       WORKER_MAX_CONCURRENT_TASKS: "10"
       WORKER_MAX_USER_CONCURRENT_TASKS: "3"
       WORKER_TASK_TIMEOUT: "5m"
+      WORKER_HEARTBEAT_INTERVAL: "30s"
+      WORKER_SHUTDOWN_TIMEOUT: "30s"
+      WORKER_CLEANUP_INTERVAL: "5m"
+      WORKER_STALE_TASK_THRESHOLD: "2h"
+      WORKER_ID_PREFIX: "voidrunner-worker"
 
       # Development Executor Settings (Relaxed Security)
+      DOCKER_ENDPOINT: "unix:///var/run/docker.sock"
       EXECUTOR_DEFAULT_MEMORY_LIMIT_MB: "256"
       EXECUTOR_DEFAULT_CPU_QUOTA: "50000"
+      EXECUTOR_DEFAULT_PIDS_LIMIT: "128"
       EXECUTOR_DEFAULT_TIMEOUT_SECONDS: "300"
+      EXECUTOR_PYTHON_IMAGE: "python:3.11-alpine"
+      EXECUTOR_BASH_IMAGE: "alpine:latest"
+      EXECUTOR_JAVASCRIPT_IMAGE: "node:18-alpine"
+      EXECUTOR_GO_IMAGE: "golang:1.21-alpine"
       EXECUTOR_ENABLE_SECCOMP: "false"  # Disabled for easier development
+      EXECUTOR_SECCOMP_PROFILE_PATH: "/opt/voidrunner/seccomp-profile.json"
+      EXECUTOR_ENABLE_APPARMOR: "false"
+      EXECUTOR_APPARMOR_PROFILE: "voidrunner-executor"
+      EXECUTOR_EXECUTION_USER: "1000:1000"
 
       # Development Logging
       LOG_LEVEL: debug
       LOG_FORMAT: console
 
       # Development JWT (Insecure)
       JWT_SECRET_KEY: "development-secret-key-change-in-production"
+      JWT_ACCESS_TOKEN_DURATION: "15m"
+      JWT_REFRESH_TOKEN_DURATION: "168h"
+      JWT_ISSUER: "voidrunner"
+      JWT_AUDIENCE: "voidrunner-api"
 
       # Development CORS (Permissive)
       CORS_ALLOWED_ORIGINS: "*"
+      CORS_ALLOWED_METHODS: "GET,POST,PUT,DELETE,OPTIONS"
+      CORS_ALLOWED_HEADERS: "Content-Type,Authorization,X-Request-ID"
 
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro

docker-compose.test.yml

@@ -17,5 +17,21 @@ services:
       retries: 5
     restart: unless-stopped
 
+  redis-test:
+    image: redis:7-alpine
+    container_name: voidrunner-test-redis
+    ports:
+      - "6379:6379"
+    command: redis-server --appendonly yes
+    volumes:
+      - redis_test_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 3s
+      retries: 5
+    restart: unless-stopped
+
 volumes:
   postgres_test_data:
+  redis_test_data: