async commit + wh

configs/config.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/rs/zerolog/log"
@@ -85,6 +86,13 @@ type ClickhouseConfig struct {
 	ChainBasedConfig             map[string]TableOverrideConfig `mapstructure:"chainBasedConfig"`
 	EnableParallelViewProcessing bool                           `mapstructure:"enableParallelViewProcessing"`
 	MaxQueryTime                 int                            `mapstructure:"maxQueryTime"`
+
+	// Readonly configuration for API endpoints
+	ReadonlyHost     string `mapstructure:"readonlyHost"`
+	ReadonlyPort     int    `mapstructure:"readonlyPort"`
+	ReadonlyUsername string `mapstructure:"readonlyUsername"`
+	ReadonlyPassword string `mapstructure:"readonlyPassword"`
+	ReadonlyDatabase string `mapstructure:"readonlyDatabase"`
 }
 
 type PostgresConfig struct {
@@ -282,5 +290,35 @@ func setCustomJSONConfigs() error {
 			Cfg.Storage.Main.Clickhouse.ChainBasedConfig = orchestratorChainConfig
 		}
 	}
+
+	// Load readonly ClickHouse configuration from environment variables
+	if readonlyHost := os.Getenv("CLICKHOUSE_HOST_READONLY"); readonlyHost != "" {
+		if Cfg.Storage.Main.Clickhouse != nil {
+			Cfg.Storage.Main.Clickhouse.ReadonlyHost = readonlyHost
+		}
+	}
+	if readonlyPort := os.Getenv("CLICKHOUSE_PORT_READONLY"); readonlyPort != "" {
+		if port, err := strconv.Atoi(readonlyPort); err == nil {
+			if Cfg.Storage.Main.Clickhouse != nil {
+				Cfg.Storage.Main.Clickhouse.ReadonlyPort = port
+			}
+		}
+	}
+	if readonlyUsername := os.Getenv("CLICKHOUSE_USER_READONLY"); readonlyUsername != "" {
+		if Cfg.Storage.Main.Clickhouse != nil {
+			Cfg.Storage.Main.Clickhouse.ReadonlyUsername = readonlyUsername
+		}
+	}
+	if readonlyPassword := os.Getenv("CLICKHOUSE_PASSWORD_READONLY"); readonlyPassword != "" {
+		if Cfg.Storage.Main.Clickhouse != nil {
+			Cfg.Storage.Main.Clickhouse.ReadonlyPassword = readonlyPassword
+		}
+	}
+	if readonlyDatabase := os.Getenv("CLICKHOUSE_DATABASE_READONLY"); readonlyDatabase != "" {
+		if Cfg.Storage.Main.Clickhouse != nil {
+			Cfg.Storage.Main.Clickhouse.ReadonlyDatabase = readonlyDatabase
+		}
+	}
+
 	return nil
 }

configs/config_readonly_example.yml

@@ -0,0 +1,42 @@
+# Example configuration for readonly ClickHouse API endpoints
+# This configuration allows you to use a separate readonly ClickHouse instance
+# for user-facing API queries while keeping the main orchestration flow unchanged
+
+storage:
+  main:
+    clickhouse:
+      # Main ClickHouse configuration (for orchestration)
+      host: "localhost"
+      port: 9000
+      username: "default"
+      password: "password"
+      database: "insight"
+
+      # Readonly ClickHouse configuration (for API endpoints)
+      # These environment variables will override the main config for API queries
+      readonlyHost: "${CLICKHOUSE_HOST_READONLY}"      # e.g., "readonly-clickhouse.example.com"
+      readonlyPort: "${CLICKHOUSE_PORT_READONLY}"      # e.g., 9000
+      readonlyUsername: "${CLICKHOUSE_USER_READONLY}"  # e.g., "readonly_user"
+      readonlyPassword: "${CLICKHOUSE_PASSWORD_READONLY}" # e.g., "readonly_password"
+      readonlyDatabase: "${CLICKHOUSE_DATABASE_READONLY}" # e.g., "insight_readonly"
+
+      # Other ClickHouse settings
+      disableTLS: false
+      maxOpenConns: 100
+      maxIdleConns: 10
+      maxRowsPerInsert: 100000
+      enableParallelViewProcessing: true
+      maxQueryTime: 300
+
+# Environment variables to set:
+# CLICKHOUSE_HOST_READONLY=readonly-clickhouse.example.com
+# CLICKHOUSE_PORT_READONLY=9000
+# CLICKHOUSE_USER_READONLY=readonly_user
+# CLICKHOUSE_PASSWORD_READONLY=readonly_password
+# CLICKHOUSE_DATABASE_READONLY=insight_readonly
+
+# How it works:
+# 1. When readonly environment variables are set, API endpoints will use the readonly ClickHouse instance
+# 2. The orchestration flow (indexer, committer, etc.) will continue to use the main ClickHouse instance
+# 3. This provides read/write separation and allows you to scale your readonly queries independently
+# 4. If readonly configuration is not provided, API endpoints will fall back to the main ClickHouse instance

docs/README_READONLY_CLICKHOUSE.md

@@ -0,0 +1,235 @@
+# Readonly ClickHouse for API Endpoints
+
+This document explains how to configure and use a readonly ClickHouse instance for user-facing API endpoints while keeping the main orchestration flow unchanged.
+
+## Overview
+
+The readonly ClickHouse feature allows you to:
+- Use a separate, read-only ClickHouse instance for API queries
+- Keep the main ClickHouse instance for orchestration (indexing, committing, etc.)
+- Scale read and write operations independently
+- Improve API performance by using dedicated read replicas
+
+## Configuration
+
+### Environment Variables
+
+Set the following environment variables to enable readonly ClickHouse:
+
+```bash
+export CLICKHOUSE_HOST_READONLY="readonly-clickhouse.example.com"
+export CLICKHOUSE_PORT_READONLY=9000
+export CLICKHOUSE_USER_READONLY="readonly_user"
+export CLICKHOUSE_PASSWORD_READONLY="readonly_password"
+export CLICKHOUSE_DATABASE_READONLY="insight_readonly"
+```
+
+### Configuration File
+
+You can also set these values in your configuration file:
+
+```yaml
+storage:
+  main:
+    clickhouse:
+      # Main ClickHouse configuration (for orchestration)
+      host: "localhost"
+      port: 9000
+      username: "default"
+      password: "password"
+      database: "insight"
+
+      # Readonly ClickHouse configuration (for API endpoints)
+      readonlyHost: "readonly-clickhouse.example.com"
+      readonlyPort: 9000
+      readonlyUsername: "readonly_user"
+      readonlyPassword: "readonly_password"
+      readonlyDatabase: "insight_readonly"
+```
+
+## How It Works
+
+### Automatic Detection
+
+The system automatically detects if readonly configuration is available:
+
+1. **With readonly config**: API endpoints use the readonly ClickHouse instance
+2. **Without readonly config**: API endpoints fall back to the main ClickHouse instance
+
+### Affected Endpoints
+
+The following API endpoints will use the readonly ClickHouse instance when configured:
+
+- `GET /{chainId}/blocks` - Block queries
+- `GET /{chainId}/transactions` - Transaction queries
+- `GET /{chainId}/events` - Event/log queries
+- `GET /{chainId}/transfers` - Token transfer queries
+- `GET /{chainId}/balances/{owner}` - Token balance queries
+- `GET /{chainId}/holders/{address}` - Token holder queries
+- `GET /{chainId}/search/{input}` - Search queries
+
+### Unaffected Operations
+
+The following operations continue to use the main ClickHouse instance:
+
+- Block indexing and polling
+- Transaction processing
+- Event/log processing
+- Staging data operations
+- Orchestration flow (committer, failure recovery, etc.)
+
+## Implementation Details
+
+### Readonly Connector
+
+The `ClickHouseReadonlyConnector` implements the same interface as the main connector but:
+
+- Only allows read operations
+- Panics on write operations (ensuring readonly behavior)
+- Uses readonly connection parameters
+- Falls back to main config if readonly config is incomplete
+
+### Storage Factory
+
+The `NewReadonlyConnector` function creates readonly connectors:
+
+```go
+// For API endpoints (readonly if configured)
+storage.NewReadonlyConnector[storage.IMainStorage](&config.Cfg.Storage.Main)
+
+// For orchestration (always main connector)
+storage.NewConnector[storage.IMainStorage](&config.Cfg.Storage.Main)
+```
+
+## Setup Instructions
+
+### 1. Create Readonly ClickHouse Instance
+
+Set up a ClickHouse replica or read-only instance:
+
+```sql
+-- On your readonly ClickHouse instance
+CREATE DATABASE insight_readonly;
+-- Grant readonly permissions to readonly_user
+GRANT SELECT ON insight_readonly.* TO readonly_user;
+```
+
+### 2. Set Environment Variables
+
+```bash
+export CLICKHOUSE_HOST_READONLY="your-readonly-host"
+export CLICKHOUSE_PORT_READONLY=9000
+export CLICKHOUSE_USER_READONLY="readonly_user"
+export CLICKHOUSE_PASSWORD_READONLY="readonly_password"
+export CLICKHOUSE_DATABASE_READONLY="insight_readonly"
+```
+
+### 3. Restart API Server
+
+Restart your API server to pick up the new configuration:
+
+```bash
+./insight api
+```
+
+### 4. Verify Configuration
+
+Check the logs to confirm which connector is being used:
+
+```
+INFO Using readonly ClickHouse connector for API endpoints
+```
+
+## Monitoring and Troubleshooting
+
+### Log Messages
+
+- **"Using readonly ClickHouse connector for API endpoints"** - Readonly mode active
+- **"Using regular ClickHouse connector for API endpoints"** - Fallback to main connector
+
+### Common Issues
+
+1. **Connection refused**: Check readonly ClickHouse host/port
+2. **Authentication failed**: Verify readonly username/password
+3. **Database not found**: Ensure readonly database exists
+4. **Permission denied**: Grant SELECT permissions to readonly user
+
+### Testing
+
+Test the readonly connection:
+
+```bash
+# Test readonly connection
+clickhouse-client --host=readonly-host --port=9000 --user=readonly_user --password=readonly_password --database=insight_readonly -q "SELECT 1"
+```
+
+## Performance Considerations
+
+### Read Replicas
+
+- Use ClickHouse read replicas for better performance
+- Consider geographic distribution for global users
+- Monitor replica lag to ensure data consistency
+
+### Connection Pooling
+
+The readonly connector uses the same connection pool settings as the main connector:
+
+```yaml
+maxOpenConns: 100
+maxIdleConns: 10
+```
+
+### Query Optimization
+
+- Readonly instances can be optimized for query performance
+- Consider different ClickHouse settings for readonly vs. write instances
+- Use materialized views on readonly instances for complex queries
+
+## Security
+
+### Network Security
+
+- Restrict readonly ClickHouse to internal networks
+- Use VPN or private subnets for readonly access
+- Consider ClickHouse's built-in network security features
+
+### User Permissions
+
+- Create dedicated readonly user with minimal permissions
+- Grant only SELECT permissions on required tables
+- Regularly rotate readonly user passwords
+
+### Data Access
+
+- Readonly users cannot modify data
+- No risk of accidental data corruption
+- Audit logs show readonly access patterns
+
+## Migration
+
+### From Single Instance
+
+1. Set up readonly ClickHouse instance
+2. Configure environment variables
+3. Restart API server
+4. Monitor performance and errors
+5. Gradually migrate more endpoints if needed
+
+### Rollback
+
+To rollback to main ClickHouse:
+
+1. Remove readonly environment variables
+2. Restart API server
+3. API endpoints will automatically use main connector
+
+## Support
+
+For issues or questions about the readonly ClickHouse feature:
+
+1. Check the logs for error messages
+2. Verify ClickHouse connectivity
+3. Review configuration parameters
+4. Check ClickHouse server logs
+5. Contact the development team

internal/handlers/logs_handlers.go

@@ -2,7 +2,6 @@ package handlers
 
 import (
 	"net/http"
-	"sync"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/gin-gonic/gin"
@@ -13,13 +12,6 @@ import (
 	"github.com/thirdweb-dev/indexer/internal/storage"
 )
 
-// package-level variables
-var (
-	mainStorage storage.IMainStorage
-	storageOnce sync.Once
-	storageErr  error
-)
-
 // @Summary Get all logs
 // @Description Retrieve all logs across all contracts
 // @Tags events
@@ -221,18 +213,6 @@ func decodeLogsIfNeeded(chainId string, logs []common.Log, eventABI *abi.Event,
 	return nil
 }
 
-func getMainStorage() (storage.IMainStorage, error) {
-	storageOnce.Do(func() {
-		var err error
-		mainStorage, err = storage.NewConnector[storage.IMainStorage](&config.Cfg.Storage.Main)
-		if err != nil {
-			storageErr = err
-			log.Error().Err(err).Msg("Error creating storage connector")
-		}
-	})
-	return mainStorage, storageErr
-}
-
 func sendJSONResponse(c *gin.Context, response interface{}) {
 	c.JSON(http.StatusOK, response)
 }