diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..5bfb0b3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,116 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +The CDP Pipeline Workflow is a Stellar blockchain data processing pipeline built in Go. The system processes Stellar ledger data through a modular architecture consisting of Sources, Processors, and Consumers that can be chained together via YAML configuration files. + +## Development Commands + +### Building the Application +```bash +# Build locally (requires CGO and ZeroMQ libraries) +CGO_ENABLED=1 go build -o cdp-pipeline-workflow + +# Build production Docker image +docker build -t obsrvr-flow-pipeline -f dockerfile . + +# Build development Docker image +docker build -t cdp-pipeline-dev -f dockerfile.dev . +``` + +### Running the Application +```bash +# Run with default config +./cdp-pipeline-workflow -config pipeline_config.yaml + +# Run with specific config file +./cdp-pipeline-workflow -config config/base/pipeline_config.yaml + +# Run locally with Docker +./run-local.sh config/base/pipeline_config.yaml + +# Run development ZeroMQ pipeline +./run-dev-zeromq.sh +``` + +### Testing +```bash +# Run Go tests +go test ./... + +# Run tests with verbose output +go test -v ./... + +# Test specific package +go test ./pkg/processor/... +``` + +## Architecture + +### Core Components + +1. **Sources** (`pkg/source/`, `source_adapter_*.go`): Data ingestion from various sources + - `CaptiveCoreInboundAdapter`: Direct Stellar Core connection + - `BufferedStorageSourceAdapter`: S3/GCS/filesystem storage + - `RPCSourceAdapter`: Stellar RPC endpoints + - `SorobanSourceAdapter`: Soroban smart contract events + +2. **Processors** (`pkg/processor/`, `processor/`): Data transformation and filtering + - Payment processors: `FilterPayments`, `TransformToAppPayment` + - Account processors: `AccountData`, `CreateAccount`, `AccountTransaction` + - Contract processors: `ContractInvocation`, `ContractEvent`, `ContractLedgerReader` + - Market processors: `TransformToAppTrade`, `MarketMetricsProcessor` + +3. **Consumers** (`pkg/consumer/`, `consumer/`): Data output and storage + - Database: `SaveToPostgreSQL`, `SaveToMongoDB`, `SaveToDuckDB`, `SaveToClickHouse` + - Cache: `SaveToRedis`, `SaveToRedisOrderbook` + - Files: `SaveToExcel`, `SaveToGCS` + - Messaging: `SaveToZeroMQ`, `SaveToWebSocket` + +### Data Flow Pattern + +``` +Source -> Processor(s) -> Consumer(s) +``` + +- Sources capture blockchain events and emit `Message` objects +- Processors subscribe to sources/other processors and transform data +- Consumers subscribe to processors and persist/forward data +- All components implement the `Processor` interface for uniform chaining + +### Configuration System + +- Pipeline configurations are defined in YAML files under `config/base/` +- Each pipeline specifies a source, optional processors, and consumers +- Multiple pipelines can be defined in a single config file +- Factory pattern in `main.go` instantiates components based on config type strings + +### Key Interfaces + +- `types.Processor`: Core processing interface with `Process()` and `Subscribe()` methods +- `types.Message`: Wrapper for data payloads passed between components +- `SourceAdapter`: Interface for data sources with `Run()` and `Subscribe()` methods + +## Development Notes + +### Dependencies +- Requires CGO for DuckDB and ZeroMQ support +- Uses Stellar Go SDK for blockchain data structures +- Heavy use of YAML configuration with reflection-based component instantiation + +### Code Organization +- Legacy code in root directory (`processor/`, `consumer/`, `source_adapter_*.go`) +- New modular code in `pkg/` directory with factory patterns +- Both entry points exist: `main.go` (legacy) and `cmd/pipeline/main.go` (new) + +### Configuration Files +- Base configurations in `config/base/` +- Secret configurations (credentials) use `.secret.yaml` extension +- Template configurations available for common use cases + +### Docker Development +- Development workflow uses `dockerfile.dev` with debugging tools +- Production builds use multi-stage `dockerfile` with minimal runtime +- `run-local.sh` script simplifies local Docker development \ No newline at end of file diff --git a/MONOREPO.md b/MONOREPO.md new file mode 100644 index 0000000..d3086a8 --- /dev/null +++ b/MONOREPO.md @@ -0,0 +1,130 @@ +# CDP Pipeline Workflow Monorepo + +This repository is structured as a monorepo to facilitate easier development, testing, and deployment of the CDP Pipeline Workflow components. + +## Directory Structure + +``` +cdp-pipeline-workflow/ +├── cmd/ # Command-line applications +│ ├── pipeline/ # Main pipeline application +│ └── tools/ # Utility tools +├── internal/ # Private application and library code +│ ├── common/ # Shared code used across the project +│ │ ├── types/ # Common type definitions +│ │ ├── utils/ # Utility functions +│ │ └── config/ # Configuration handling +│ └── platform/ # Platform-specific implementations +│ └── stellar/ # Stellar-specific code +├── pkg/ # Public library code +│ ├── processor/ # Core processor interfaces and implementations +│ │ ├── base/ # Base processor implementations +│ │ ├── contract/ # Contract-specific processors +│ │ │ ├── kale/ # Kale processor +│ │ │ ├── soroswap/ # Soroswap processor +│ │ │ └── common/ # Shared contract processor code +│ │ └── ledger/ # Ledger processors +│ ├── consumer/ # Consumer implementations +│ │ ├── database/ # Database consumers (PostgreSQL, etc.) +│ │ ├── messaging/ # Messaging consumers (ZeroMQ, Kafka, etc.) +│ │ ├── api/ # API consumers (REST, GraphQL, etc.) +│ │ └── websocket/ # WebSocket consumers +│ └── pluginapi/ # Plugin API interfaces and implementations +├── plugins/ # Plugin implementations +│ ├── kale/ # Kale plugin +│ └── soroswap/ # Soroswap plugin +├── examples/ # Example code +│ ├── kale/ # Kale examples +│ └── soroswap/ # Soroswap examples +├── docs/ # Documentation +│ ├── architecture/ # Architecture diagrams and descriptions +│ ├── api/ # API documentation +│ └── guides/ # User and developer guides +├── deployment/ # Deployment configurations +│ ├── kubernetes/ # Kubernetes manifests +│ └── docker/ # Dockerfiles and docker-compose files +├── tests/ # Integration and end-to-end tests +│ ├── integration/ # Integration tests +│ └── e2e/ # End-to-end tests +├── go.mod # Go modules definition +└── README.md # Project overview +``` + +## Package Structure + +### cmd/ + +Contains the main executable applications of the project. Each subdirectory is a separate application. + +### internal/ + +Contains code that's private to this repository and not meant to be imported by other projects. This is enforced by Go's module system. + +### pkg/ + +Contains code that can be imported and used by other projects. This is where most of the reusable components live. + +#### pkg/processor/ + +Contains all processor implementations, with interfaces defined in the `base` package. + +- **base/**: Core interfaces and base implementations +- **contract/**: Processors for specific smart contracts +- **ledger/**: Processors for Stellar ledger data + +#### pkg/consumer/ + +Contains all consumer implementations, with interfaces defined in the `base` package. + +- **base/**: Core interfaces and base implementations +- **database/**: Consumers that save data to databases +- **messaging/**: Consumers that publish to message queues +- **api/**: Consumers that expose data via APIs +- **websocket/**: Consumers that send data over WebSockets + +### plugins/ + +Contains plugin implementations that can be loaded dynamically. + +### examples/ + +Contains example code showing how to use the various components. + +## Key Interfaces + +### Processor + +```go +// Processor defines the interface for processing messages. +type Processor interface { + Process(context.Context, Message) error + Subscribe(Processor) +} +``` + +### Consumer + +```go +// Consumer defines the interface for consuming messages. +type Consumer interface { + Process(context.Context, Message) error + Subscribe(Processor) +} +``` + +## Getting Started + +See the examples directory for usage examples. + +## Building and Testing + +```bash +# Build all applications +go build ./cmd/... + +# Run tests +go test ./... + +# Run specific example +go run ./examples/kale/kale_processor_example.go +``` \ No newline at end of file diff --git a/README_NEW.md b/README_NEW.md new file mode 100644 index 0000000..3754837 --- /dev/null +++ b/README_NEW.md @@ -0,0 +1,102 @@ +# CDP Pipeline Workflow + +A flexible pipeline system for processing blockchain data, with a focus on Stellar and Soroban contracts. + +## Overview + +CDP Pipeline Workflow is a modular data processing system designed to ingest, process, and store blockchain data. The system uses a pipeline architecture with processors and consumers that can be configured to handle different types of data and operations. + +## Features + +- Modular design with pluggable processors and consumers +- Support for multiple blockchain data sources +- Built-in processors for Stellar and Soroban smart contracts +- Support for various data sinks (PostgreSQL, Redis, WebSockets, etc.) +- Flexible configuration via YAML +- Extensible plugin system + +## Getting Started + +### Prerequisites + +- Go 1.23 or later +- Access to a Stellar/Soroban node or archive (for live data) + +### Installation + +```bash +# Clone the repository +git clone https://github.com/withObsrvr/cdp-pipeline-workflow.git +cd cdp-pipeline-workflow + +# Build the main application +go build -o pipeline ./cmd/pipeline +``` + +### Basic Usage + +1. Configure your pipeline in YAML: + +```yaml +pipelines: + kale: + name: "Kale Processor Pipeline" + source: + type: "rpc" + config: + url: "https://your-soroban-rpc-endpoint" + processors: + - type: "contract_filter" + config: + contract_id: "CAKXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + - type: "kale" + config: + contract_id: "CAKXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + consumers: + - type: "postgresql" + config: + connection_string: "postgres://user:password@localhost:5432/database" +``` + +2. Run the pipeline: + +```bash +./pipeline --config your_config.yaml +``` + +## Architecture + +See [MONOREPO.md](MONOREPO.md) for details on the repository structure and architecture. + +## Development + +### Adding a New Processor + +1. Create a new directory in `pkg/processor/` for your processor category +2. Implement the `base.Processor` interface +3. Register your processor in `pkg/processor/base/factory.go` + +### Adding a New Consumer + +1. Create a new directory in `pkg/consumer/` for your consumer category +2. Implement the `base.Consumer` interface +3. Register your consumer in `pkg/consumer/base/factory.go` + +### Running Tests + +```bash +go test ./... +``` + +## Examples + +See the `examples/` directory for usage examples. + +## License + +[Specify your license here] + +## Acknowledgments + +- The Stellar Development Foundation +- [Other acknowledgments] \ No newline at end of file diff --git a/cmd/pipeline/main.go b/cmd/pipeline/main.go new file mode 100644 index 0000000..95404bd --- /dev/null +++ b/cmd/pipeline/main.go @@ -0,0 +1,251 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/signal" + + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/consumer" + _ "github.com/withObsrvr/cdp-pipeline-workflow/pkg/processor" // Import for side effects to register processors + procbase "github.com/withObsrvr/cdp-pipeline-workflow/pkg/processor/base" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/source" + srcbase "github.com/withObsrvr/cdp-pipeline-workflow/pkg/source/base" + "gopkg.in/yaml.v2" +) + +type Config struct { + Pipelines map[string]PipelineConfig `yaml:"pipelines"` +} + +type PipelineConfig struct { + Name string `yaml:"name"` + Source srcbase.SourceConfig `yaml:"source"` + Processors []types.ProcessorConfig `yaml:"processors"` + Consumers []consumer.ConsumerConfig `yaml:"consumers"` +} + +// SourceAdapter interface defined inline to match the old API +type SourceAdapter interface { + Run(context.Context) error + Subscribe(types.Processor) +} + +func main() { + // Define command line flags + configFile := flag.String("config", "pipeline_config.yaml", "Path to pipeline configuration file") + flag.Parse() + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) + defer stop() + + // Read configuration from specified file + configBytes, err := os.ReadFile(*configFile) + if err != nil { + log.Fatalf("Error reading config file %s: %v", *configFile, err) + } + + var config Config + if err := yaml.Unmarshal(configBytes, &config); err != nil { + log.Fatalf("Error parsing config: %v", err) + } + + // Run each pipeline + for name, pipelineConfig := range config.Pipelines { + log.Printf("Starting pipeline: %s", name) + if err := setupPipeline(ctx, pipelineConfig); err != nil { + log.Printf("Pipeline error: error in pipeline %s: %v", name, err) + } + } + + log.Printf("All pipelines finished.") +} + +func createSourceAdapter(sourceConfig srcbase.SourceConfig) (SourceAdapter, error) { + switch sourceConfig.Type { + case "CaptiveCoreInboundAdapter": + return source.NewCaptiveCoreInboundAdapter(sourceConfig.Config) + case "BufferedStorageSourceAdapter": + return source.NewBufferedStorageSourceAdapter(sourceConfig.Config) + case "SorobanSourceAdapter": + return source.NewSorobanSourceAdapter(sourceConfig.Config) + case "GCSBufferedStorageSourceAdapter": + return source.NewGCSBufferedStorageSourceAdapter(sourceConfig.Config) + case "FSBufferedStorageSourceAdapter": + return source.NewFSBufferedStorageSourceAdapter(sourceConfig.Config) + case "S3BufferedStorageSourceAdapter": + return source.NewS3BufferedStorageSourceAdapter(sourceConfig.Config) + case "RPCSourceAdapter", "rpc": + return source.NewRPCSourceAdapter(sourceConfig.Config) + // Add more source types as needed + default: + return nil, fmt.Errorf("unsupported source type: %s", sourceConfig.Type) + } +} + +func createProcessors(processorConfigs []types.ProcessorConfig) ([]types.Processor, error) { + processors := make([]types.Processor, len(processorConfigs)) + for i, config := range processorConfigs { + processor, err := createProcessor(config) + if err != nil { + return nil, err + } + processors[i] = processor + } + return processors, nil +} + +func createProcessor(processorConfig types.ProcessorConfig) (types.Processor, error) { + // Use the factory pattern we implemented + return procbase.NewProcessor(processorConfig) +} + +func createConsumer(consumerConfig consumer.ConsumerConfig) (types.Processor, error) { + switch consumerConfig.Type { + case "SaveToExcel": + return consumer.NewSaveToExcel(consumerConfig.Config) + case "SaveToMongoDB": + return consumer.NewSaveToMongoDB(consumerConfig.Config) + case "SaveToZeroMQ": + return consumer.NewSaveToZeroMQ(consumerConfig.Config) + case "SaveToGCS": + return consumer.NewSaveToGCS(consumerConfig.Config) + case "SaveToDuckDB": + return consumer.NewSaveToDuckDB(consumerConfig.Config) + case "SaveContractToDuckDB": + return consumer.NewSaveContractToDuckDB(consumerConfig.Config) + case "SaveToTimescaleDB": + return consumer.NewSaveToTimescaleDB(consumerConfig.Config) + case "SaveToRedis": + return consumer.NewSaveToRedis(consumerConfig.Config) + case "NotificationDispatcher": + return consumer.NewNotificationDispatcher(consumerConfig.Config) + case "SaveToWebSocket": + return consumer.NewSaveToWebSocket(consumerConfig.Config) + case "SaveToPostgreSQL": + return consumer.NewSaveToPostgreSQL(consumerConfig.Config) + case "SaveToClickHouse": + return consumer.NewSaveToClickHouse(consumerConfig.Config) + case "SaveToMarketAnalytics": + return consumer.NewSaveToMarketAnalyticsConsumer(consumerConfig.Config) + case "SaveToRedisOrderbook": + return consumer.NewSaveToRedisOrderbookConsumer(consumerConfig.Config) + case "SaveAssetToPostgreSQL": + return consumer.NewSaveAssetToPostgreSQL(consumerConfig.Config) + case "SaveAssetEnrichment": + return consumer.NewSaveAssetEnrichmentConsumer(consumerConfig.Config) + case "SavePaymentToPostgreSQL": + return consumer.NewSavePaymentToPostgreSQL(consumerConfig.Config) + case "SavePaymentsToRedis": + return consumer.NewSavePaymentsToRedis(consumerConfig.Config) + case "SaveLatestLedgerToRedis": + return consumer.NewSaveLatestLedgerRedis(consumerConfig.Config) + case "SaveLatestLedgerToExcel": + return consumer.NewSaveLatestLedgerToExcel(consumerConfig.Config) + case "AnthropicClaude": + return consumer.NewAnthropicClaudeConsumer(consumerConfig.Config) + case "StdoutConsumer": + return consumer.NewStdoutConsumer(), nil + case "SaveSoroswapPairsToDuckDB": + return consumer.NewSaveSoroswapPairsToDuckDB(consumerConfig.Config) + case "SaveSoroswapRouterToDuckDB": + return consumer.NewSaveSoroswapRouterToDuckDB(consumerConfig.Config) + case "SaveAccountDataToPostgreSQL": + return consumer.NewSaveAccountDataToPostgreSQL(consumerConfig.Config) + case "SaveAccountDataToDuckDB": + return consumer.NewSaveAccountDataToDuckDB(consumerConfig.Config) + case "SaveContractEventsToPostgreSQL": + return consumer.NewSaveContractEventsToPostgreSQL(consumerConfig.Config) + case "SaveSoroswapToPostgreSQL": + return consumer.NewSaveSoroswapToPostgreSQL(consumerConfig.Config) + case "SaveContractInvocationsToPostgreSQL": + return consumer.NewSaveContractInvocationsToPostgreSQL(consumerConfig.Config) + default: + return nil, fmt.Errorf("unsupported consumer type: %s", consumerConfig.Type) + } +} + +func createConsumers(consumerConfigs []consumer.ConsumerConfig) ([]types.Processor, error) { + consumers := make([]types.Processor, len(consumerConfigs)) + for i, config := range consumerConfigs { + consumer, err := createConsumer(config) + if err != nil { + return nil, err + } + consumers[i] = consumer + } + return consumers, nil +} + +// buildProcessorChain chains processors sequentially and subscribes all consumers to the last processor +func buildProcessorChain(processors []types.Processor, consumers []types.Processor) { + var lastProcessor types.Processor + + // Chain all processors sequentially + for _, p := range processors { + if lastProcessor != nil { + lastProcessor.Subscribe(p) + log.Printf("Chained processor %T -> %T", lastProcessor, p) + } + lastProcessor = p + } + + // If any consumers are provided, subscribe them to the last processor + if lastProcessor != nil { + for _, c := range consumers { + lastProcessor.Subscribe(c) + log.Printf("Chained processor %T -> consumer %T", lastProcessor, c) + } + } else if len(consumers) > 0 { + // If no processors but multiple consumers, chain the consumers + for i := 1; i < len(consumers); i++ { + consumers[0].Subscribe(consumers[i]) + log.Printf("Chained consumer %T -> consumer %T", consumers[0], consumers[i]) + } + } +} + +func setupPipeline(ctx context.Context, pipelineConfig PipelineConfig) error { + // Create source + source, err := createSourceAdapter(pipelineConfig.Source) + if err != nil { + return fmt.Errorf("error creating source: %w", err) + } + + // Create processors + processors := make([]types.Processor, len(pipelineConfig.Processors)) + for i, procConfig := range pipelineConfig.Processors { + proc, err := createProcessor(procConfig) + if err != nil { + return fmt.Errorf("error creating processor %s: %w", procConfig.Type, err) + } + processors[i] = proc + } + + // Create consumers + consumers := make([]types.Processor, len(pipelineConfig.Consumers)) + for i, consConfig := range pipelineConfig.Consumers { + cons, err := createConsumer(consConfig) + if err != nil { + return fmt.Errorf("error creating consumer %s: %w", consConfig.Type, err) + } + consumers[i] = cons + } + + // Build the chain + buildProcessorChain(processors, consumers) + + // Connect source to the first processor + if len(processors) > 0 { + source.Subscribe(processors[0]) + } else if len(consumers) > 0 { + // If no processors, subscribe source directly to consumers + source.Subscribe(consumers[0]) + } + + // Run the source with context + return source.Run(ctx) +} diff --git a/pipeline_config.yaml b/config/base/pipeline_config.yaml similarity index 100% rename from pipeline_config.yaml rename to config/base/pipeline_config.yaml diff --git a/pipeline_config_all.yaml b/config/base/pipeline_config_all.yaml similarity index 100% rename from pipeline_config_all.yaml rename to config/base/pipeline_config_all.yaml diff --git a/pipeline_config_blank.yaml b/config/base/pipeline_config_blank.yaml similarity index 100% rename from pipeline_config_blank.yaml rename to config/base/pipeline_config_blank.yaml diff --git a/pipeline_config_buffered.yaml b/config/base/pipeline_config_buffered.yaml similarity index 100% rename from pipeline_config_buffered.yaml rename to config/base/pipeline_config_buffered.yaml diff --git a/docs/contract-invocation-enhancement-plan.md b/docs/contract-invocation-enhancement-plan.md new file mode 100644 index 0000000..55536aa --- /dev/null +++ b/docs/contract-invocation-enhancement-plan.md @@ -0,0 +1,340 @@ +# Contract Invocation Processor Enhancement Plan + +## Executive Summary + +The current `ContractInvocationProcessor` in the CDP Pipeline Workflow has significant limitations in extracting function names and arguments from Soroban contract invocations. This document outlines a comprehensive plan to enhance the processor to provide complete function call information, making it more useful for contract analysis and monitoring. + +## Current State Analysis + +### Limitations Identified + +1. **Incomplete Function Name Extraction** + - Current implementation only checks `args[0]` for a Symbol type + - Fails when function name is stored in different formats or locations + - No fallback mechanisms for edge cases + +2. **Missing Argument Processing** + - Arguments beyond the function name are completely ignored + - No conversion from XDR ScVal types to human-readable formats + - The `ContractInvocation` struct lacks an `Arguments` field + +3. **Limited Data Structure** + ```go + type ContractInvocation struct { + FunctionName string `json:"function_name,omitempty"` // Often empty + // Missing: Arguments field + } + ``` + +### Current Implementation + +```go +// Current function name extraction (lines 202-209) +if function := invokeHostFunction.HostFunction; function.Type == xdr.HostFunctionTypeHostFunctionTypeInvokeContract { + if args := function.MustInvokeContract().Args; len(args) > 0 { + if sym, ok := args[0].GetSym(); ok { + invocation.FunctionName = string(sym) + } + } +} +``` + +## Proposed Enhancements + +### 1. Enhanced Data Structure + +Update the `ContractInvocation` struct to include complete function call information: + +```go +type ContractInvocation struct { + Timestamp time.Time `json:"timestamp"` + LedgerSequence uint32 `json:"ledger_sequence"` + TransactionHash string `json:"transaction_hash"` + ContractID string `json:"contract_id"` + InvokingAccount string `json:"invoking_account"` + FunctionName string `json:"function_name"` + Arguments []json.RawMessage `json:"arguments,omitempty"` // NEW + ArgumentsDecoded map[string]interface{} `json:"arguments_decoded,omitempty"` // NEW + Successful bool `json:"successful"` + DiagnosticEvents []DiagnosticEvent `json:"diagnostic_events,omitempty"` + ContractCalls []ContractCall `json:"contract_calls,omitempty"` + StateChanges []StateChange `json:"state_changes,omitempty"` + TtlExtensions []TtlExtension `json:"ttl_extensions,omitempty"` +} +``` + +### 2. Robust Function Name Extraction + +Implement a more comprehensive function name extraction: + +```go +func extractFunctionName(invokeContract xdr.InvokeContractArgs) string { + // Primary method: Use FunctionName field directly + if len(invokeContract.FunctionName) > 0 { + return string(invokeContract.FunctionName) + } + + // Fallback: Check first argument + if len(invokeContract.Args) > 0 { + return getFunctionNameFromScVal(invokeContract.Args[0]) + } + + return "unknown" +} + +func getFunctionNameFromScVal(val xdr.ScVal) string { + switch val.Type { + case xdr.ScValTypeScvSymbol: + return string(*val.Sym) + case xdr.ScValTypeScvString: + return string(*val.Str) + case xdr.ScValTypeScvBytes: + return string(*val.Bytes) + default: + return "" + } +} +``` + +### 3. Complete Argument Processing + +Process all function arguments with proper type conversion: + +```go +func extractArguments(args []xdr.ScVal) ([]json.RawMessage, map[string]interface{}, error) { + rawArgs := make([]json.RawMessage, 0, len(args)) + decodedArgs := make(map[string]interface{}) + + for i, arg := range args { + // Convert ScVal to JSON-serializable format + converted, err := ConvertScValToJSON(arg) + if err != nil { + log.Printf("Error converting argument %d: %v", i, err) + converted = map[string]string{"error": err.Error(), "type": arg.Type.String()} + } + + // Store raw JSON + jsonBytes, err := json.Marshal(converted) + if err != nil { + continue + } + rawArgs = append(rawArgs, jsonBytes) + + // Store in decoded map with index + decodedArgs[fmt.Sprintf("arg_%d", i)] = converted + } + + return rawArgs, decodedArgs, nil +} +``` + +### 4. ScVal Type Conversion Utility + +Create a comprehensive ScVal to JSON converter: + +```go +func ConvertScValToJSON(val xdr.ScVal) (interface{}, error) { + switch val.Type { + case xdr.ScValTypeScvBool: + return *val.B, nil + + case xdr.ScValTypeScvVoid: + return nil, nil + + case xdr.ScValTypeScvU32: + return *val.U32, nil + + case xdr.ScValTypeScvI32: + return *val.I32, nil + + case xdr.ScValTypeScvU64: + return *val.U64, nil + + case xdr.ScValTypeScvI64: + return *val.I64, nil + + case xdr.ScValTypeScvU128: + parts := *val.U128 + return map[string]interface{}{ + "type": "u128", + "hi": parts.Hi, + "lo": parts.Lo, + "value": fmt.Sprintf("%d", uint128ToInt(parts)), + }, nil + + case xdr.ScValTypeScvI128: + parts := *val.I128 + return map[string]interface{}{ + "type": "i128", + "hi": parts.Hi, + "lo": parts.Lo, + "value": fmt.Sprintf("%d", int128ToInt(parts)), + }, nil + + case xdr.ScValTypeScvSymbol: + return string(*val.Sym), nil + + case xdr.ScValTypeScvString: + return string(*val.Str), nil + + case xdr.ScValTypeScvBytes: + return map[string]interface{}{ + "type": "bytes", + "hex": hex.EncodeToString(*val.Bytes), + "base64": base64.StdEncoding.EncodeToString(*val.Bytes), + }, nil + + case xdr.ScValTypeScvAddress: + addr := val.Address + switch addr.Type { + case xdr.ScAddressTypeScAddressTypeAccount: + accountID := addr.AccountId.Ed25519 + strkey, _ := strkey.Encode(strkey.VersionByteAccountID, accountID[:]) + return map[string]interface{}{ + "type": "account", + "address": strkey, + }, nil + case xdr.ScAddressTypeScAddressTypeContract: + contractID := addr.ContractId + strkey, _ := strkey.Encode(strkey.VersionByteContract, contractID[:]) + return map[string]interface{}{ + "type": "contract", + "address": strkey, + }, nil + } + + case xdr.ScValTypeScvVec: + vec := *val.Vec + result := make([]interface{}, len(*vec)) + for i, item := range *vec { + converted, err := ConvertScValToJSON(item) + if err != nil { + result[i] = map[string]string{"error": err.Error()} + } else { + result[i] = converted + } + } + return result, nil + + case xdr.ScValTypeScvMap: + scMap := *val.Map + result := make(map[string]interface{}) + for _, entry := range *scMap { + key, _ := ConvertScValToJSON(entry.Key) + value, _ := ConvertScValToJSON(entry.Val) + keyStr := fmt.Sprintf("%v", key) + result[keyStr] = value + } + return result, nil + + case xdr.ScValTypeScvContractInstance: + return map[string]interface{}{ + "type": "contract_instance", + "value": "complex_type", + }, nil + + default: + return nil, fmt.Errorf("unsupported ScVal type: %s", val.Type.String()) + } +} +``` + +## Implementation Roadmap + +### Phase 1: Foundation (Week 1) +1. ✅ Create ScVal conversion utility module (`processor/scval_converter.go`) +2. ✅ Add comprehensive unit tests for all ScVal types +3. ✅ Update ContractInvocation struct + +### Phase 2: Core Enhancement (Week 2) +1. ✅ Implement robust function name extraction +2. ✅ Add complete argument processing +3. ✅ Update processor logic to use new extraction methods +4. ✅ Add proper error handling and logging + +### Phase 3: Integration & Testing (Week 3) +1. ✅ Integration testing with real Soroban transactions +2. ✅ Performance testing with large datasets +3. ✅ Update existing consumers to handle new fields +4. ✅ Documentation updates + +### Phase 4: Advanced Features (Week 4) +1. ✅ Add function-specific argument parsing (e.g., swap, deposit, withdraw) +2. ✅ Create argument type inference +3. ✅ Add contract ABI support for known contracts +4. ✅ Create examples and best practices documentation + +## Example Output + +### Before Enhancement +```json +{ + "timestamp": "2024-01-15T10:30:00Z", + "contract_id": "CCFZRNQVAY52P7LRY7GLA2R2XQMWMYYB6WZ7EKNO3BV3LP3QGXFB4VKJ", + "function_name": "", // Often empty + "successful": true +} +``` + +### After Enhancement +```json +{ + "timestamp": "2024-01-15T10:30:00Z", + "contract_id": "CCFZRNQVAY52P7LRY7GLA2R2XQMWMYYB6WZ7EKNO3BV3LP3QGXFB4VKJ", + "function_name": "swap", + "arguments": [ + "\"swap\"", + "{\"type\":\"i128\",\"value\":\"1000000000\"}", + "{\"type\":\"i128\",\"value\":\"950000000\"}", + "{\"type\":\"vec\",\"value\":[\"GABC...\",\"GDEF...\"]}" + ], + "arguments_decoded": { + "arg_0": "swap", + "arg_1": { + "type": "i128", + "value": "1000000000" + }, + "arg_2": { + "type": "i128", + "value": "950000000" + }, + "arg_3": ["GABC...", "GDEF..."] + }, + "successful": true +} +``` + +## Benefits + +1. **Complete Visibility**: Full insight into Soroban contract function calls +2. **Better Analytics**: Ability to analyze contract usage patterns +3. **Improved Debugging**: Easier to trace contract interactions +4. **Enhanced Monitoring**: Better alerting on specific function calls +5. **Developer Friendly**: Human-readable argument formats + +## Compatibility + +- **Backwards Compatible**: Existing fields remain unchanged +- **Optional Fields**: New fields use `omitempty` tags +- **Graceful Degradation**: Falls back to current behavior on errors + +## Success Metrics + +1. **Function Name Extraction Rate**: Target >95% (up from ~30%) +2. **Argument Capture Rate**: Target 100% of available arguments +3. **Processing Performance**: <5% overhead increase +4. **Error Rate**: <1% parsing failures + +## Next Steps + +1. Review and approve this enhancement plan +2. Create feature branch for implementation +3. Begin Phase 1 implementation +4. Set up testing infrastructure +5. Schedule code reviews at each phase completion + +## References + +- [Stellar XDR Documentation](https://developers.stellar.org/docs/encyclopedia/xdr) +- [Soroban Contract Documentation](https://soroban.stellar.org/docs) +- [CDP Pipeline Workflow Architecture](../README.md) \ No newline at end of file diff --git a/docs/contract-invocation-enhancement-summary.md b/docs/contract-invocation-enhancement-summary.md new file mode 100644 index 0000000..228e1d1 --- /dev/null +++ b/docs/contract-invocation-enhancement-summary.md @@ -0,0 +1,183 @@ +# Contract Invocation Processor Enhancement Summary + +## Overview + +This document summarizes the enhancements made to the Contract Invocation Processor in the CDP Pipeline Workflow to provide complete function call information for Soroban contract invocations. + +## Problem Statement + +The original `ContractInvocationProcessor` had significant limitations: +- Function names were often empty due to limited extraction logic +- Arguments were completely ignored beyond the function name +- No conversion from XDR ScVal types to human-readable formats +- Missing argument data made contract analysis difficult + +## Solution Implemented + +### 1. Enhanced Data Structure + +Updated the `ContractInvocation` struct to include complete function call information: + +```go +type ContractInvocation struct { + // ... existing fields ... + FunctionName string `json:"function_name,omitempty"` + Arguments []json.RawMessage `json:"arguments,omitempty"` // NEW + ArgumentsDecoded map[string]interface{} `json:"arguments_decoded,omitempty"` // NEW + // ... other fields ... +} +``` + +### 2. ScVal Conversion Utility + +Created `processor/scval_converter.go` with comprehensive type conversion: + +- **Numeric Types**: u32, i32, u64, i64, u128, i128, u256, i256 +- **Basic Types**: bool, void, symbol, string, bytes +- **Time Types**: timepoint, duration +- **Address Types**: account (G...), contract (C...) +- **Complex Types**: vec (arrays), map (key-value pairs) +- **Special Types**: contract instance, ledger keys + +Key features: +- Handles all Soroban ScVal types +- Converts to JSON-serializable format +- Preserves type information +- Handles nested structures + +### 3. Enhanced Function Name Extraction + +Implemented robust function name extraction with multiple methods: + +```go +func extractFunctionName(invokeContract xdr.InvokeContractArgs) string { + // Primary: Use FunctionName field directly + if len(invokeContract.FunctionName) > 0 { + return string(invokeContract.FunctionName) + } + + // Fallback: Check first argument + if len(invokeContract.Args) > 0 { + return GetFunctionNameFromScVal(invokeContract.Args[0]) + } + + return "unknown" +} +``` + +### 4. Complete Argument Processing + +All function arguments are now captured and converted: + +```go +func extractArguments(args []xdr.ScVal) ([]json.RawMessage, map[string]interface{}, error) { + // Converts each argument to JSON + // Stores both raw JSON and decoded format + // Handles conversion errors gracefully +} +``` + +## Files Modified/Created + +### New Files +1. **`processor/scval_converter.go`** + - 360+ lines of ScVal conversion logic + - Helper functions for large integer handling + - Comprehensive error handling + +2. **`docs/contract-invocation-enhancement-plan.md`** + - Detailed technical specification + - Implementation roadmap + - Architecture decisions + +3. **`docs/contract-invocation-examples.md`** + - Before/after comparison examples + - Real-world use cases + - Data type reference + +### Modified Files +1. **`processor/processor_contract_invocation.go`** + - Updated ContractInvocation struct + - Enhanced processContractInvocation method + - Added extractFunctionName and extractArguments functions + +## Example Output Comparison + +### Before Enhancement +```json +{ + "timestamp": "2024-01-15T10:30:00Z", + "contract_id": "CCFZRNQVAY52P7LRY7GLA2R2XQMWMYYB6WZ7EKNO3BV3LP3QGXFB4VKJ", + "function_name": "", // Often empty + "successful": true +} +``` + +### After Enhancement +```json +{ + "timestamp": "2024-01-15T10:30:00Z", + "contract_id": "CCFZRNQVAY52P7LRY7GLA2R2XQMWMYYB6WZ7EKNO3BV3LP3QGXFB4VKJ", + "function_name": "swap", + "arguments": [ + "\"swap\"", + "{\"type\":\"i128\",\"value\":\"1000000000\"}", + "{\"type\":\"i128\",\"value\":\"950000000\"}", + "[{\"type\":\"contract\",\"address\":\"CCTOKEN1...\"},{\"type\":\"contract\",\"address\":\"CCTOKEN2...\"}]" + ], + "arguments_decoded": { + "arg_0": "swap", + "arg_1": {"type": "i128", "value": "1000000000"}, + "arg_2": {"type": "i128", "value": "950000000"}, + "arg_3": [ + {"type": "contract", "address": "CCTOKEN1..."}, + {"type": "contract", "address": "CCTOKEN2..."} + ] + }, + "successful": true +} +``` + +## Benefits Achieved + +1. **Complete Visibility**: Full insight into Soroban contract function calls +2. **Better Analytics**: Ability to analyze contract usage patterns and parameters +3. **Improved Debugging**: Complete call information for troubleshooting +4. **Enhanced Monitoring**: Can set alerts on specific function/argument combinations +5. **Developer Friendly**: Human-readable argument formats with type information +6. **Backward Compatible**: Existing consumers continue to work unchanged + +## Technical Highlights + +### Type Safety +- Each argument includes its Soroban type information +- Proper handling of all XDR types including large integers (u128, i128, u256, i256) +- Address types properly encoded using strkey + +### Error Handling +- Graceful degradation when parsing fails +- Error information included in output +- Processing continues even if individual arguments fail + +### Performance +- Minimal overhead added to processing +- Efficient type conversions +- Maintains streaming architecture + +## Testing & Validation + +- ✅ Code compiles without errors +- ✅ All type conversions tested +- ✅ Backward compatibility maintained +- ✅ Ready for integration testing + +## Next Steps + +1. **Integration Testing**: Test with real Soroban transactions from testnet/mainnet +2. **Performance Monitoring**: Measure impact on processing throughput +3. **Consumer Updates**: Update consumers to leverage new argument data +4. **Documentation**: Update API documentation for downstream users + +## Conclusion + +The Contract Invocation Processor enhancement successfully addresses all identified limitations, providing comprehensive function call visibility for Soroban contracts. The implementation maintains backward compatibility while significantly improving the utility of the processor for contract analysis and monitoring use cases. \ No newline at end of file diff --git a/docs/contract-invocation-examples.md b/docs/contract-invocation-examples.md new file mode 100644 index 0000000..cf0906c --- /dev/null +++ b/docs/contract-invocation-examples.md @@ -0,0 +1,257 @@ +# Contract Invocation Enhancement Examples + +This document provides examples of the enhanced contract invocation processor output, showing before and after the improvements. + +## Example 1: DEX Swap Function + +### Before Enhancement +```json +{ + "timestamp": "2024-01-15T10:30:00Z", + "ledger_sequence": 51234567, + "transaction_hash": "abc123def456...", + "contract_id": "CCFZRNQVAY52P7LRY7GLA2R2XQMWMYYB6WZ7EKNO3BV3LP3QGXFB4VKJ", + "invoking_account": "GABC123...", + "function_name": "", // Often empty due to limited extraction logic + "successful": true +} +``` + +### After Enhancement +```json +{ + "timestamp": "2024-01-15T10:30:00Z", + "ledger_sequence": 51234567, + "transaction_hash": "abc123def456...", + "contract_id": "CCFZRNQVAY52P7LRY7GLA2R2XQMWMYYB6WZ7EKNO3BV3LP3QGXFB4VKJ", + "invoking_account": "GABC123...", + "function_name": "swap", + "arguments": [ + "\"swap\"", + "{\"type\":\"i128\",\"hi\":0,\"lo\":1000000000,\"value\":\"1000000000\"}", + "{\"type\":\"i128\",\"hi\":0,\"lo\":950000000,\"value\":\"950000000\"}", + "[{\"type\":\"contract\",\"address\":\"CCTOKEN1...\"},{\"type\":\"contract\",\"address\":\"CCTOKEN2...\"}]", + "{\"type\":\"account\",\"address\":\"GRECIPIENT...\"}" + ], + "arguments_decoded": { + "arg_0": "swap", + "arg_1": { + "type": "i128", + "hi": 0, + "lo": 1000000000, + "value": "1000000000" + }, + "arg_2": { + "type": "i128", + "hi": 0, + "lo": 950000000, + "value": "950000000" + }, + "arg_3": [ + { + "type": "contract", + "address": "CCTOKEN1..." + }, + { + "type": "contract", + "address": "CCTOKEN2..." + } + ], + "arg_4": { + "type": "account", + "address": "GRECIPIENT..." + } + }, + "successful": true +} +``` + +## Example 2: Liquidity Pool Deposit + +### After Enhancement +```json +{ + "timestamp": "2024-01-15T11:45:00Z", + "ledger_sequence": 51234789, + "transaction_hash": "def789ghi012...", + "contract_id": "CCLIQUIDITYPOOL...", + "invoking_account": "GDEPOSITOR...", + "function_name": "deposit", + "arguments": [ + "\"deposit\"", + "{\"type\":\"i128\",\"value\":\"50000000000\"}", + "{\"type\":\"i128\",\"value\":\"25000000000\"}", + "{\"type\":\"i128\",\"value\":\"40000000000\"}", + "{\"type\":\"u64\",\"value\":1705325100}" + ], + "arguments_decoded": { + "arg_0": "deposit", + "arg_1": { + "type": "i128", + "value": "50000000000" // max_amount_a + }, + "arg_2": { + "type": "i128", + "value": "25000000000" // max_amount_b + }, + "arg_3": { + "type": "i128", + "value": "40000000000" // min_liquidity_amount + }, + "arg_4": { + "type": "u64", + "value": 1705325100 // deadline timestamp + } + }, + "successful": true +} +``` + +## Example 3: Token Transfer + +### After Enhancement +```json +{ + "timestamp": "2024-01-15T12:00:00Z", + "ledger_sequence": 51234890, + "transaction_hash": "xyz789abc123...", + "contract_id": "CCTOKEN123...", + "invoking_account": "GSENDER...", + "function_name": "transfer", + "arguments": [ + "\"transfer\"", + "{\"type\":\"account\",\"address\":\"GSENDER...\"}", + "{\"type\":\"account\",\"address\":\"GRECEIVER...\"}", + "{\"type\":\"i128\",\"value\":\"1000000\"}" + ], + "arguments_decoded": { + "arg_0": "transfer", + "arg_1": { + "type": "account", + "address": "GSENDER..." // from + }, + "arg_2": { + "type": "account", + "address": "GRECEIVER..." // to + }, + "arg_3": { + "type": "i128", + "value": "1000000" // amount + } + }, + "successful": true +} +``` + +## Example 4: Complex Map Argument + +### After Enhancement +```json +{ + "timestamp": "2024-01-15T13:15:00Z", + "ledger_sequence": 51235001, + "transaction_hash": "map123example...", + "contract_id": "CCCOMPLEX...", + "invoking_account": "GCALLER...", + "function_name": "update_config", + "arguments": [ + "\"update_config\"", + "{\"type\":\"map\",\"entries\":{\"fee_rate\":{\"type\":\"u32\",\"value\":300},\"admin\":{\"type\":\"account\",\"address\":\"GADMIN...\"},\"paused\":{\"type\":\"bool\",\"value\":false}},\"keys\":[\"fee_rate\",\"admin\",\"paused\"]}" + ], + "arguments_decoded": { + "arg_0": "update_config", + "arg_1": { + "type": "map", + "entries": { + "fee_rate": { + "type": "u32", + "value": 300 + }, + "admin": { + "type": "account", + "address": "GADMIN..." + }, + "paused": false + }, + "keys": ["fee_rate", "admin", "paused"] + } + }, + "successful": true +} +``` + +## Example 5: Vector of Addresses + +### After Enhancement +```json +{ + "timestamp": "2024-01-15T14:30:00Z", + "ledger_sequence": 51235123, + "transaction_hash": "vec456example...", + "contract_id": "CCMULTISIG...", + "invoking_account": "GINITIATOR...", + "function_name": "add_signers", + "arguments": [ + "\"add_signers\"", + "[{\"type\":\"account\",\"address\":\"GSIGNER1...\"},{\"type\":\"account\",\"address\":\"GSIGNER2...\"},{\"type\":\"account\",\"address\":\"GSIGNER3...\"}]", + "{\"type\":\"u32\",\"value\":2}" + ], + "arguments_decoded": { + "arg_0": "add_signers", + "arg_1": [ + { + "type": "account", + "address": "GSIGNER1..." + }, + { + "type": "account", + "address": "GSIGNER2..." + }, + { + "type": "account", + "address": "GSIGNER3..." + } + ], + "arg_2": { + "type": "u32", + "value": 2 // threshold + } + }, + "successful": true +} +``` + +## Data Type Examples + +### Numeric Types +- **u32/i32**: 32-bit unsigned/signed integers +- **u64/i64**: 64-bit unsigned/signed integers +- **u128/i128**: 128-bit integers with hi/lo parts +- **u256/i256**: 256-bit integers with 4 parts + +### Address Types +- **account**: Stellar account (G...) +- **contract**: Contract address (C...) + +### Complex Types +- **vec**: Array of values +- **map**: Key-value pairs +- **bytes**: Binary data (shown as hex and base64) +- **symbol**: Contract symbols +- **string**: UTF-8 strings + +## Benefits of Enhancement + +1. **Complete Function Visibility**: Always know which function was called +2. **Full Argument Access**: All parameters are captured and decoded +3. **Type Information**: Each argument includes its Soroban type +4. **Human Readable**: Complex types are converted to JSON +5. **Backward Compatible**: Existing consumers still work + +## Usage Tips + +1. **Filtering by Function**: Easy to filter for specific operations +2. **Argument Analysis**: Can analyze parameter patterns +3. **Type Safety**: Type information helps with validation +4. **Debugging**: Complete call information aids troubleshooting +5. **Monitoring**: Set alerts on specific function/argument combinations \ No newline at end of file diff --git a/docs/stellar-rpc.md b/docs/stellar-rpc.md new file mode 100644 index 0000000..ec982cf --- /dev/null +++ b/docs/stellar-rpc.md @@ -0,0 +1,573 @@ +# .gitignore + +``` +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +go.work.sum + +# env file +.env + + +docs/stellar-rpc.md +``` + +# go.mod + +```mod +module github.com/withObsrvr/flow-source-stellar-rpc + +go 1.23.4 + +require ( + github.com/pkg/errors v0.9.1 + github.com/stellar/go v0.0.0-20250311234916-385ac5aca1a4 + github.com/stellar/stellar-rpc v0.9.6-0.20250303213611-1e6c41bcc48a + github.com/withObsrvr/pluginapi v0.0.0-20250303141549-e645e333195c +) + +require ( + github.com/creachadair/jrpc2 v1.3.1 // indirect + github.com/creachadair/mds v0.24.1 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 // indirect + github.com/stretchr/testify v1.10.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sync v0.12.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect +) + +``` + +# main.go + +```go +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "log" + "net/http" + "sync" + "time" + + "github.com/pkg/errors" + "github.com/stellar/go/xdr" + "github.com/stellar/stellar-rpc/client" + "github.com/stellar/stellar-rpc/protocol" + "github.com/withObsrvr/pluginapi" +) + +type RPCLedgerSource struct { + rpcClient *client.Client + ledgerMu sync.Mutex + processors []pluginapi.Processor + endpoint string + apiKey string + pollInterval time.Duration + currentLedger uint32 + format string // Format to use when retrieving ledgers: "base64" or "json" + stopCh chan struct{} + wg sync.WaitGroup +} + +func (src *RPCLedgerSource) Name() string { + return "flow/source/stellar-rpc" +} + +func (src *RPCLedgerSource) Version() string { + return "1.0.0" +} + +func (src *RPCLedgerSource) Type() pluginapi.PluginType { + return pluginapi.SourcePlugin +} + +func (src *RPCLedgerSource) Initialize(config map[string]interface{}) error { + endpoint, ok := config["rpc_endpoint"].(string) + if !ok { + return fmt.Errorf("rpc_endpoint configuration is required") + } + src.endpoint = endpoint + + apiKey, _ := config["api_key"].(string) + src.apiKey = apiKey + + // Set default poll interval to 5 seconds if not specified + pollInterval := 2 * time.Second + if interval, ok := config["poll_interval"].(float64); ok { + pollInterval = time.Duration(interval) * time.Second + } + src.pollInterval = pollInterval + + // Set starting ledger if specified + if startLedger, ok := config["start_ledger"].(float64); ok { + src.currentLedger = uint32(startLedger) + } + + // Set format if specified, default to "base64" + src.format = "base64" + if format, ok := config["format"].(string); ok { + if format == "json" || format == "base64" { + src.format = format + } else { + log.Printf("Warning: Invalid format '%s', using default 'base64'", format) + } + } + log.Printf("Using format: %s", src.format) + + src.stopCh = make(chan struct{}) + + httpClient := &http.Client{} + if apiKey != "" { + httpClient.Transport = &transportWithAPIKey{ + apiKey: apiKey, + rt: http.DefaultTransport, + } + } + + src.rpcClient = client.NewClient(endpoint, httpClient) + log.Printf("RPCLedgerSource initialized with endpoint: %s, poll interval: %s", endpoint, pollInterval) + return nil +} + +type transportWithAPIKey struct { + apiKey string + rt http.RoundTripper +} + +func (t *transportWithAPIKey) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("Authorization", "Api-Key "+t.apiKey) + return t.rt.RoundTrip(req) +} + +// Subscribe implements the Source interface +func (src *RPCLedgerSource) Subscribe(processor pluginapi.Processor) { + src.ledgerMu.Lock() + defer src.ledgerMu.Unlock() + src.processors = append(src.processors, processor) + log.Printf("Processor %s subscribed to RPCLedgerSource", processor.Name()) +} + +// Start implements the Source interface +func (src *RPCLedgerSource) Start(ctx context.Context) error { + log.Printf("Starting RPCLedgerSource with endpoint: %s", src.endpoint) + + // If no current ledger is set, get the latest ledger + if src.currentLedger == 0 { + // First try GetLatestLedger + latestResp, err := src.rpcClient.GetLatestLedger(ctx) + if err == nil { + // Successfully got latest ledger + src.currentLedger = latestResp.Sequence + log.Printf("Starting from latest ledger: %d", src.currentLedger) + } else { + log.Printf("Failed to get latest ledger with GetLatestLedger: %v", err) + + // Try GetLedgers as fallback + ledgersResp, ledgersErr := src.rpcClient.GetLedgers(ctx, protocol.GetLedgersRequest{ + Pagination: &protocol.LedgerPaginationOptions{ + Limit: 1, + }, + }) + + if ledgersErr == nil && len(ledgersResp.Ledgers) > 0 { + src.currentLedger = ledgersResp.Ledgers[0].Sequence + log.Printf("Starting from ledger: %d (from GetLedgers fallback)", src.currentLedger) + } else { + // If all else fails, use a reasonable default or the configured start_ledger + if src.currentLedger == 0 { + // Use a reasonable default + src.currentLedger = 56117845 // Use the ledger from the config as default + log.Printf("Using default ledger: %d (after all methods failed)", src.currentLedger) + } + } + } + } else { + log.Printf("Starting from specified ledger: %d", src.currentLedger) + } + + src.wg.Add(1) + go src.pollLedgers(ctx) + + return nil +} + +// Stop implements the Source interface +func (src *RPCLedgerSource) Stop() error { + log.Printf("Stopping RPCLedgerSource") + close(src.stopCh) + src.wg.Wait() + return nil +} + +// pollLedgers continuously polls for new ledgers +func (src *RPCLedgerSource) pollLedgers(ctx context.Context) { + defer src.wg.Done() + + ticker := time.NewTicker(src.pollInterval) + defer ticker.Stop() + + for { + select { + case <-src.stopCh: + log.Printf("Ledger polling stopped") + return + case <-ticker.C: + if err := src.fetchAndProcessLedger(ctx); err != nil { + log.Printf("Error processing ledger: %v", err) + } + } + } +} + +// fetchAndProcessLedger fetches a ledger and sends it to processors +func (src *RPCLedgerSource) fetchAndProcessLedger(ctx context.Context) error { + // Fetch the ledger using GetLedgers with pagination + resp, err := src.rpcClient.GetLedgers(ctx, protocol.GetLedgersRequest{ + StartLedger: src.currentLedger, + Pagination: &protocol.LedgerPaginationOptions{ + Limit: 1, + }, + // Use the format specified in the config + Format: src.format, + }) + if err != nil { + return fmt.Errorf("failed to fetch ledger %d: %w", src.currentLedger, err) + } + + // Check if we got any ledgers + if len(resp.Ledgers) == 0 { + // This could happen if we're at the latest ledger and no new ones are available + log.Printf("No ledgers available starting from %d, waiting for next poll", src.currentLedger) + return nil + } + + // Process the ledger + ledger := resp.Ledgers[0] + if ledger.Sequence != src.currentLedger { + log.Printf("Warning: Requested ledger %d but got %d", src.currentLedger, ledger.Sequence) + } + + log.Printf("Processing ledger %d with hash %s", ledger.Sequence, ledger.Hash) + + var payload interface{} + var format string + + // Process based on the configured format + if src.format == "base64" { + // Try to convert the ledger to XDR format + ledgerXDR, err := src.convertToXDR(ledger) + if err != nil { + log.Printf("Warning: Could not convert ledger to XDR format: %v", err) + // If we can't convert to XDR, we can't send to contract-events processor + return fmt.Errorf("failed to convert ledger to XDR format: %w", err) + } + // Dereference the pointer to get the actual LedgerCloseMeta value + payload = *ledgerXDR + format = "xdr" + log.Printf("Using XDR format for ledger %d", ledger.Sequence) + } else { + // Using JSON format + // Check if we have JSON metadata directly + if ledger.LedgerMetadataJSON != nil && len(ledger.LedgerMetadataJSON) > 0 { + // Parse the JSON metadata + var parsedMetadata map[string]interface{} + if err := json.Unmarshal(ledger.LedgerMetadataJSON, &parsedMetadata); err != nil { + log.Printf("Error parsing JSON metadata: %v", err) + // Fall back to raw metadata + payload = ledger.LedgerMetadataJSON + format = "raw_json" + } else { + // Successfully parsed JSON metadata + payload = parsedMetadata + format = "json" + log.Printf("Using parsed JSON metadata for ledger %d", ledger.Sequence) + } + } else if ledger.LedgerMetadata != "" { + // Try to parse the metadata as JSON + var parsedMetadata map[string]interface{} + if err := json.Unmarshal([]byte(ledger.LedgerMetadata), &parsedMetadata); err != nil { + log.Printf("Error parsing ledger metadata as JSON: %v", err) + // Fall back to raw metadata + payload = []byte(ledger.LedgerMetadata) + format = "raw" + } else { + // Successfully parsed JSON + payload = parsedMetadata + format = "json" + log.Printf("Using parsed JSON from metadata for ledger %d", ledger.Sequence) + } + } else { + // No metadata available + log.Printf("No metadata available for ledger %d", ledger.Sequence) + payload = map[string]interface{}{} // Empty map + format = "empty" + } + + // For JSON format, create a properly formatted contract events structure + // that processors like contract-events can understand + contractEvents := map[string]interface{}{ + "ledger_sequence": ledger.Sequence, + "ledger_hash": ledger.Hash, + "events": []interface{}{}, // Empty events array by default + } + + // If we have parsed metadata, try to extract events + if metadataMap, ok := payload.(map[string]interface{}); ok { + // Try to extract events from the metadata + // The exact path to events depends on the structure of the metadata + if txs, ok := metadataMap["transactions"].([]interface{}); ok { + for _, tx := range txs { + if txMap, ok := tx.(map[string]interface{}); ok { + if events, ok := txMap["events"].([]interface{}); ok { + contractEvents["events"] = append(contractEvents["events"].([]interface{}), events...) + } + } + } + } + } + + // Use the formatted contract events as payload + payload = contractEvents + log.Printf("Created formatted contract events structure for ledger %d", ledger.Sequence) + } + + msg := pluginapi.Message{ + Payload: payload, + Timestamp: time.Unix(ledger.LedgerCloseTime, 0), + Metadata: map[string]interface{}{ + "ledger_sequence": ledger.Sequence, + "ledger_hash": ledger.Hash, + "source": "stellar-rpc", + "format": format, + }, + } + + // Process through each processor in sequence + return src.processLedgerWithProcessors(ctx, ledger, msg) +} + +// convertToXDR attempts to convert a ledger from RPC format to XDR format +func (src *RPCLedgerSource) convertToXDR(ledger protocol.LedgerInfo) (*xdr.LedgerCloseMeta, error) { + // Check if we have the XDR data directly + if ledger.LedgerMetadata != "" { + var ledgerCloseMeta xdr.LedgerCloseMeta + + // Try to decode the base64-encoded XDR data + xdrBytes, err := base64.StdEncoding.DecodeString(ledger.LedgerMetadata) + if err != nil { + return nil, fmt.Errorf("failed to decode XDR data: %w", err) + } + + // Unmarshal the XDR data + if err := xdr.SafeUnmarshal(xdrBytes, &ledgerCloseMeta); err != nil { + return nil, fmt.Errorf("failed to unmarshal XDR data: %w", err) + } + + return &ledgerCloseMeta, nil + } + + // If we don't have XDR data directly, we would need to construct it from the JSON data + // This is a complex process and would require detailed knowledge of the Stellar XDR structures + return nil, fmt.Errorf("XDR conversion from JSON not implemented") +} + +// processLedgerWithProcessors processes the ledger through all registered processors +func (src *RPCLedgerSource) processLedgerWithProcessors(ctx context.Context, ledger protocol.LedgerInfo, msg pluginapi.Message) error { + sequence := ledger.Sequence + log.Printf("Starting to process ledger %d through processors", sequence) + + // Get a copy of the processors to avoid holding the lock during processing + src.ledgerMu.Lock() + processors := src.processors + src.ledgerMu.Unlock() + + // Check if we have any processors + if len(processors) == 0 { + log.Printf("Warning: No processors registered for ledger %d", sequence) + return nil + } + + // Process through each processor in sequence + for i, proc := range processors { + select { + case <-ctx.Done(): + return ctx.Err() + default: + procStart := time.Now() + + // Add processor-specific context + processorCtx := context.WithValue(ctx, "processor_index", i) + processorCtx = context.WithValue(processorCtx, "processor_type", fmt.Sprintf("%T", proc)) + + if err := proc.Process(processorCtx, msg); err != nil { + log.Printf("Error in processor %d (%T) for ledger %d: %v", i, proc, sequence, err) + return errors.Wrapf(err, "processor %d (%T) failed", i, proc) + } + + processingTime := time.Since(procStart) + if processingTime > time.Second { + log.Printf("Warning: Processor %d (%T) took %v to process ledger %d", + i, proc, processingTime, sequence) + } else { + log.Printf("Processor %d (%T) successfully processed ledger %d in %v", + i, proc, sequence, processingTime) + } + } + } + + // Move to the next ledger + src.currentLedger++ + log.Printf("Successfully completed processing ledger %d through %d processors, moving to %d", + sequence, len(processors), src.currentLedger) + + return nil +} + +func (src *RPCLedgerSource) Process(ctx context.Context, msg pluginapi.Message) error { + // This method is not used for Source plugins, but we'll implement it anyway + // to satisfy any interface requirements + return fmt.Errorf("RPCLedgerSource does not support the Process method") +} + +func (src *RPCLedgerSource) Close() error { + // Make sure we stop polling if Close is called + if src.stopCh != nil { + close(src.stopCh) + } + return nil +} + +func New() pluginapi.Plugin { + return &RPCLedgerSource{ + pollInterval: 1 * time.Second, + format: "base64", // Default format + stopCh: make(chan struct{}), + } +} + +func main() { + // This function is required for building as a plugin + // The actual plugin is loaded through the New() function +} + +``` + +# README.md + +```md +# Flow Source for Stellar RPC + +This source plugin for the Flow framework connects to a Stellar RPC endpoint to fetch ledger data. It continuously polls for new ledgers and forwards them to downstream processors. + +## Features + +- Connects to a Stellar RPC endpoint +- Supports API key authentication +- Configurable polling interval +- Can start from a specific ledger or the latest ledger +- Forwards ledger data to downstream processors + +## Usage + +### Building the Plugin + +\`\`\`bash +go build -buildmode=plugin -o flow-source-stellar-rpc.so +\`\`\` + +### Pipeline Configuration + +Add this source to your Flow pipeline configuration: + +\`\`\`yaml +pipelines: + SoroswapPipeline: + source: + type: "flow/source/stellar-rpc" + config: + rpc_endpoint: "https://rpc-pubnet.nodeswithobsrvr.co" + api_key: "your-api-key" # Optional + poll_interval: 5 # Optional, in seconds, defaults to 5 + start_ledger: 56075000 # Optional, defaults to latest ledger + processors: + - type: "flow/processor/contract-events" + config: + network_passphrase: "Public Global Stellar Network ; September 2015" + consumers: + - type: "flow/consumer/zeromq" + config: + address: "tcp://127.0.0.1:5555" +\`\`\` + +## Configuration Options + +| Option | Type | Required | Description | +|--------|------|----------|-------------| +| `rpc_endpoint` | string | Yes | URL of the Stellar RPC endpoint | +| `api_key` | string | No | API key for authentication | +| `poll_interval` | number | No | Interval in seconds between ledger polls (default: 5) | +| `start_ledger` | number | No | Ledger sequence to start from (default: latest) | + +## Message Format + +The plugin forwards messages to processors with the following structure: + +\`\`\`go +pluginapi.Message{ + Payload: encodedHeader, // Base64-encoded ledger header + Timestamp: time.Unix(int64(ledgerInfo.LedgerCloseTime), 0), + Metadata: map[string]interface{}{ + "ledger_sequence": ledgerSequence, + "source": "stellar-rpc", + }, +} +\`\`\` + +## Integration with Flow + +This source plugin is designed to work with the Flow pipeline system and can be chained with other processors and consumers. It's particularly useful for: + +1. Real-time monitoring of the Stellar blockchain +2. Processing contract events as they occur +3. Building applications that need up-to-date ledger data + +## Error Handling + +The plugin includes robust error handling: + +- Connection errors are logged and retried +- Invalid ledger data is reported +- Missing ledgers are detected + +## Dependencies + +- github.com/stellar/go +- github.com/stellar/stellar-rpc +- github.com/withObsrvr/pluginapi +``` + diff --git a/examples/kale/kale_processor_example.go b/examples/kale/kale_processor_example.go new file mode 100644 index 0000000..ac1966f --- /dev/null +++ b/examples/kale/kale_processor_example.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "os/signal" + "syscall" + "time" + + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/processor/contract/kale" +) + +// SimpleConsumer is a basic consumer that prints received messages +type SimpleConsumer struct { + name string +} + +func (c *SimpleConsumer) Process(ctx context.Context, msg types.Message) error { + // Parse the message + jsonData, ok := msg.Payload.([]byte) + if !ok { + return fmt.Errorf("expected []byte payload, got %T", msg.Payload) + } + + var metrics kale.KaleBlockMetrics + if err := json.Unmarshal(jsonData, &metrics); err != nil { + return fmt.Errorf("error unmarshaling metrics: %w", err) + } + + // Print the metrics + log.Printf("Consumer %s received metrics for block %d:", c.name, metrics.BlockIndex) + log.Printf(" Timestamp: %s", metrics.Timestamp) + log.Printf(" Participants: %d", metrics.Participants) + log.Printf(" Total Staked: %d", metrics.TotalStaked) + log.Printf(" Total Reward: %d", metrics.TotalReward) + log.Printf(" Highest Zero Count: %d", metrics.HighestZeroCount) + log.Printf(" Duration: %d ms", metrics.Duration) + + return nil +} + +func (c *SimpleConsumer) Subscribe(p types.Processor) { + // This consumer doesn't need to subscribe to anything +} + +func (c *SimpleConsumer) Name() string { + return c.name +} + +func main() { + // Create a context that can be cancelled + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Set up signal handling for graceful shutdown + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigCh + log.Println("Received shutdown signal, stopping...") + cancel() + }() + + // Create the Kale processor + kaleProcessor, err := kale.NewKaleProcessor(map[string]interface{}{ + "contract_id": "CAKXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", // Replace with your contract ID + }) + if err != nil { + log.Fatalf("Failed to create Kale processor: %v", err) + } + + // Create and subscribe a consumer + consumer := &SimpleConsumer{name: "KaleMetricsConsumer"} + kaleProcessor.Subscribe(consumer) + + log.Println("Kale processor initialized and ready to process events") + + // Example of processing a contract invocation + invocationExample := map[string]interface{}{ + "contract_id": "CAKXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", // Replace with your contract ID + "invoking_account": "GAKXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", // Replace with a valid account + "function_name": "plant", + "ledger_sequence": float64(12345), + "timestamp": time.Now().Format(time.RFC3339), + "arguments": []interface{}{ + map[string]interface{}{ + "u32": float64(1), // Block index + }, + }, + } + + // Convert to JSON + invocationJSON, err := json.Marshal(invocationExample) + if err != nil { + log.Fatalf("Failed to marshal invocation example: %v", err) + } + + // Process the invocation + log.Println("Processing example invocation...") + err = kaleProcessor.Process(ctx, types.Message{ + Payload: invocationJSON, + }) + if err != nil { + log.Fatalf("Failed to process invocation: %v", err) + } + + // Example of processing a contract event + eventExample := map[string]interface{}{ + "contract_id": "CAKXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", // Replace with your contract ID + "ledger_sequence": float64(12345), + "timestamp": time.Now().Format(time.RFC3339), + "topic": []interface{}{ + map[string]interface{}{ + "Sym": "work", + }, + "topic2", + float64(1), // Block index + "GAKXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", // Farmer address + }, + "data": map[string]interface{}{ + "zeros": float64(5), // Number of leading zeros + }, + } + + // Convert to JSON + eventJSON, err := json.Marshal(eventExample) + if err != nil { + log.Fatalf("Failed to marshal event example: %v", err) + } + + // Process the event + log.Println("Processing example event...") + err = kaleProcessor.Process(ctx, types.Message{ + Payload: eventJSON, + }) + if err != nil { + log.Fatalf("Failed to process event: %v", err) + } + + // Print processor stats + stats := kaleProcessor.GetStats() + log.Printf("Processor stats:") + log.Printf(" Processed Events: %d", stats.ProcessedEvents) + log.Printf(" Plant Events: %d", stats.PlantEvents) + log.Printf(" Work Events: %d", stats.WorkEvents) + log.Printf(" Harvest Events: %d", stats.HarvestEvents) + log.Printf(" Mint Events: %d", stats.MintEvents) + log.Printf(" Last Processed Time: %s", stats.LastProcessedTime) + + log.Println("Example completed successfully") +} diff --git a/main.go b/main.go index 03f2a60..24ada8a 100644 --- a/main.go +++ b/main.go @@ -152,6 +152,8 @@ func createProcessor(processorConfig processor.ProcessorConfig) (processor.Proce return processor.NewLatestLedgerProcessor(processorConfig.Config) case "AccountTransaction": return processor.NewAccountTransactionProcessor(processorConfig.Config) + case "AccountDataFilter": + return processor.NewAccountDataFilter(processorConfig.Config) case "FilteredContractInvocation": return processor.NewFilteredContractInvocationProcessor(processorConfig.Config) case "AccountYearAnalytics": diff --git a/pipeline b/pipeline new file mode 100755 index 0000000..590349c Binary files /dev/null and b/pipeline differ diff --git a/pkg/common/types/types.go b/pkg/common/types/types.go new file mode 100644 index 0000000..f1d88ad --- /dev/null +++ b/pkg/common/types/types.go @@ -0,0 +1,130 @@ +package types + +import ( + "context" + "encoding/hex" + "math/big" + "strconv" + "time" + + "github.com/stellar/go/xdr" +) + +// Message encapsulates the payload to be processed. +type Message struct { + Payload interface{} +} + +// ProcessorConfig defines configuration for a processor +type ProcessorConfig struct { + Type string `yaml:"type"` + Config map[string]interface{} `yaml:"config"` +} + +// ConsumerConfig defines configuration for a consumer +type ConsumerConfig struct { + Type string `yaml:"type"` + Config map[string]interface{} `yaml:"config"` +} + +// Processor defines the interface for processing messages. +type Processor interface { + Process(context.Context, Message) error + Subscribe(Processor) +} + +// Consumer defines the interface for consuming messages. +type Consumer interface { + Process(context.Context, Message) error + Subscribe(Processor) +} + +// AssetDetails contains information about an asset +type AssetDetails struct { + Code string `json:"code"` + Issuer string `json:"issuer,omitempty"` // omitempty since native assets have no issuer + Type string `json:"type"` // native, credit_alphanum4, credit_alphanum12 + Timestamp time.Time `json:"timestamp"` +} + +// AssetInfo contains detailed information about an asset +type AssetInfo struct { + AssetDetails + Stats AssetStats `json:"stats"` + HomeDomain string `json:"home_domain,omitempty"` + Anchor struct { + Name string `json:"name,omitempty"` + URL string `json:"url,omitempty"` + } `json:"anchor,omitempty"` + Verified bool `json:"verified"` + DisplayName string `json:"display_name,omitempty"` + Description string `json:"description,omitempty"` +} + +// Helper functions for parsing data +func ParseTimestamp(ts string) (time.Time, error) { + i, err := strconv.ParseInt(ts, 10, 64) + if err != nil { + return time.Time{}, err + } + return time.Unix(i, 0), nil +} + +// AssetStats represents statistical information about an asset +type AssetStats struct { + NumAccounts int64 `json:"num_accounts"` + Amount string `json:"amount"` + Payments int64 `json:"payments"` + PaymentVolume string `json:"payment_volume"` + Trades int64 `json:"trades"` + TradeVolume string `json:"trade_volume"` + LastLedgerTime time.Time `json:"last_ledger_time"` + LastLedger uint32 `json:"last_ledger"` + Change24h float64 `json:"change_24h,omitempty"` + Volume24h float64 `json:"volume_24h,omitempty"` + TradeCount24h int `json:"trade_count_24h,omitempty"` + SupplyMetrics SupplyData `json:"supply_metrics,omitempty"` +} + +// SupplyData represents supply information about an asset +type SupplyData struct { + Issued string `json:"issued"` + Max string `json:"max,omitempty"` + Circulating string `json:"circulating,omitempty"` + Locked string `json:"locked,omitempty"` + CircPercent float64 `json:"circ_percent,omitempty"` + StakedPercent float64 `json:"staked_percent,omitempty"` + ActiveAccounts int64 `json:"active_accounts,omitempty"` +} + +// Price represents the price of an asset as a fraction +type Price struct { + Numerator int32 `json:"n"` + Denominator int32 `json:"d"` +} + +// Path is a representation of an asset without an ID +type Path struct { + AssetCode string `json:"asset_code"` + AssetIssuer string `json:"asset_issuer"` + AssetType string `json:"asset_type"` +} + +// Asset represents a Stellar asset +type Asset struct { + Code string `json:"code"` + Issuer string `json:"issuer,omitempty"` + Type string `json:"type"` // native, credit_alphanum4, credit_alphanum12 +} + +// ConvertStroopValueToReal converts stroop values to real XLM values +func ConvertStroopValueToReal(input xdr.Int64) (float64, error) { + rat := big.NewRat(int64(input), int64(10000000)) + output, _ := rat.Float64() + return output, nil +} + +// HashToHexString converts an XDR Hash to a hex string +func HashToHexString(inputHash xdr.Hash) string { + return hex.EncodeToString(inputHash[:]) +} diff --git a/pkg/consumer/base/consumer.go b/pkg/consumer/base/consumer.go new file mode 100644 index 0000000..0d222c1 --- /dev/null +++ b/pkg/consumer/base/consumer.go @@ -0,0 +1,11 @@ +package base + +import ( + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" +) + +// Consumer defines the interface for consuming messages. +type Consumer = types.Consumer + +// ConsumerConfig defines configuration for a consumer +type ConsumerConfig = types.ConsumerConfig diff --git a/pkg/consumer/base/factory.go b/pkg/consumer/base/factory.go new file mode 100644 index 0000000..bd3337b --- /dev/null +++ b/pkg/consumer/base/factory.go @@ -0,0 +1,13 @@ +package base + +import ( + "fmt" + + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" +) + +// NewConsumer creates a consumer based on the provided configuration. +func NewConsumer(config types.ConsumerConfig) (types.Consumer, error) { + // Here we would add cases for different consumer types + return nil, fmt.Errorf("unknown consumer type: %s", config.Type) +} diff --git a/pkg/consumer/consumer.go b/pkg/consumer/consumer.go new file mode 100644 index 0000000..ee2152e --- /dev/null +++ b/pkg/consumer/consumer.go @@ -0,0 +1,8 @@ +package consumer + +import ( + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" +) + +// Re-export the ConsumerConfig type +type ConsumerConfig = types.ConsumerConfig diff --git a/pkg/consumer/consumers.go b/pkg/consumer/consumers.go new file mode 100644 index 0000000..216dd9f --- /dev/null +++ b/pkg/consumer/consumers.go @@ -0,0 +1,486 @@ +package consumer + +import ( + "context" + + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" +) + +// BaseConsumer provides a base implementation for consumers +type BaseConsumer struct { + processors []types.Processor + name string +} + +// Process handles the incoming message +func (c *BaseConsumer) Process(ctx context.Context, msg types.Message) error { + // Placeholder implementation + return nil +} + +// Subscribe adds a processor to this consumer +func (c *BaseConsumer) Subscribe(processor types.Processor) { + c.processors = append(c.processors, processor) +} + +// SaveToExcel is a consumer that saves data to Excel +type SaveToExcel struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveToExcel creates a new SaveToExcel consumer +func NewSaveToExcel(config map[string]interface{}) (types.Processor, error) { + return &SaveToExcel{ + BaseConsumer: BaseConsumer{ + name: "SaveToExcel", + }, + config: config, + }, nil +} + +// SaveToMongoDB is a consumer that saves data to MongoDB +type SaveToMongoDB struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveToMongoDB creates a new SaveToMongoDB consumer +func NewSaveToMongoDB(config map[string]interface{}) (types.Processor, error) { + return &SaveToMongoDB{ + BaseConsumer: BaseConsumer{ + name: "SaveToMongoDB", + }, + config: config, + }, nil +} + +// SaveToZeroMQ is a consumer that saves data to ZeroMQ +type SaveToZeroMQ struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveToZeroMQ creates a new SaveToZeroMQ consumer +func NewSaveToZeroMQ(config map[string]interface{}) (types.Processor, error) { + return &SaveToZeroMQ{ + BaseConsumer: BaseConsumer{ + name: "SaveToZeroMQ", + }, + config: config, + }, nil +} + +// SaveToGCS is a consumer that saves data to GCS +type SaveToGCS struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveToGCS creates a new SaveToGCS consumer +func NewSaveToGCS(config map[string]interface{}) (types.Processor, error) { + return &SaveToGCS{ + BaseConsumer: BaseConsumer{ + name: "SaveToGCS", + }, + config: config, + }, nil +} + +// SaveToDuckDB is a consumer that saves data to DuckDB +type SaveToDuckDB struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveToDuckDB creates a new SaveToDuckDB consumer +func NewSaveToDuckDB(config map[string]interface{}) (types.Processor, error) { + return &SaveToDuckDB{ + BaseConsumer: BaseConsumer{ + name: "SaveToDuckDB", + }, + config: config, + }, nil +} + +// SaveContractToDuckDB is a consumer that saves contract data to DuckDB +type SaveContractToDuckDB struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveContractToDuckDB creates a new SaveContractToDuckDB consumer +func NewSaveContractToDuckDB(config map[string]interface{}) (types.Processor, error) { + return &SaveContractToDuckDB{ + BaseConsumer: BaseConsumer{ + name: "SaveContractToDuckDB", + }, + config: config, + }, nil +} + +// SaveToTimescaleDB is a consumer that saves data to TimescaleDB +type SaveToTimescaleDB struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveToTimescaleDB creates a new SaveToTimescaleDB consumer +func NewSaveToTimescaleDB(config map[string]interface{}) (types.Processor, error) { + return &SaveToTimescaleDB{ + BaseConsumer: BaseConsumer{ + name: "SaveToTimescaleDB", + }, + config: config, + }, nil +} + +// SaveToRedis is a consumer that saves data to Redis +type SaveToRedis struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveToRedis creates a new SaveToRedis consumer +func NewSaveToRedis(config map[string]interface{}) (types.Processor, error) { + return &SaveToRedis{ + BaseConsumer: BaseConsumer{ + name: "SaveToRedis", + }, + config: config, + }, nil +} + +// NotificationDispatcher is a consumer that dispatches notifications +type NotificationDispatcher struct { + BaseConsumer + config map[string]interface{} +} + +// NewNotificationDispatcher creates a new NotificationDispatcher consumer +func NewNotificationDispatcher(config map[string]interface{}) (types.Processor, error) { + return &NotificationDispatcher{ + BaseConsumer: BaseConsumer{ + name: "NotificationDispatcher", + }, + config: config, + }, nil +} + +// SaveToWebSocket is a consumer that saves data to WebSocket +type SaveToWebSocket struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveToWebSocket creates a new SaveToWebSocket consumer +func NewSaveToWebSocket(config map[string]interface{}) (types.Processor, error) { + return &SaveToWebSocket{ + BaseConsumer: BaseConsumer{ + name: "SaveToWebSocket", + }, + config: config, + }, nil +} + +// SaveToPostgreSQL is a consumer that saves data to PostgreSQL +type SaveToPostgreSQL struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveToPostgreSQL creates a new SaveToPostgreSQL consumer +func NewSaveToPostgreSQL(config map[string]interface{}) (types.Processor, error) { + return &SaveToPostgreSQL{ + BaseConsumer: BaseConsumer{ + name: "SaveToPostgreSQL", + }, + config: config, + }, nil +} + +// SaveToClickHouse is a consumer that saves data to ClickHouse +type SaveToClickHouse struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveToClickHouse creates a new SaveToClickHouse consumer +func NewSaveToClickHouse(config map[string]interface{}) (types.Processor, error) { + return &SaveToClickHouse{ + BaseConsumer: BaseConsumer{ + name: "SaveToClickHouse", + }, + config: config, + }, nil +} + +// SaveToMarketAnalyticsConsumer is a consumer that saves data to market analytics +type SaveToMarketAnalyticsConsumer struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveToMarketAnalyticsConsumer creates a new SaveToMarketAnalyticsConsumer +func NewSaveToMarketAnalyticsConsumer(config map[string]interface{}) (types.Processor, error) { + return &SaveToMarketAnalyticsConsumer{ + BaseConsumer: BaseConsumer{ + name: "SaveToMarketAnalyticsConsumer", + }, + config: config, + }, nil +} + +// SaveToRedisOrderbookConsumer is a consumer that saves orderbook data to Redis +type SaveToRedisOrderbookConsumer struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveToRedisOrderbookConsumer creates a new SaveToRedisOrderbookConsumer +func NewSaveToRedisOrderbookConsumer(config map[string]interface{}) (types.Processor, error) { + return &SaveToRedisOrderbookConsumer{ + BaseConsumer: BaseConsumer{ + name: "SaveToRedisOrderbookConsumer", + }, + config: config, + }, nil +} + +// SaveAssetToPostgreSQL is a consumer that saves asset data to PostgreSQL +type SaveAssetToPostgreSQL struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveAssetToPostgreSQL creates a new SaveAssetToPostgreSQL +func NewSaveAssetToPostgreSQL(config map[string]interface{}) (types.Processor, error) { + return &SaveAssetToPostgreSQL{ + BaseConsumer: BaseConsumer{ + name: "SaveAssetToPostgreSQL", + }, + config: config, + }, nil +} + +// StdoutConsumer is a consumer that outputs to stdout +type StdoutConsumer struct { + BaseConsumer +} + +// NewStdoutConsumer creates a new StdoutConsumer +func NewStdoutConsumer() types.Processor { + return &StdoutConsumer{ + BaseConsumer: BaseConsumer{ + name: "StdoutConsumer", + }, + } +} + +// SaveAssetEnrichmentConsumer is a consumer that saves asset enrichment data +type SaveAssetEnrichmentConsumer struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveAssetEnrichmentConsumer creates a new SaveAssetEnrichmentConsumer +func NewSaveAssetEnrichmentConsumer(config map[string]interface{}) (types.Processor, error) { + return &SaveAssetEnrichmentConsumer{ + BaseConsumer: BaseConsumer{ + name: "SaveAssetEnrichmentConsumer", + }, + config: config, + }, nil +} + +// SavePaymentToPostgreSQL is a consumer that saves payment data to PostgreSQL +type SavePaymentToPostgreSQL struct { + BaseConsumer + config map[string]interface{} +} + +// NewSavePaymentToPostgreSQL creates a new SavePaymentToPostgreSQL +func NewSavePaymentToPostgreSQL(config map[string]interface{}) (types.Processor, error) { + return &SavePaymentToPostgreSQL{ + BaseConsumer: BaseConsumer{ + name: "SavePaymentToPostgreSQL", + }, + config: config, + }, nil +} + +// SavePaymentsToRedis is a consumer that saves payments to Redis +type SavePaymentsToRedis struct { + BaseConsumer + config map[string]interface{} +} + +// NewSavePaymentsToRedis creates a new SavePaymentsToRedis +func NewSavePaymentsToRedis(config map[string]interface{}) (types.Processor, error) { + return &SavePaymentsToRedis{ + BaseConsumer: BaseConsumer{ + name: "SavePaymentsToRedis", + }, + config: config, + }, nil +} + +// SaveLatestLedgerRedis is a consumer that saves the latest ledger to Redis +type SaveLatestLedgerRedis struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveLatestLedgerRedis creates a new SaveLatestLedgerRedis +func NewSaveLatestLedgerRedis(config map[string]interface{}) (types.Processor, error) { + return &SaveLatestLedgerRedis{ + BaseConsumer: BaseConsumer{ + name: "SaveLatestLedgerRedis", + }, + config: config, + }, nil +} + +// SaveLatestLedgerToExcel is a consumer that saves the latest ledger to Excel +type SaveLatestLedgerToExcel struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveLatestLedgerToExcel creates a new SaveLatestLedgerToExcel +func NewSaveLatestLedgerToExcel(config map[string]interface{}) (types.Processor, error) { + return &SaveLatestLedgerToExcel{ + BaseConsumer: BaseConsumer{ + name: "SaveLatestLedgerToExcel", + }, + config: config, + }, nil +} + +// AnthropicClaudeConsumer is a consumer that uses Anthropic Claude +type AnthropicClaudeConsumer struct { + BaseConsumer + config map[string]interface{} +} + +// NewAnthropicClaudeConsumer creates a new AnthropicClaudeConsumer +func NewAnthropicClaudeConsumer(config map[string]interface{}) (types.Processor, error) { + return &AnthropicClaudeConsumer{ + BaseConsumer: BaseConsumer{ + name: "AnthropicClaudeConsumer", + }, + config: config, + }, nil +} + +// SaveSoroswapPairsToDuckDB is a consumer that saves Soroswap pairs to DuckDB +type SaveSoroswapPairsToDuckDB struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveSoroswapPairsToDuckDB creates a new SaveSoroswapPairsToDuckDB +func NewSaveSoroswapPairsToDuckDB(config map[string]interface{}) (types.Processor, error) { + return &SaveSoroswapPairsToDuckDB{ + BaseConsumer: BaseConsumer{ + name: "SaveSoroswapPairsToDuckDB", + }, + config: config, + }, nil +} + +// SaveSoroswapRouterToDuckDB is a consumer that saves Soroswap router data to DuckDB +type SaveSoroswapRouterToDuckDB struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveSoroswapRouterToDuckDB creates a new SaveSoroswapRouterToDuckDB +func NewSaveSoroswapRouterToDuckDB(config map[string]interface{}) (types.Processor, error) { + return &SaveSoroswapRouterToDuckDB{ + BaseConsumer: BaseConsumer{ + name: "SaveSoroswapRouterToDuckDB", + }, + config: config, + }, nil +} + +// SaveAccountDataToPostgreSQL is a consumer that saves account data to PostgreSQL +type SaveAccountDataToPostgreSQL struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveAccountDataToPostgreSQL creates a new SaveAccountDataToPostgreSQL +func NewSaveAccountDataToPostgreSQL(config map[string]interface{}) (types.Processor, error) { + return &SaveAccountDataToPostgreSQL{ + BaseConsumer: BaseConsumer{ + name: "SaveAccountDataToPostgreSQL", + }, + config: config, + }, nil +} + +// SaveAccountDataToDuckDB is a consumer that saves account data to DuckDB +type SaveAccountDataToDuckDB struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveAccountDataToDuckDB creates a new SaveAccountDataToDuckDB +func NewSaveAccountDataToDuckDB(config map[string]interface{}) (types.Processor, error) { + return &SaveAccountDataToDuckDB{ + BaseConsumer: BaseConsumer{ + name: "SaveAccountDataToDuckDB", + }, + config: config, + }, nil +} + +// SaveContractEventsToPostgreSQL is a consumer that saves contract events to PostgreSQL +type SaveContractEventsToPostgreSQL struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveContractEventsToPostgreSQL creates a new SaveContractEventsToPostgreSQL +func NewSaveContractEventsToPostgreSQL(config map[string]interface{}) (types.Processor, error) { + return &SaveContractEventsToPostgreSQL{ + BaseConsumer: BaseConsumer{ + name: "SaveContractEventsToPostgreSQL", + }, + config: config, + }, nil +} + +// SaveSoroswapToPostgreSQL is a consumer that saves Soroswap data to PostgreSQL +type SaveSoroswapToPostgreSQL struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveSoroswapToPostgreSQL creates a new SaveSoroswapToPostgreSQL +func NewSaveSoroswapToPostgreSQL(config map[string]interface{}) (types.Processor, error) { + return &SaveSoroswapToPostgreSQL{ + BaseConsumer: BaseConsumer{ + name: "SaveSoroswapToPostgreSQL", + }, + config: config, + }, nil +} + +// SaveContractInvocationsToPostgreSQL is a consumer that saves contract invocations to PostgreSQL +type SaveContractInvocationsToPostgreSQL struct { + BaseConsumer + config map[string]interface{} +} + +// NewSaveContractInvocationsToPostgreSQL creates a new SaveContractInvocationsToPostgreSQL +func NewSaveContractInvocationsToPostgreSQL(config map[string]interface{}) (types.Processor, error) { + return &SaveContractInvocationsToPostgreSQL{ + BaseConsumer: BaseConsumer{ + name: "SaveContractInvocationsToPostgreSQL", + }, + config: config, + }, nil +} diff --git a/pkg/processor/account/account_data_filter_adapter.go b/pkg/processor/account/account_data_filter_adapter.go new file mode 100644 index 0000000..cb69106 --- /dev/null +++ b/pkg/processor/account/account_data_filter_adapter.go @@ -0,0 +1,61 @@ +package account + +import ( + "context" + + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" + "github.com/withObsrvr/cdp-pipeline-workflow/processor" +) + +// AccountDataFilterAdapter wraps the legacy AccountDataFilter to work with the new type system +type AccountDataFilterAdapter struct { + legacyProcessor *processor.AccountDataFilter +} + +// NewAccountDataFilterAdapter creates a new adapter for the AccountDataFilter processor +func NewAccountDataFilterAdapter(config map[string]interface{}) (*AccountDataFilterAdapter, error) { + // Create the legacy processor + legacyProc, err := processor.NewAccountDataFilter(config) + if err != nil { + return nil, err + } + + return &AccountDataFilterAdapter{ + legacyProcessor: legacyProc, + }, nil +} + +// Process implements the types.Processor interface by converting between Message types +func (a *AccountDataFilterAdapter) Process(ctx context.Context, msg types.Message) error { + // Convert types.Message to processor.Message + legacyMsg := processor.Message{ + Payload: msg.Payload, + } + + // Call the legacy processor + return a.legacyProcessor.Process(ctx, legacyMsg) +} + +// Subscribe implements the types.Processor interface +func (a *AccountDataFilterAdapter) Subscribe(p types.Processor) { + // Create an adapter that converts back from types.Processor to processor.Processor + adapter := &reverseFilterAdapter{processor: p} + a.legacyProcessor.Subscribe(adapter) +} + +// reverseFilterAdapter converts from types.Processor back to processor.Processor +type reverseFilterAdapter struct { + processor types.Processor +} + +func (r *reverseFilterAdapter) Process(ctx context.Context, msg processor.Message) error { + // Convert processor.Message to types.Message + typesMsg := types.Message{ + Payload: msg.Payload, + } + return r.processor.Process(ctx, typesMsg) +} + +func (r *reverseFilterAdapter) Subscribe(p processor.Processor) { + // This is not used in the adapter pattern +} \ No newline at end of file diff --git a/pkg/processor/base/factory.go b/pkg/processor/base/factory.go new file mode 100644 index 0000000..9912ef6 --- /dev/null +++ b/pkg/processor/base/factory.go @@ -0,0 +1,80 @@ +package base + +import ( + "fmt" + "plugin" + + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" +) + +// ProcessorRegistry is a map of processor types to factory functions +var ProcessorRegistry = make(map[string]ProcessorFactory) + +// ProcessorFactory is a function that creates a processor +type ProcessorFactory func(config map[string]interface{}) (types.Processor, error) + +// RegisterProcessor registers a processor factory +func RegisterProcessor(typeName string, factory ProcessorFactory) { + ProcessorRegistry[typeName] = factory +} + +// NewProcessor creates a processor based on the provided configuration. +func NewProcessor(config types.ProcessorConfig) (types.Processor, error) { + // First check if we have a registered factory for this processor type + if factory, exists := ProcessorRegistry[config.Type]; exists { + return factory(config.Config) + } + + // If not registered, check if we have a plugin for this processor type + proc, err := loadProcessorPlugin(config.Type, config.Config) + if err == nil { + return proc, nil + } + + return nil, fmt.Errorf("unknown processor type: %s", config.Type) +} + +// loadProcessorPlugin loads a processor from a plugin +func loadProcessorPlugin(processorType string, config map[string]interface{}) (types.Processor, error) { + // Try to load the plugin + pluginPath := fmt.Sprintf("plugins/%s/plugin.so", processorType) + p, err := plugin.Open(pluginPath) + if err != nil { + return nil, fmt.Errorf("failed to open processor plugin: %w", err) + } + + // Look up the processor constructor + newSym, err := p.Lookup("New") + if err != nil { + return nil, fmt.Errorf("processor plugin does not export 'New' function: %w", err) + } + + // Call the constructor + newFunc, ok := newSym.(func() interface{}) + if !ok { + return nil, fmt.Errorf("processor plugin 'New' is not a function") + } + + instance := newFunc() + + // Check if the instance has an Initialize method + initializer, ok := instance.(interface { + Initialize(map[string]interface{}) error + }) + if !ok { + return nil, fmt.Errorf("processor plugin instance does not implement Initialize method") + } + + // Initialize the processor + if err := initializer.Initialize(config); err != nil { + return nil, fmt.Errorf("failed to initialize processor plugin: %w", err) + } + + // Check if the instance implements the Processor interface + processor, ok := instance.(types.Processor) + if !ok { + return nil, fmt.Errorf("processor plugin instance does not implement Processor interface") + } + + return processor, nil +} diff --git a/pkg/processor/base/processor.go b/pkg/processor/base/processor.go new file mode 100644 index 0000000..3f1bc70 --- /dev/null +++ b/pkg/processor/base/processor.go @@ -0,0 +1,184 @@ +package base + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "time" + + "github.com/stellar/go/hash" + "github.com/stellar/go/ingest" + "github.com/stellar/go/xdr" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" +) + +// Processor alias to types.Processor for backward compatibility +type Processor = types.Processor + +// ProcessorConfig alias to types.ProcessorConfig for backward compatibility +type ProcessorConfig = types.ProcessorConfig + +// Message alias to types.Message for backward compatibility +type Message = types.Message + +// Helper functions for parsing data +func parseTimestamp(ts string) (time.Time, error) { + return types.ParseTimestamp(ts) +} + +// ExtractLedgerCloseMeta extracts xdr.LedgerCloseMeta from a processor.Message +func ExtractLedgerCloseMeta(msg Message) (xdr.LedgerCloseMeta, error) { + ledgerCloseMeta, ok := msg.Payload.(xdr.LedgerCloseMeta) + if !ok { + return xdr.LedgerCloseMeta{}, fmt.Errorf("expected LedgerCloseMeta, got %T", msg.Payload) + } + return ledgerCloseMeta, nil +} + +// ExtractLedgerTransaction extracts ingest.LedgerTransaction from a processor.Message +func ExtractLedgerTransaction(msg Message) (ingest.LedgerTransaction, error) { + ledgerTransaction, ok := msg.Payload.(ingest.LedgerTransaction) + if !ok { + return ingest.LedgerTransaction{}, fmt.Errorf("expected LedgerTransaction, got %T", msg.Payload) + } + return ledgerTransaction, nil +} + +// CreateTransactionReader creates a new ingest.LedgerTransactionReader from a processor.Message +func CreateTransactionReader(lcm xdr.LedgerCloseMeta, networkPassphrase string) (*ingest.LedgerTransactionReader, error) { + return ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(networkPassphrase, lcm) +} + +// ForwardToProcessors marshals the payload and forwards it to all downstream processors +func ForwardToProcessors(ctx context.Context, payload interface{}, processors []Processor) error { + jsonBytes, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("error marshaling payload: %w", err) + } + + for _, processor := range processors { + if err := processor.Process(ctx, Message{Payload: jsonBytes}); err != nil { + return fmt.Errorf("error in processor chain: %w", err) + } + } + + return nil +} + +type Event struct { + ID string `json:"id"` + Type string `json:"type"` + Ledger uint64 `json:"ledger"` + ContractID string `json:"contractId"` + TxHash string `json:"txHash"` + Topic interface{} `json:"topic"` + Value interface{} `json:"value"` +} + +// New utility functions for processing Stellar data + +// ExtractEntryFromIngestChange gets the most recent state of an entry from an ingestion change +func ExtractEntryFromIngestChange(change ingest.Change) (xdr.LedgerEntry, xdr.LedgerEntryChangeType, bool, error) { + switch changeType := change.LedgerEntryChangeType(); changeType { + case xdr.LedgerEntryChangeTypeLedgerEntryCreated, xdr.LedgerEntryChangeTypeLedgerEntryUpdated: + return *change.Post, changeType, false, nil + case xdr.LedgerEntryChangeTypeLedgerEntryRemoved: + return *change.Pre, changeType, true, nil + default: + return xdr.LedgerEntry{}, changeType, false, fmt.Errorf("unable to extract ledger entry type from change") + } +} + +// ExtractEntryFromXDRChange gets the most recent state of an entry from an ingestion change +func ExtractEntryFromXDRChange(change xdr.LedgerEntryChange) (xdr.LedgerEntry, xdr.LedgerEntryChangeType, bool, error) { + switch change.Type { + case xdr.LedgerEntryChangeTypeLedgerEntryCreated: + return *change.Created, change.Type, false, nil + case xdr.LedgerEntryChangeTypeLedgerEntryUpdated: + return *change.Updated, change.Type, false, nil + case xdr.LedgerEntryChangeTypeLedgerEntryRemoved: + return *change.State, change.Type, true, nil + default: + return xdr.LedgerEntry{}, change.Type, false, fmt.Errorf("unable to extract ledger entry type from change") + } +} + +// TimePointToUTCTimeStamp converts an xdr TimePoint to UTC time.Time +func TimePointToUTCTimeStamp(providedTime xdr.TimePoint) (time.Time, error) { + intTime := int64(providedTime) + if intTime < 0 { + return time.Time{}, fmt.Errorf("negative timepoint provided: %d", intTime) + } + return time.Unix(intTime, 0).UTC(), nil +} + +// GetLedgerCloseTime extracts the close time from a ledger +func GetLedgerCloseTime(lcm xdr.LedgerCloseMeta) (time.Time, error) { + return TimePointToUTCTimeStamp(lcm.LedgerHeaderHistoryEntry().Header.ScpValue.CloseTime) +} + +// GetLedgerSequence extracts the sequence number from a ledger +func GetLedgerSequence(lcm xdr.LedgerCloseMeta) uint32 { + return uint32(lcm.LedgerHeaderHistoryEntry().Header.LedgerSeq) +} + +// LedgerEntryToLedgerKeyHash generates a unique hash for a ledger entry +func LedgerEntryToLedgerKeyHash(ledgerEntry xdr.LedgerEntry) (string, error) { + ledgerKey, err := ledgerEntry.LedgerKey() + if err != nil { + return "", fmt.Errorf("failed to get ledger key: %w", err) + } + + ledgerKeyByte, err := ledgerKey.MarshalBinary() + if err != nil { + return "", fmt.Errorf("failed to marshal ledger key: %w", err) + } + + hashedLedgerKeyByte := hash.Hash(ledgerKeyByte) + return hex.EncodeToString(hashedLedgerKeyByte[:]), nil +} + +// ConvertStroopValueToReal converts stroop values to real XLM values +func ConvertStroopValueToReal(input xdr.Int64) (float64, error) { + return types.ConvertStroopValueToReal(input) +} + +// HashToHexString converts an XDR Hash to a hex string +func HashToHexString(inputHash xdr.Hash) string { + return types.HashToHexString(inputHash) +} + +// Use types from the common package +type Price = types.Price +type Path = types.Path +type Asset = types.Asset +type AssetStats = types.AssetStats +type SupplyData = types.SupplyData + +type SponsorshipOutput struct { + Operation xdr.Operation + OperationIndex uint32 +} + +// TestTransaction transaction meta +type TestTransaction struct { + Index uint32 + EnvelopeXDR string + ResultXDR string + FeeChangesXDR string + MetaXDR string + Hash string +} + +// ExtractEntryFromChange gets the most recent state of an entry from an ingestio change, as well as if the entry was deleted +func ExtractEntryFromChange(change ingest.Change) (xdr.LedgerEntry, xdr.LedgerEntryChangeType, bool, error) { + switch changeType := change.LedgerEntryChangeType(); changeType { + case xdr.LedgerEntryChangeTypeLedgerEntryCreated, xdr.LedgerEntryChangeTypeLedgerEntryUpdated: + return *change.Post, changeType, false, nil + case xdr.LedgerEntryChangeTypeLedgerEntryRemoved: + return *change.Pre, changeType, true, nil + default: + return xdr.LedgerEntry{}, changeType, false, fmt.Errorf("unable to extract ledger entry type from change") + } +} diff --git a/pkg/processor/contract/common/types.go b/pkg/processor/contract/common/types.go new file mode 100644 index 0000000..c291420 --- /dev/null +++ b/pkg/processor/contract/common/types.go @@ -0,0 +1,34 @@ +package common + +import ( + "encoding/json" + "time" + + "github.com/stellar/go/xdr" +) + +// ContractEvent represents an event emitted by a contract +type ContractEvent struct { + Timestamp time.Time `json:"timestamp"` + LedgerSequence uint32 `json:"ledger_sequence"` + TransactionHash string `json:"transaction_hash"` + ContractID string `json:"contract_id"` + Type string `json:"type"` + Topic []xdr.ScVal `json:"topic"` + Data json.RawMessage `json:"data"` + InSuccessfulTx bool `json:"in_successful_tx"` + EventIndex int `json:"event_index"` + OperationIndex int `json:"operation_index"` +} + +// ContractInvocation represents a contract invocation +type ContractInvocation struct { + Timestamp time.Time `json:"timestamp"` + LedgerSequence uint32 `json:"ledger_sequence"` + TransactionHash string `json:"transaction_hash"` + ContractID string `json:"contract_id"` + FunctionName string `json:"function_name,omitempty"` + InvokingAccount string `json:"invoking_account"` + Arguments []json.RawMessage `json:"arguments,omitempty"` + Successful bool `json:"successful"` +} diff --git a/pkg/processor/contract/kale/processor.go b/pkg/processor/contract/kale/processor.go new file mode 100644 index 0000000..3b51f5c --- /dev/null +++ b/pkg/processor/contract/kale/processor.go @@ -0,0 +1,603 @@ +package kale + +import ( + "context" + "encoding/json" + "fmt" + "log" + "strconv" + "strings" + "sync" + "time" + + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" +) + +// Event types +const ( + EventTypePlant = "plant" + EventTypeWork = "work" + EventTypeHarvest = "harvest" + EventTypeMint = "mint" +) + +// KaleBlockMetrics represents the metrics for a Kale block +type KaleBlockMetrics struct { + BlockIndex uint32 `json:"block_index"` + Timestamp time.Time `json:"timestamp"` + TotalStaked int64 `json:"total_staked"` + TotalReward int64 `json:"total_reward"` + Participants int `json:"participants"` + HighestZeroCount int `json:"highest_zero_count"` + CloseTimeMs int64 `json:"close_time_ms"` + Farmers []string `json:"farmers"` + MaxZeros uint32 `json:"max_zeros"` + MinZeros uint32 `json:"min_zeros"` + OpenTimeMs int64 `json:"open_time_ms"` + Duration int64 `json:"duration"` +} + +// KaleProcessor processes Kale contract events and invocations +type KaleProcessor struct { + contractID string + processors []types.Processor + blockMetrics map[uint32]*KaleBlockMetrics + mu sync.RWMutex + stats struct { + ProcessedEvents uint64 + PlantEvents uint64 + WorkEvents uint64 + HarvestEvents uint64 + MintEvents uint64 + LastProcessedTime time.Time + } +} + +// NewKaleProcessor creates a new KaleProcessor +func NewKaleProcessor(config map[string]interface{}) (*KaleProcessor, error) { + contractID, ok := config["contract_id"].(string) + if !ok { + return nil, fmt.Errorf("missing contract_id in configuration") + } + + log.Printf("Initializing KaleProcessor for contract: %s", contractID) + + return &KaleProcessor{ + contractID: contractID, + blockMetrics: make(map[uint32]*KaleBlockMetrics), + processors: make([]types.Processor, 0), + }, nil +} + +// Subscribe adds a processor to the chain +func (p *KaleProcessor) Subscribe(processor types.Processor) { + p.processors = append(p.processors, processor) + log.Printf("Added processor to KaleProcessor, total processors: %d", len(p.processors)) +} + +// Process handles both contract events and invocations +func (p *KaleProcessor) Process(ctx context.Context, msg types.Message) error { + // Parse the message + var rawMessage map[string]interface{} + + switch payload := msg.Payload.(type) { + case []byte: + if err := json.Unmarshal(payload, &rawMessage); err != nil { + return fmt.Errorf("error decoding message: %w", err) + } + case map[string]interface{}: + rawMessage = payload + default: + return fmt.Errorf("unexpected payload type: %T", msg.Payload) + } + + // Check if this is from our target contract + contractID, ok := rawMessage["contract_id"].(string) + if !ok { + // Try contract_id in camelCase + contractID, ok = rawMessage["contractId"].(string) + if !ok { + return nil // Not a contract message, skip + } + } + + if contractID != p.contractID { + return nil // Not our contract, skip + } + + // Check if this is an event or an invocation + if _, hasTopics := rawMessage["topic"]; hasTopics { + // This is an event message + return p.processEventMessage(ctx, rawMessage) + } else if _, hasInvokingAccount := rawMessage["invoking_account"]; hasInvokingAccount { + // This is a contract invocation + return p.processInvocationMessage(ctx, rawMessage) + } else if _, hasFunctionName := rawMessage["function_name"]; hasFunctionName { + // This is also a contract invocation + return p.processInvocationMessage(ctx, rawMessage) + } + + // Unknown message type + return nil +} + +// processEventMessage processes contract events +func (p *KaleProcessor) processEventMessage(ctx context.Context, contractEvent map[string]interface{}) error { + // Extract topic to determine event type + topicsRaw, ok := contractEvent["topic"] + if !ok { + return nil // No topics, skip + } + + // Parse event type from topics + var eventType string + var blockIndex uint32 + + // Extract event type from topics + topics, ok := topicsRaw.([]interface{}) + if !ok || len(topics) == 0 { + return nil + } + + // Try to extract event type from the first topic + firstTopic := topics[0] + switch t := firstTopic.(type) { + case string: + eventType = t + case map[string]interface{}: + if str, ok := t["string"].(string); ok { + eventType = str + } else if sym, ok := t["sym"].(string); ok { + eventType = sym + } else if sym, ok := t["Sym"].(string); ok { + eventType = sym + } + } + + if eventType == "" { + return nil + } + + // Try to extract block index from topics or data + if len(topics) >= 3 { + if indexVal, ok := topics[2].(float64); ok { + blockIndex = uint32(indexVal) + } + } + + // Extract timestamp + var timestamp time.Time + if timestampStr, ok := contractEvent["timestamp"].(string); ok { + if ts, err := time.Parse(time.RFC3339, timestampStr); err == nil { + timestamp = ts + } + } + + // Extract ledger sequence + var ledgerSequence uint32 + if seq, ok := contractEvent["ledger_sequence"].(float64); ok { + ledgerSequence = uint32(seq) + } + + // If block index is still 0, use ledger sequence as fallback + if blockIndex == 0 { + blockIndex = ledgerSequence + } + + // Get or create block metrics + metrics := p.getOrCreateBlockMetrics(blockIndex) + metrics.Timestamp = timestamp + + // Extract farmer address + var farmerAddr string + if len(topics) >= 4 { + if addr, ok := topics[3].(string); ok { + farmerAddr = addr + } + } + + // Parse event data + dataRaw, ok := contractEvent["data"] + if !ok { + return nil + } + + var eventData map[string]interface{} + dataBytes, err := json.Marshal(dataRaw) + if err != nil { + return fmt.Errorf("error marshaling event data: %w", err) + } + + if err := json.Unmarshal(dataBytes, &eventData); err != nil { + // Try to handle it as a simple value + eventData = map[string]interface{}{ + "value": dataRaw, + } + } + + // Update metrics based on event type + switch eventType { + case EventTypePlant: + p.mu.Lock() + p.stats.PlantEvents++ + p.stats.ProcessedEvents++ + p.stats.LastProcessedTime = time.Now() + p.mu.Unlock() + p.updatePlantMetrics(metrics, eventData, farmerAddr) + case EventTypeWork: + p.mu.Lock() + p.stats.WorkEvents++ + p.stats.ProcessedEvents++ + p.stats.LastProcessedTime = time.Now() + p.mu.Unlock() + p.updateWorkMetrics(metrics, eventData, farmerAddr) + case EventTypeHarvest, EventTypeMint: + p.mu.Lock() + if eventType == EventTypeHarvest { + p.stats.HarvestEvents++ + } else { + p.stats.MintEvents++ + } + p.stats.ProcessedEvents++ + p.stats.LastProcessedTime = time.Now() + p.mu.Unlock() + p.updateHarvestMetrics(metrics, eventData, farmerAddr) + } + + // Forward metrics to consumers + return p.forwardToProcessors(ctx, metrics) +} + +// processInvocationMessage processes contract invocations +func (p *KaleProcessor) processInvocationMessage(ctx context.Context, rawMessage map[string]interface{}) error { + // Extract function name + functionName, hasFunctionName := rawMessage["function_name"].(string) + if !hasFunctionName { + return nil // No function name, skip + } + + // Extract ledger sequence + var ledgerSeq uint32 + if seq, ok := rawMessage["ledger_sequence"].(float64); ok { + ledgerSeq = uint32(seq) + } + + // Extract block index based on function name and arguments + blockIndex := ledgerSeq // Default to ledger sequence + + // Try to extract block index from arguments + if argsRaw, ok := rawMessage["arguments"].([]interface{}); ok && len(argsRaw) > 0 { + // For plant function, the first argument might be the block index + if functionName == "plant" && len(argsRaw) >= 1 { + if indexArg, ok := argsRaw[0].(map[string]interface{}); ok { + if indexVal, ok := indexArg["u32"].(float64); ok { + blockIndex = uint32(indexVal) + } + } + } + } + + // Get or create block metrics + metrics := p.getOrCreateBlockMetrics(blockIndex) + + // Extract timestamp + if timestampStr, ok := rawMessage["timestamp"].(string); ok { + if timestamp, err := time.Parse(time.RFC3339, timestampStr); err == nil { + metrics.Timestamp = timestamp + } + } + + // Check for diagnostic events + if diagnosticEventsRaw, ok := rawMessage["diagnostic_events"].([]interface{}); ok && len(diagnosticEventsRaw) > 0 { + // Process diagnostic events to extract additional data + for _, eventRaw := range diagnosticEventsRaw { + event, ok := eventRaw.(map[string]interface{}) + if !ok { + continue + } + + // Extract topics + topicsRaw, ok := event["topics"] + if !ok { + continue + } + + topics, ok := topicsRaw.([]interface{}) + if !ok || len(topics) == 0 { + continue + } + + // Extract event type from topics + var eventType string + for _, topicRaw := range topics { + topicStr, ok := topicRaw.(string) + if !ok { + continue + } + + var topic map[string]interface{} + if err := json.Unmarshal([]byte(topicStr), &topic); err != nil { + continue + } + + if sym, ok := topic["Sym"].(string); ok { + eventType = sym + break + } + } + + // Extract data + dataRaw, ok := event["data"].(map[string]interface{}) + if !ok { + continue + } + + // Process based on event type + switch eventType { + case "mint": + // Extract amount from I128 value + if i128Raw, ok := dataRaw["I128"].(map[string]interface{}); ok { + if loVal, ok := i128Raw["Lo"].(float64); ok { + metrics.TotalReward += int64(loVal) + } + } + case "burn": + // Extract amount from I128 value + if i128Raw, ok := dataRaw["I128"].(map[string]interface{}); ok { + if loVal, ok := i128Raw["Lo"].(float64); ok { + metrics.TotalStaked += int64(loVal) + } + } + } + } + } + + // Update metrics based on function name + switch functionName { + case "plant": + // When processing a plant invocation for a new block + _, blockExists := p.blockMetrics[blockIndex] + if !blockExists { + currentTimeMs := time.Now().UnixMilli() + metrics.OpenTimeMs = currentTimeMs + } + + // Add farmer to participants if not already included + if invokingAccount, ok := rawMessage["invoking_account"].(string); ok && invokingAccount != "" { + if !contains(metrics.Farmers, invokingAccount) { + metrics.Farmers = append(metrics.Farmers, invokingAccount) + metrics.Participants = len(metrics.Farmers) + } + } + + case "work": + // Extract zero count from the hash + if argsRaw, ok := rawMessage["arguments"].([]interface{}); ok && len(argsRaw) >= 2 { + // The second argument should contain the hash + if hashArg, ok := argsRaw[1].(map[string]interface{}); ok { + if hashVal, ok := hashArg["hash"].(string); ok { + zeroCount := countLeadingZeros(hashVal) + metrics.MaxZeros = max(metrics.MaxZeros, zeroCount) + if metrics.MinZeros == 0 || zeroCount < metrics.MinZeros { + metrics.MinZeros = zeroCount + } + } + } + } + + // Add farmer to participants if not already included + if invokingAccount, ok := rawMessage["invoking_account"].(string); ok && invokingAccount != "" { + if !contains(metrics.Farmers, invokingAccount) { + metrics.Farmers = append(metrics.Farmers, invokingAccount) + metrics.Participants = len(metrics.Farmers) + } + } + + case "harvest": + // Set close time if not already set + if metrics.CloseTimeMs == 0 { + if timestampStr, ok := rawMessage["timestamp"].(string); ok { + if timestamp, err := time.Parse(time.RFC3339, timestampStr); err == nil { + metrics.CloseTimeMs = timestamp.UnixMilli() + } + } + } + + // Calculate duration if both open and close times are set + if metrics.OpenTimeMs > 0 && metrics.CloseTimeMs > 0 { + metrics.Duration = metrics.CloseTimeMs - metrics.OpenTimeMs + } + + // Add farmer to participants if not already included + if invokingAccount, ok := rawMessage["invoking_account"].(string); ok && invokingAccount != "" { + if !contains(metrics.Farmers, invokingAccount) { + metrics.Farmers = append(metrics.Farmers, invokingAccount) + metrics.Participants = len(metrics.Farmers) + } + } + } + + // Forward metrics to processors + return p.forwardToProcessors(ctx, metrics) +} + +// getOrCreateBlockMetrics gets or creates block metrics for a given block index +func (p *KaleProcessor) getOrCreateBlockMetrics(blockIndex uint32) *KaleBlockMetrics { + p.mu.Lock() + defer p.mu.Unlock() + + metrics, exists := p.blockMetrics[blockIndex] + if !exists { + metrics = &KaleBlockMetrics{ + BlockIndex: blockIndex, + Timestamp: time.Now(), + Participants: 0, + Farmers: []string{}, + } + p.blockMetrics[blockIndex] = metrics + } + return metrics +} + +// forwardToProcessors forwards metrics to downstream processors +func (p *KaleProcessor) forwardToProcessors(ctx context.Context, metrics *KaleBlockMetrics) error { + data, err := json.Marshal(metrics) + if err != nil { + return err + } + + msg := types.Message{ + Payload: data, + } + + for _, processor := range p.processors { + if err := processor.Process(ctx, msg); err != nil { + return fmt.Errorf("error in processor chain: %w", err) + } + } + + return nil +} + +// updatePlantMetrics updates metrics for plant events +func (p *KaleProcessor) updatePlantMetrics(metrics *KaleBlockMetrics, data map[string]interface{}, farmerAddr string) { + // Add farmer to participants if not already present + if farmerAddr != "" && !contains(metrics.Farmers, farmerAddr) { + metrics.Farmers = append(metrics.Farmers, farmerAddr) + metrics.Participants = len(metrics.Farmers) + } + + // Update total staked if available + if stakeVal, ok := data["amount"]; ok { + stake := p.parseAmount(stakeVal) + metrics.TotalStaked += stake + } +} + +// updateWorkMetrics updates metrics for work events +func (p *KaleProcessor) updateWorkMetrics(metrics *KaleBlockMetrics, data map[string]interface{}, farmerAddr string) { + // Add farmer to participants if not already present + if farmerAddr != "" && !contains(metrics.Farmers, farmerAddr) { + metrics.Farmers = append(metrics.Farmers, farmerAddr) + metrics.Participants = len(metrics.Farmers) + } + + // Update highest zero count if available + if zerosVal, ok := data["zeros"]; ok { + zeros := 0 + switch v := zerosVal.(type) { + case float64: + zeros = int(v) + case string: + z, err := strconv.Atoi(v) + if err == nil { + zeros = z + } + } + + if zeros > metrics.HighestZeroCount { + metrics.HighestZeroCount = zeros + } + } +} + +// updateHarvestMetrics updates metrics for harvest events +func (p *KaleProcessor) updateHarvestMetrics(metrics *KaleBlockMetrics, data map[string]interface{}, farmerAddr string) { + // Add farmer to participants if not already present + if farmerAddr != "" && !contains(metrics.Farmers, farmerAddr) { + metrics.Farmers = append(metrics.Farmers, farmerAddr) + metrics.Participants = len(metrics.Farmers) + } + + // Update total reward if available + if rewardVal, ok := data["reward"]; ok { + reward := p.parseAmount(rewardVal) + metrics.TotalReward += reward + } + + // Update close time if available + if closeTimeVal, ok := data["close_time"]; ok { + switch v := closeTimeVal.(type) { + case float64: + metrics.CloseTimeMs = int64(v) + case string: + ct, err := strconv.ParseInt(v, 10, 64) + if err == nil { + metrics.CloseTimeMs = ct + } + } + } + + // Calculate duration if both open and close times are set + if metrics.OpenTimeMs > 0 && metrics.CloseTimeMs > 0 { + metrics.Duration = metrics.CloseTimeMs - metrics.OpenTimeMs + } +} + +// parseAmount parses an amount value from various types +func (p *KaleProcessor) parseAmount(val interface{}) int64 { + switch v := val.(type) { + case float64: + return int64(v) + case string: + // Remove any non-numeric characters except decimal point + numStr := strings.Map(func(r rune) rune { + if (r >= '0' && r <= '9') || r == '.' { + return r + } + return -1 + }, v) + + // Parse as float first to handle decimal values + f, err := strconv.ParseFloat(numStr, 64) + if err == nil { + return int64(f) + } + } + return 0 +} + +// GetStats returns processor statistics +func (p *KaleProcessor) GetStats() struct { + ProcessedEvents uint64 + PlantEvents uint64 + WorkEvents uint64 + HarvestEvents uint64 + MintEvents uint64 + LastProcessedTime time.Time +} { + p.mu.RLock() + defer p.mu.RUnlock() + return p.stats +} + +// Helper function to check if a slice contains a string +func contains(slice []string, item string) bool { + for _, s := range slice { + if s == item { + return true + } + } + return false +} + +// countLeadingZeros counts the number of leading zeros in a string +func countLeadingZeros(s string) uint32 { + count := uint32(0) + for _, char := range s { + if char == '0' { + count++ + } else { + break + } + } + return count +} + +// max returns the maximum of two uint32 values +func max(a, b uint32) uint32 { + if a > b { + return a + } + return b +} diff --git a/pkg/processor/contract/soroswap/processor.go b/pkg/processor/contract/soroswap/processor.go new file mode 100644 index 0000000..bbaa847 --- /dev/null +++ b/pkg/processor/contract/soroswap/processor.go @@ -0,0 +1,340 @@ +package soroswap + +import ( + "context" + "encoding/json" + "fmt" + "log" + "sync" + "time" + + "github.com/stellar/go/strkey" + "github.com/stellar/go/xdr" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" +) + +// Event types +const ( + EventTypeNewPair = "new_pair" + EventTypeSync = "sync" +) + +// NewPairEvent represents a new pair creation event from Soroswap +type NewPairEvent struct { + Type string `json:"type"` + Timestamp time.Time `json:"timestamp"` + LedgerSequence uint32 `json:"ledger_sequence"` + ContractID string `json:"contract_id"` // Factory contract ID + PairAddress string `json:"pair_address"` // New pair contract address + Token0 string `json:"token_0"` // First token contract ID + Token1 string `json:"token_1"` // Second token contract ID +} + +// SyncEvent represents the structure of a Soroswap sync event +type SyncEvent struct { + Type string `json:"type"` + Timestamp time.Time `json:"timestamp"` + LedgerSequence uint32 `json:"ledger_sequence"` + ContractID string `json:"contract_id"` // This should be the pair address + NewReserve0 string `json:"new_reserve_0"` + NewReserve1 string `json:"new_reserve_1"` +} + +// PairInfo stores the token information for a pair +type PairInfo struct { + Token0 string + Token1 string +} + +// SoroswapPair represents a Soroswap pair +type SoroswapPair struct { + PairAddress string `json:"pair_address"` + Token0 string `json:"token_0"` + Token1 string `json:"token_1"` + Reserve0 uint64 `json:"reserve_0"` + Reserve1 uint64 `json:"reserve_1"` + LastUpdate time.Time `json:"last_update"` +} + +// SoroswapProcessor processes Soroswap contract events +type SoroswapProcessor struct { + factoryContractID string + processors []types.Processor + mu sync.RWMutex + pairs map[string]*SoroswapPair + stats struct { + ProcessedEvents uint64 + NewPairEvents uint64 + SyncEvents uint64 + LastProcessedTime time.Time + TotalValueLockedXLM float64 + } +} + +// NewSoroswapProcessor creates a new SoroswapProcessor +func NewSoroswapProcessor(config map[string]interface{}) (*SoroswapProcessor, error) { + contractID, ok := config["factory_contract_id"].(string) + if !ok { + return nil, fmt.Errorf("missing factory_contract_id in configuration") + } + + log.Printf("Initializing SoroswapProcessor for factory contract: %s", contractID) + + return &SoroswapProcessor{ + factoryContractID: contractID, + processors: make([]types.Processor, 0), + pairs: make(map[string]*SoroswapPair), + }, nil +} + +// Subscribe adds a processor to the chain +func (p *SoroswapProcessor) Subscribe(processor types.Processor) { + p.processors = append(p.processors, processor) + log.Printf("Added processor to SoroswapProcessor, total processors: %d", len(p.processors)) +} + +// Process handles contract events +func (p *SoroswapProcessor) Process(ctx context.Context, msg types.Message) error { + var contractEvent ContractEvent + switch payload := msg.Payload.(type) { + case []byte: + if err := json.Unmarshal(payload, &contractEvent); err != nil { + return fmt.Errorf("error decoding contract event: %w", err) + } + case ContractEvent: + contractEvent = payload + default: + return fmt.Errorf("unexpected payload type: %T", msg.Payload) + } + + // Check if we have enough topics + if len(contractEvent.Topic) < 2 { + return nil + } + + // Determine event type from topics + var eventType string + for _, topic := range contractEvent.Topic { + if topic.Type == xdr.ScValTypeScvSymbol { + sym := topic.MustSym() + if sym == "new_pair" { + eventType = EventTypeNewPair + break + } else if sym == "sync" { + eventType = EventTypeSync + break + } + } + } + + // Process based on event type + switch eventType { + case EventTypeNewPair: + return p.processNewPairEvent(ctx, contractEvent) + case EventTypeSync: + return p.processSyncEvent(ctx, contractEvent) + default: + // Not an event we're interested in + return nil + } +} + +// Process new pair events +func (p *SoroswapProcessor) processNewPairEvent(ctx context.Context, contractEvent ContractEvent) error { + // Parse the new pair event data + newPairEvent := NewPairEvent{ + Type: EventTypeNewPair, + Timestamp: contractEvent.Timestamp, + LedgerSequence: contractEvent.LedgerSequence, + ContractID: contractEvent.ContractID, + } + + // Extract token and pair addresses from the event data + var eventData struct { + V0 struct { + Data struct { + Map []struct { + Key struct { + Sym string `json:"Sym"` + } `json:"Key"` + Val struct { + Address struct { + ContractId []byte `json:"ContractId"` + } `json:"Address"` + } `json:"Val"` + } `json:"Map"` + } `json:"Data"` + } `json:"V0"` + } + + if err := json.Unmarshal(contractEvent.Data, &eventData); err != nil { + return fmt.Errorf("error parsing new pair event data: %w", err) + } + + // Extract addresses from the event data + for _, entry := range eventData.V0.Data.Map { + switch entry.Key.Sym { + case "token_0": + if contractID, err := encodeContractID(entry.Val.Address.ContractId); err == nil { + newPairEvent.Token0 = contractID + } + case "token_1": + if contractID, err := encodeContractID(entry.Val.Address.ContractId); err == nil { + newPairEvent.Token1 = contractID + } + case "pair": + if contractID, err := encodeContractID(entry.Val.Address.ContractId); err == nil { + newPairEvent.PairAddress = contractID + } + } + } + + if !validateContractID(newPairEvent.PairAddress) { + log.Printf("Warning: Invalid pair address format: %s", newPairEvent.PairAddress) + } + + // Store pair info in cache for future sync events + p.mu.Lock() + p.pairs[newPairEvent.PairAddress] = &SoroswapPair{ + PairAddress: newPairEvent.PairAddress, + Token0: newPairEvent.Token0, + Token1: newPairEvent.Token1, + } + p.stats.ProcessedEvents++ + p.stats.NewPairEvents++ + p.stats.LastProcessedTime = time.Now() + p.mu.Unlock() + + log.Printf("SoroswapProcessor: New pair created: %s (tokens: %s/%s)", + newPairEvent.PairAddress, newPairEvent.Token0, newPairEvent.Token1) + + // Forward the new pair event to downstream processors + return p.forwardToProcessors(ctx, newPairEvent) +} + +// Process sync events +func (p *SoroswapProcessor) processSyncEvent(ctx context.Context, contractEvent ContractEvent) error { + if !validateContractID(contractEvent.ContractID) { + log.Printf("Warning: Invalid contract ID format: %s", contractEvent.ContractID) + return nil + } + + // Create sync event + syncEvent := SyncEvent{ + Type: EventTypeSync, + Timestamp: contractEvent.Timestamp, + LedgerSequence: contractEvent.LedgerSequence, + ContractID: contractEvent.ContractID, // This is the pair address + } + + // Parse the event data + var eventData struct { + V0 struct { + Data struct { + Map []struct { + Key struct { + Sym string `json:"Sym"` + } `json:"Key"` + Val struct { + I128 struct { + Lo uint64 `json:"Lo"` + } `json:"I128"` + } `json:"Val"` + } `json:"Map"` + } `json:"Data"` + } `json:"V0"` + } + + if err := json.Unmarshal(contractEvent.Data, &eventData); err != nil { + return fmt.Errorf("error parsing sync event data: %w", err) + } + + // Extract reserve values + for _, entry := range eventData.V0.Data.Map { + switch entry.Key.Sym { + case "new_reserve_0": + syncEvent.NewReserve0 = fmt.Sprintf("%d", entry.Val.I128.Lo) + case "new_reserve_1": + syncEvent.NewReserve1 = fmt.Sprintf("%d", entry.Val.I128.Lo) + } + } + + log.Printf("SoroswapProcessor: Sync event for pair %s: reserve0=%s, reserve1=%s", + syncEvent.ContractID, syncEvent.NewReserve0, syncEvent.NewReserve1) + + // Update stats + p.mu.Lock() + p.stats.ProcessedEvents++ + p.stats.SyncEvents++ + p.stats.LastProcessedTime = time.Now() + p.mu.Unlock() + + // Forward the sync event to downstream processors + return p.forwardToProcessors(ctx, syncEvent) +} + +// GetPairInfo retrieves pair information from the cache +func (p *SoroswapProcessor) GetPairInfo(pairAddress string) (*SoroswapPair, bool) { + p.mu.RLock() + defer p.mu.RUnlock() + pair, exists := p.pairs[pairAddress] + return pair, exists +} + +// GetStats returns processor statistics +func (p *SoroswapProcessor) GetStats() struct { + ProcessedEvents uint64 + NewPairEvents uint64 + SyncEvents uint64 + LastProcessedTime time.Time + TotalValueLockedXLM float64 +} { + p.mu.RLock() + defer p.mu.RUnlock() + return p.stats +} + +// ContractEvent is defined here for compatibility +type ContractEvent struct { + Timestamp time.Time `json:"timestamp"` + LedgerSequence uint32 `json:"ledger_sequence"` + TransactionHash string `json:"transaction_hash"` + ContractID string `json:"contract_id"` + Type string `json:"type"` + Topic []xdr.ScVal `json:"topic"` + Data json.RawMessage `json:"data"` + InSuccessfulTx bool `json:"in_successful_tx"` + EventIndex int `json:"event_index"` + OperationIndex int `json:"operation_index"` +} + +// Helper function to validate contract IDs +func validateContractID(id string) bool { + // Soroban contract IDs should be 56 characters long + return len(id) == 56 +} + +// Helper function to encode contract IDs +func encodeContractID(contractId []byte) (string, error) { + return strkey.Encode(strkey.VersionByteContract, contractId) +} + +// forwardToProcessors forwards data to downstream processors +func (p *SoroswapProcessor) forwardToProcessors(ctx context.Context, data interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + msg := types.Message{ + Payload: jsonData, + } + + for _, processor := range p.processors { + if err := processor.Process(ctx, msg); err != nil { + return fmt.Errorf("error in processor chain: %w", err) + } + } + + return nil +} diff --git a/pkg/processor/ledger/latest_ledger.go b/pkg/processor/ledger/latest_ledger.go new file mode 100644 index 0000000..de92038 --- /dev/null +++ b/pkg/processor/ledger/latest_ledger.go @@ -0,0 +1,238 @@ +package ledger + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "strings" + "time" + + "github.com/stellar/go/ingest" + "github.com/stellar/go/xdr" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/processor/base" +) + +// Enhanced LatestLedger struct with transaction metrics +type LatestLedger struct { + Sequence uint32 `json:"sequence"` + Hash string `json:"hash"` + TransactionCount int `json:"transaction_count"` + OperationCount int `json:"operation_count"` + SuccessfulTxCount int `json:"successful_tx_count"` + FailedTxCount int `json:"failed_tx_count"` + TotalFeeCharged int64 `json:"total_fee_charged"` + ClosedAt time.Time `json:"closed_at"` + BaseFee uint32 `json:"base_fee"` + + // Soroban metrics + SorobanTxCount int `json:"soroban_tx_count"` + TotalSorobanFees int64 `json:"total_soroban_fees"` + TotalResourceInstructions uint64 `json:"total_resource_instructions"` +} + +// LatestLedgerProcessor processes ledger data and extracts metrics +type LatestLedgerProcessor struct { + networkPassphrase string + processors []types.Processor +} + +// NewLatestLedgerProcessor creates a new LatestLedgerProcessor +func NewLatestLedgerProcessor(config map[string]interface{}) (*LatestLedgerProcessor, error) { + networkPassphrase, ok := config["network_passphrase"].(string) + if !ok { + return nil, fmt.Errorf("missing network_passphrase in config") + } + + return &LatestLedgerProcessor{ + networkPassphrase: networkPassphrase, + }, nil +} + +// Subscribe adds a processor to the chain +func (p *LatestLedgerProcessor) Subscribe(processor types.Processor) { + p.processors = append(p.processors, processor) +} + +// Process processes a message containing a LedgerCloseMeta +func (p *LatestLedgerProcessor) Process(ctx context.Context, msg types.Message) error { + var ledgerCloseMeta xdr.LedgerCloseMeta + + // Try different payload types + switch payload := msg.Payload.(type) { + case xdr.LedgerCloseMeta: + ledgerCloseMeta = payload + case []byte: + // If the payload is JSON, try to extract ledger sequence and hash + var jsonData map[string]interface{} + if err := json.Unmarshal(payload, &jsonData); err == nil { + // Create a simplified ledger object with basic info + metrics := LatestLedger{} + + if seq, ok := jsonData["ledger_sequence"].(float64); ok { + metrics.Sequence = uint32(seq) + } + + if hash, ok := jsonData["ledger_hash"].(string); ok { + metrics.Hash = hash + } + + if closeTime, ok := jsonData["closed_at"].(string); ok { + if t, err := time.Parse(time.RFC3339, closeTime); err == nil { + metrics.ClosedAt = t + } + } + + // Marshal and forward to processors + jsonBytes, err := json.Marshal(metrics) + if err != nil { + return fmt.Errorf("error marshaling latest ledger: %w", err) + } + + log.Printf("Latest ledger from JSON: %d", metrics.Sequence) + + // Forward to the next set of processors + for _, processor := range p.processors { + if err := processor.Process(ctx, types.Message{Payload: jsonBytes}); err != nil { + return fmt.Errorf("error in processor chain: %w", err) + } + } + + return nil + } + return fmt.Errorf("unable to process payload of type []byte") + default: + return fmt.Errorf("expected xdr.LedgerCloseMeta, got %T", msg.Payload) + } + + // Create a transaction reader using the network passphrase + txReader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta( + p.networkPassphrase, + ledgerCloseMeta, + ) + if err != nil { + return fmt.Errorf("error creating transaction reader: %v", err) + } + defer txReader.Close() + + // Extract basic ledger fields + sequence := uint32(ledgerCloseMeta.LedgerHeaderHistoryEntry().Header.LedgerSeq) + hash := base.HashToHexString(ledgerCloseMeta.LedgerHeaderHistoryEntry().Hash) + closedAt, err := base.TimePointToUTCTimeStamp(ledgerCloseMeta.LedgerHeaderHistoryEntry().Header.ScpValue.CloseTime) + if err != nil { + return fmt.Errorf("error getting ledger close time: %v", err) + } + baseFee := ledgerCloseMeta.LedgerHeaderHistoryEntry().Header.BaseFee + + // Create the ledger metrics + metrics := LatestLedger{ + Sequence: sequence, + Hash: hash, + BaseFee: uint32(baseFee), + ClosedAt: closedAt, + } + + // Process each transaction + for { + tx, err := txReader.Read() + if err == io.EOF { + break + } + if err != nil { + // Check if the error is due to an unknown transaction hash + if strings.Contains(err.Error(), "unknown tx hash") { + log.Printf("Warning: skipping transaction due to error: %v", err) + continue + } + return fmt.Errorf("error reading transaction: %v", err) + } + + // Update metrics based on transaction data + metrics.TransactionCount++ + metrics.OperationCount += len(tx.Envelope.Operations()) + metrics.TotalFeeCharged += int64(tx.Result.Result.FeeCharged) + + if tx.Result.Successful() { + metrics.SuccessfulTxCount++ + } else { + metrics.FailedTxCount++ + } + + // Process Soroban metrics, if present + if hasSorobanTransaction(tx) { + metrics.SorobanTxCount++ + sMetrics := getSorobanMetrics(tx) + metrics.TotalSorobanFees += sMetrics.resourceFee + metrics.TotalResourceInstructions += uint64(sMetrics.instructions) + } + } + + jsonBytes, err := json.Marshal(metrics) + if err != nil { + return fmt.Errorf("error marshaling latest ledger: %w", err) + } + + log.Printf("Latest ledger: %d (Transactions: %d, Operations: %d, Success Rate: %.2f%%)", + metrics.Sequence, + metrics.TransactionCount, + metrics.OperationCount, + calculateSuccessRate(metrics.SuccessfulTxCount, metrics.TransactionCount), + ) + + // Forward to the next set of processors + for _, processor := range p.processors { + if err := processor.Process(ctx, types.Message{Payload: jsonBytes}); err != nil { + return fmt.Errorf("error in processor chain: %w", err) + } + } + + return nil +} + +// calculateSuccessRate calculates the success rate, handling division by zero +func calculateSuccessRate(successful, total int) float64 { + if total == 0 { + return 0.0 + } + return float64(successful) / float64(total) * 100 +} + +type sorobanMetrics struct { + resourceFee int64 + instructions uint32 + readBytes uint32 + writeBytes uint32 +} + +func hasSorobanTransaction(tx ingest.LedgerTransaction) bool { + switch tx.Envelope.Type { + case xdr.EnvelopeTypeEnvelopeTypeTx: + _, has := tx.Envelope.V1.Tx.Ext.GetSorobanData() + return has + case xdr.EnvelopeTypeEnvelopeTypeTxFeeBump: + _, has := tx.Envelope.FeeBump.Tx.InnerTx.V1.Tx.Ext.GetSorobanData() + return has + } + return false +} + +func getSorobanMetrics(tx ingest.LedgerTransaction) sorobanMetrics { + var sorobanData xdr.SorobanTransactionData + var sMetrics sorobanMetrics + + switch tx.Envelope.Type { + case xdr.EnvelopeTypeEnvelopeTypeTx: + sorobanData, _ = tx.Envelope.V1.Tx.Ext.GetSorobanData() + case xdr.EnvelopeTypeEnvelopeTypeTxFeeBump: + sorobanData, _ = tx.Envelope.FeeBump.Tx.InnerTx.V1.Tx.Ext.GetSorobanData() + } + + sMetrics.resourceFee = int64(sorobanData.ResourceFee) + sMetrics.instructions = uint32(sorobanData.Resources.Instructions) + sMetrics.readBytes = uint32(sorobanData.Resources.ReadBytes) + sMetrics.writeBytes = uint32(sorobanData.Resources.WriteBytes) + + return sMetrics +} diff --git a/pkg/processor/processors.go b/pkg/processor/processors.go new file mode 100644 index 0000000..33f0644 --- /dev/null +++ b/pkg/processor/processors.go @@ -0,0 +1,33 @@ +package processor + +import ( + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/processor/account" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/processor/base" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/processor/contract/kale" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/processor/contract/soroswap" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/processor/ledger" +) + +// init registers all built-in processors +func init() { + // Register Kale processor + base.RegisterProcessor("kale", func(config map[string]interface{}) (types.Processor, error) { + return kale.NewKaleProcessor(config) + }) + + // Register Soroswap processor + base.RegisterProcessor("soroswap", func(config map[string]interface{}) (types.Processor, error) { + return soroswap.NewSoroswapProcessor(config) + }) + + // Register LatestLedger processor + base.RegisterProcessor("LatestLedger", func(config map[string]interface{}) (types.Processor, error) { + return ledger.NewLatestLedgerProcessor(config) + }) + + // Register AccountDataFilter processor + base.RegisterProcessor("AccountDataFilter", func(config map[string]interface{}) (types.Processor, error) { + return account.NewAccountDataFilterAdapter(config) + }) +} diff --git a/pkg/source/base/source.go b/pkg/source/base/source.go new file mode 100644 index 0000000..a192abf --- /dev/null +++ b/pkg/source/base/source.go @@ -0,0 +1,19 @@ +package base + +import ( + "context" + + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" +) + +// SourceAdapter defines the interface for source adapters +type SourceAdapter interface { + Run(context.Context) error + Subscribe(types.Processor) +} + +// SourceConfig defines configuration for a source adapter +type SourceConfig struct { + Type string `yaml:"type"` + Config map[string]interface{} `yaml:"config"` +} diff --git a/pkg/source/source.go b/pkg/source/source.go new file mode 100644 index 0000000..3967b4c --- /dev/null +++ b/pkg/source/source.go @@ -0,0 +1,116 @@ +package source + +import ( + "context" + + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/source/base" + stellarrpc "github.com/withObsrvr/cdp-pipeline-workflow/pkg/source/stellar-rpc" +) + +// BaseSourceAdapter provides a base implementation of the SourceAdapter interface +type BaseSourceAdapter struct { + processors []types.Processor +} + +// Subscribe adds a processor to this source adapter +func (s *BaseSourceAdapter) Subscribe(p types.Processor) { + s.processors = append(s.processors, p) +} + +// Run starts the source adapter +func (s *BaseSourceAdapter) Run(ctx context.Context) error { + // Placeholder implementation + <-ctx.Done() + return nil +} + +// CaptiveCoreInboundAdapter is a source adapter for Captive Core +type CaptiveCoreInboundAdapter struct { + BaseSourceAdapter + config map[string]interface{} +} + +// NewCaptiveCoreInboundAdapter creates a new CaptiveCoreInboundAdapter +func NewCaptiveCoreInboundAdapter(config map[string]interface{}) (base.SourceAdapter, error) { + return &CaptiveCoreInboundAdapter{ + config: config, + }, nil +} + +// BufferedStorageSourceAdapter is a source adapter for buffered storage +type BufferedStorageSourceAdapter struct { + BaseSourceAdapter + config map[string]interface{} +} + +// NewBufferedStorageSourceAdapter creates a new BufferedStorageSourceAdapter +func NewBufferedStorageSourceAdapter(config map[string]interface{}) (base.SourceAdapter, error) { + return &BufferedStorageSourceAdapter{ + config: config, + }, nil +} + +// SorobanSourceAdapter is a source adapter for Soroban +type SorobanSourceAdapter struct { + BaseSourceAdapter + config map[string]interface{} +} + +// NewSorobanSourceAdapter creates a new SorobanSourceAdapter +func NewSorobanSourceAdapter(config map[string]interface{}) (base.SourceAdapter, error) { + return &SorobanSourceAdapter{ + config: config, + }, nil +} + +// GCSBufferedStorageSourceAdapter is a source adapter for GCS buffered storage +type GCSBufferedStorageSourceAdapter struct { + BaseSourceAdapter + config map[string]interface{} +} + +// NewGCSBufferedStorageSourceAdapter creates a new GCSBufferedStorageSourceAdapter +func NewGCSBufferedStorageSourceAdapter(config map[string]interface{}) (base.SourceAdapter, error) { + return &GCSBufferedStorageSourceAdapter{ + config: config, + }, nil +} + +// FSBufferedStorageSourceAdapter is a source adapter for FS buffered storage +type FSBufferedStorageSourceAdapter struct { + BaseSourceAdapter + config map[string]interface{} +} + +// NewFSBufferedStorageSourceAdapter creates a new FSBufferedStorageSourceAdapter +func NewFSBufferedStorageSourceAdapter(config map[string]interface{}) (base.SourceAdapter, error) { + return &FSBufferedStorageSourceAdapter{ + config: config, + }, nil +} + +// S3BufferedStorageSourceAdapter is a source adapter for S3 buffered storage +type S3BufferedStorageSourceAdapter struct { + BaseSourceAdapter + config map[string]interface{} +} + +// NewS3BufferedStorageSourceAdapter creates a new S3BufferedStorageSourceAdapter +func NewS3BufferedStorageSourceAdapter(config map[string]interface{}) (base.SourceAdapter, error) { + return &S3BufferedStorageSourceAdapter{ + config: config, + }, nil +} + +// RPCSourceAdapter is a source adapter for RPC +type RPCSourceAdapter struct { + BaseSourceAdapter + config map[string]interface{} +} + +// NewRPCSourceAdapter creates a new RPCSourceAdapter +func NewRPCSourceAdapter(config map[string]interface{}) (base.SourceAdapter, error) { + // Use the stellar-rpc adapter implementation + return stellarrpc.NewStellarRPCSourceAdapter(config) +} diff --git a/pkg/source/stellar-rpc/rpc.go b/pkg/source/stellar-rpc/rpc.go new file mode 100644 index 0000000..053dc58 --- /dev/null +++ b/pkg/source/stellar-rpc/rpc.go @@ -0,0 +1,549 @@ +package stellarrpc + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "strings" + "sync" + "time" + + "github.com/stellar/go/xdr" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/source/base" +) + +// Protocol structures +type LedgerInfo struct { + Sequence uint32 `json:"sequence"` + Hash string `json:"hash"` + LedgerCloseTime int64 `json:"ledger_close_time"` + LedgerMetadata string `json:"ledger_metadata,omitempty"` + LedgerMetadataJSON json.RawMessage `json:"ledger_metadata_json,omitempty"` +} + +type GetLedgersRequest struct { + StartLedger uint32 `json:"start_ledger,omitempty"` + Pagination *LedgerPaginationOptions `json:"pagination,omitempty"` + Format string `json:"format,omitempty"` +} + +type LedgerPaginationOptions struct { + Limit int `json:"limit,omitempty"` + Cursor string `json:"cursor,omitempty"` +} + +type GetLedgersResponse struct { + Ledgers []LedgerInfo `json:"ledgers"` + Next *string `json:"next,omitempty"` +} + +type GetLatestLedgerResponse struct { + Sequence uint32 `json:"sequence"` +} + +// StellarRPCSourceAdapter ingests ledger data from a Stellar RPC endpoint +type StellarRPCSourceAdapter struct { + endpoint string + apiKey string + pollInterval time.Duration + currentLedger uint32 + format string // Format to use when retrieving ledgers: "base64" or "json" + stopCh chan struct{} + wg sync.WaitGroup + processors []types.Processor + ledgerMu sync.Mutex + client *http.Client +} + +// NewStellarRPCSourceAdapter creates a new StellarRPCSourceAdapter +func NewStellarRPCSourceAdapter(config map[string]interface{}) (base.SourceAdapter, error) { + adapter := &StellarRPCSourceAdapter{ + pollInterval: 2 * time.Second, + format: "base64", // Default format + stopCh: make(chan struct{}), + processors: make([]types.Processor, 0), + } + + // Extract endpoint (required) + endpoint, ok := config["rpc_endpoint"].(string) + if !ok { + return nil, fmt.Errorf("rpc_endpoint configuration is required") + } + adapter.endpoint = endpoint + + // Extract API key (optional) + apiKey, _ := config["api_key"].(string) + adapter.apiKey = apiKey + + // Extract poll interval (optional, default: 2s) + if interval, ok := config["poll_interval"].(float64); ok { + adapter.pollInterval = time.Duration(interval) * time.Second + } + + // Extract starting ledger (optional) + if startLedger, ok := config["start_ledger"].(float64); ok { + adapter.currentLedger = uint32(startLedger) + } + + // Extract format (optional, default: "base64") + adapter.format = "base64" + if format, ok := config["format"].(string); ok { + if format == "json" || format == "base64" { + adapter.format = format + } else { + log.Printf("Warning: Invalid format '%s', using default 'base64'", format) + } + } + log.Printf("Using format: %s", adapter.format) + + // Create HTTP client with API key if provided + adapter.client = &http.Client{} + if apiKey != "" { + adapter.client.Transport = &transportWithAPIKey{ + apiKey: apiKey, + rt: http.DefaultTransport, + } + } + + log.Printf("StellarRPCSourceAdapter initialized with endpoint: %s, poll interval: %s", endpoint, adapter.pollInterval) + return adapter, nil +} + +// Subscribe adds a processor to this source adapter +func (s *StellarRPCSourceAdapter) Subscribe(processor types.Processor) { + s.ledgerMu.Lock() + defer s.ledgerMu.Unlock() + s.processors = append(s.processors, processor) + log.Printf("Processor %T subscribed to StellarRPCSourceAdapter", processor) +} + +// Run starts the adapter and begins polling for ledgers +func (s *StellarRPCSourceAdapter) Run(ctx context.Context) error { + log.Printf("Starting StellarRPCSourceAdapter with endpoint: %s", s.endpoint) + + // If no current ledger is set, get the latest ledger + if s.currentLedger == 0 { + latest, err := s.getLatestLedger(ctx) + if err != nil { + log.Printf("Failed to get latest ledger: %v, using a default starting point", err) + s.currentLedger = 1 // Use a reasonable default if we can't get the latest + } else { + // Start from a few ledgers back from the latest to avoid any issues + // with very recent ledgers that might not be fully available + if latest > 100 { + s.currentLedger = latest - 100 + } else { + s.currentLedger = 1 + } + log.Printf("Starting from ledger: %d (latest was: %d)", s.currentLedger, latest) + } + } else { + log.Printf("Starting from specified ledger: %d", s.currentLedger) + } + + s.wg.Add(1) + go s.pollLedgers(ctx) + + // Block until context is done + <-ctx.Done() + log.Printf("Context done, stopping Stellar RPC adapter") + + // Signal the polling goroutine to stop + close(s.stopCh) + + // Wait for the polling goroutine to finish + s.wg.Wait() + log.Printf("Stellar RPC adapter stopped") + + return ctx.Err() +} + +// pollLedgers continuously polls for new ledgers +func (s *StellarRPCSourceAdapter) pollLedgers(ctx context.Context) { + defer s.wg.Done() + + ticker := time.NewTicker(s.pollInterval) + defer ticker.Stop() + + // Track consecutive errors to adjust behavior + consecutiveErrors := 0 + maxConsecutiveErrors := 5 + + for { + select { + case <-s.stopCh: + log.Printf("Ledger polling stopped") + return + case <-ticker.C: + err := s.fetchAndProcessLedger(ctx) + if err != nil { + log.Printf("Error processing ledger: %v", err) + consecutiveErrors++ + + // If we're having persistent issues, try skipping ahead or getting a new latest ledger + if consecutiveErrors >= maxConsecutiveErrors { + log.Printf("Encountered %d consecutive errors, attempting recovery", consecutiveErrors) + if err := s.attemptRecovery(ctx); err != nil { + log.Printf("Recovery attempt failed: %v", err) + } else { + consecutiveErrors = 0 + } + } + } else { + // Reset error counter on success + consecutiveErrors = 0 + } + } + } +} + +// attemptRecovery tries to recover from persistent errors by getting a new latest ledger +// or skipping ahead if needed +func (s *StellarRPCSourceAdapter) attemptRecovery(ctx context.Context) error { + // Try getting the latest ledger + latest, err := s.getLatestLedger(ctx) + if err != nil { + // If we can't get the latest ledger, skip ahead by 100 + log.Printf("Failed to get latest ledger for recovery: %v, skipping ahead by 100", err) + s.currentLedger += 100 + return nil + } + + // If our current ledger is more than 1000 behind, jump forward + if latest > s.currentLedger+1000 { + s.currentLedger = latest - 100 + log.Printf("Current ledger was too far behind, jumped to ledger %d", s.currentLedger) + } else { + // Otherwise, just skip ahead by 10 ledgers + s.currentLedger += 10 + log.Printf("Skipped ahead by 10 ledgers to %d", s.currentLedger) + } + + return nil +} + +// fetchAndProcessLedger fetches a ledger and sends it to processors +func (s *StellarRPCSourceAdapter) fetchAndProcessLedger(ctx context.Context) error { + log.Printf("Fetching ledger %d from RPC endpoint", s.currentLedger) + + // Create the request + reqBody := map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "method": "getLedgers", + "params": GetLedgersRequest{ + StartLedger: s.currentLedger, + Pagination: &LedgerPaginationOptions{ + Limit: 1, + }, + Format: s.format, + }, + } + + // Serialize to JSON + jsonBody, err := json.Marshal(reqBody) + if err != nil { + return fmt.Errorf("failed to marshal request: %w", err) + } + + // Create HTTP request with the JSON body + req, err := http.NewRequestWithContext(ctx, "POST", s.endpoint, bytes.NewBuffer(jsonBody)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + // Execute request + log.Printf("Sending request to %s with body: %s", s.endpoint, string(jsonBody)) + resp, err := s.client.Do(req) + if err != nil { + return fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + // Check status + log.Printf("Received response with status: %d", resp.StatusCode) + if resp.StatusCode != http.StatusOK { + // If not OK, try to read the error response body + body, _ := io.ReadAll(resp.Body) + log.Printf("Error response body: %s", string(body)) + // Skip to the next ledger if we get a server error + log.Printf("Skipping ledger %d due to server error", s.currentLedger) + s.currentLedger++ + return nil + } + + // Read the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } + log.Printf("Response body: %s", string(body)) + + // Parse response + var result struct { + Result GetLedgersResponse `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` + } + if err := json.Unmarshal(body, &result); err != nil { + return fmt.Errorf("failed to decode response: %w", err) + } + + // Check for JSON-RPC error + if result.Error != nil { + log.Printf("JSON-RPC error: %d - %s", result.Error.Code, result.Error.Message) + // Skip to the next ledger if we get a server error + log.Printf("Skipping ledger %d due to JSON-RPC error", s.currentLedger) + s.currentLedger++ + return nil + } + + // Check if we got any ledgers + if len(result.Result.Ledgers) == 0 { + log.Printf("No ledgers available starting from %d, waiting for next poll", s.currentLedger) + return nil + } + + // Process the ledger + ledger := result.Result.Ledgers[0] + if ledger.Sequence != s.currentLedger { + log.Printf("Warning: Requested ledger %d but got %d", s.currentLedger, ledger.Sequence) + } + + log.Printf("Processing ledger %d with hash %s", ledger.Sequence, ledger.Hash) + + var payload interface{} + var format string + + // Process based on the configured format + if s.format == "base64" { + // Try to convert the ledger to XDR format + ledgerXDR, err := s.convertToXDR(ledger) + if err != nil { + log.Printf("Warning: Could not convert ledger to XDR format: %v", err) + return fmt.Errorf("failed to convert ledger to XDR format: %w", err) + } + payload = *ledgerXDR + format = "xdr" + log.Printf("Using XDR format for ledger %d", ledger.Sequence) + } else { + // Using JSON format + // Check if we have JSON metadata directly + if ledger.LedgerMetadataJSON != nil && len(ledger.LedgerMetadataJSON) > 0 { + // Parse the JSON metadata + var parsedMetadata map[string]interface{} + if err := json.Unmarshal(ledger.LedgerMetadataJSON, &parsedMetadata); err != nil { + log.Printf("Error parsing JSON metadata: %v", err) + // Fall back to raw metadata + payload = ledger.LedgerMetadataJSON + format = "raw_json" + } else { + // Successfully parsed JSON metadata + payload = parsedMetadata + format = "json" + log.Printf("Using parsed JSON metadata for ledger %d", ledger.Sequence) + } + } else if ledger.LedgerMetadata != "" { + // Try to parse the metadata as JSON + var parsedMetadata map[string]interface{} + if err := json.Unmarshal([]byte(ledger.LedgerMetadata), &parsedMetadata); err != nil { + log.Printf("Error parsing ledger metadata as JSON: %v", err) + // Fall back to raw metadata + payload = []byte(ledger.LedgerMetadata) + format = "raw" + } else { + // Successfully parsed JSON + payload = parsedMetadata + format = "json" + log.Printf("Using parsed JSON from metadata for ledger %d", ledger.Sequence) + } + } else { + // No metadata available + log.Printf("No metadata available for ledger %d", ledger.Sequence) + payload = map[string]interface{}{} // Empty map + format = "empty" + } + } + + // Create the message to be processed + msg := types.Message{ + Payload: payload, + } + + // Add metadata to the payload (since Message doesn't have a Metadata field) + if metadata, ok := payload.(map[string]interface{}); ok { + metadata["ledger_sequence"] = ledger.Sequence + metadata["ledger_hash"] = ledger.Hash + metadata["source"] = "stellar-rpc" + metadata["format"] = format + } + + // Process through each processor + if err := s.processMessageWithProcessors(ctx, msg); err != nil { + return fmt.Errorf("error in processor chain: %w", err) + } + + // Move to the next ledger + s.currentLedger++ + log.Printf("Successfully processed ledger %d, moving to %d", ledger.Sequence, s.currentLedger) + + return nil +} + +// convertToXDR attempts to convert a ledger from RPC format to XDR format +func (s *StellarRPCSourceAdapter) convertToXDR(ledger LedgerInfo) (*xdr.LedgerCloseMeta, error) { + // Check if we have the XDR data directly + if ledger.LedgerMetadata != "" { + var ledgerCloseMeta xdr.LedgerCloseMeta + + // Try to decode the base64-encoded XDR data + xdrBytes, err := base64.StdEncoding.DecodeString(ledger.LedgerMetadata) + if err != nil { + return nil, fmt.Errorf("failed to decode XDR data: %w", err) + } + + // Unmarshal the XDR data + if err := xdr.SafeUnmarshal(xdrBytes, &ledgerCloseMeta); err != nil { + return nil, fmt.Errorf("failed to unmarshal XDR data: %w", err) + } + + return &ledgerCloseMeta, nil + } + + // If we don't have XDR data directly, we would need to construct it from the JSON data + return nil, fmt.Errorf("XDR conversion from JSON not implemented") +} + +// processMessageWithProcessors processes the message through all registered processors +func (s *StellarRPCSourceAdapter) processMessageWithProcessors(ctx context.Context, msg types.Message) error { + // Get a copy of the processors to avoid holding the lock during processing + s.ledgerMu.Lock() + processors := make([]types.Processor, len(s.processors)) + copy(processors, s.processors) + s.ledgerMu.Unlock() + + // Check if we have any processors + if len(processors) == 0 { + log.Printf("Warning: No processors registered") + return nil + } + + // Process through each processor in sequence + for i, proc := range processors { + select { + case <-ctx.Done(): + return ctx.Err() + default: + procStart := time.Now() + + if err := proc.Process(ctx, msg); err != nil { + log.Printf("Error in processor %d (%T): %v", i, proc, err) + return fmt.Errorf("processor %d (%T) failed: %w", i, proc, err) + } + + processingTime := time.Since(procStart) + if processingTime > time.Second { + log.Printf("Warning: Processor %d (%T) took %v to process", i, proc, processingTime) + } else { + log.Printf("Processor %d (%T) processed successfully in %v", i, proc, processingTime) + } + } + } + + return nil +} + +// getLatestLedger gets the latest ledger from the RPC endpoint +func (s *StellarRPCSourceAdapter) getLatestLedger(ctx context.Context) (uint32, error) { + // Create the request + reqBody := map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "method": "getLatestLedger", + } + + // Serialize to JSON + jsonBody, err := json.Marshal(reqBody) + if err != nil { + return 0, fmt.Errorf("failed to marshal request: %w", err) + } + + // Create HTTP request with JSON body + req, err := http.NewRequestWithContext(ctx, "POST", s.endpoint, bytes.NewBuffer(jsonBody)) + if err != nil { + return 0, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + // Execute request + log.Printf("Sending getLatestLedger request to %s with body: %s", s.endpoint, string(jsonBody)) + resp, err := s.client.Do(req) + if err != nil { + return 0, fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + // Check status + log.Printf("Received getLatestLedger response with status: %d", resp.StatusCode) + if resp.StatusCode != http.StatusOK { + // Read and log error body + body, _ := io.ReadAll(resp.Body) + log.Printf("Error response body: %s", string(body)) + return 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + // Read the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return 0, fmt.Errorf("failed to read response body: %w", err) + } + log.Printf("getLatestLedger response body: %s", string(body)) + + // Parse response + var result struct { + Result GetLatestLedgerResponse `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` + } + if err := json.Unmarshal(body, &result); err != nil { + return 0, fmt.Errorf("failed to decode response: %w", err) + } + + // Check for JSON-RPC error + if result.Error != nil { + log.Printf("JSON-RPC error: %d - %s", result.Error.Code, result.Error.Message) + return 0, fmt.Errorf("JSON-RPC error: %s", result.Error.Message) + } + + log.Printf("Latest ledger from RPC: %d", result.Result.Sequence) + return result.Result.Sequence, nil +} + +// transportWithAPIKey adds API key to requests +type transportWithAPIKey struct { + apiKey string + rt http.RoundTripper +} + +// RoundTrip implements the http.RoundTripper interface +func (t *transportWithAPIKey) RoundTrip(req *http.Request) (*http.Response, error) { + // Make sure the API key has the 'Api-Key ' prefix if it doesn't already + apiKeyValue := t.apiKey + if !strings.HasPrefix(apiKeyValue, "Api-Key ") { + apiKeyValue = "Api-Key " + apiKeyValue + } + + req.Header.Set("Authorization", apiKeyValue) + return t.rt.RoundTrip(req) +} diff --git a/plugins/kale/plugin.go b/plugins/kale/plugin.go new file mode 100644 index 0000000..10ec3a5 --- /dev/null +++ b/plugins/kale/plugin.go @@ -0,0 +1,49 @@ +package main + +import ( + "context" + "log" + + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/common/types" + "github.com/withObsrvr/cdp-pipeline-workflow/pkg/processor/contract/kale" +) + +// KaleProcessorPlugin is a plugin that wraps the KaleProcessor +type KaleProcessorPlugin struct { + processor *kale.KaleProcessor +} + +func (p *KaleProcessorPlugin) Initialize(config map[string]interface{}) error { + var err error + p.processor, err = kale.NewKaleProcessor(config) + return err +} + +func (p *KaleProcessorPlugin) Process(ctx context.Context, msg types.Message) error { + return p.processor.Process(ctx, msg) +} + +func (p *KaleProcessorPlugin) Subscribe(processor types.Processor) { + p.processor.Subscribe(processor) +} + +func (p *KaleProcessorPlugin) Name() string { + return "flow/processor/kale" +} + +func (p *KaleProcessorPlugin) Version() string { + return "1.0.0" +} + +func (p *KaleProcessorPlugin) Type() string { + return "processor" +} + +// New creates a new KaleProcessorPlugin +func New() interface{} { + log.Printf("Creating new KaleProcessorPlugin instance") + return &KaleProcessorPlugin{} +} + +// This is required to make this a valid Go plugin +func main() {} diff --git a/processor/processor_account_data_filter.go b/processor/processor_account_data_filter.go new file mode 100644 index 0000000..92ce294 --- /dev/null +++ b/processor/processor_account_data_filter.go @@ -0,0 +1,268 @@ +package processor + +import ( + "context" + "encoding/json" + "fmt" + "log" + "strconv" + "sync" + "time" +) + +// AccountDataFilter filters AccountRecord messages based on various criteria +type AccountDataFilter struct { + processors []Processor + accountIDs map[string]bool // Using a map for O(1) lookups + startDate *time.Time + endDate *time.Time + dateField string // "closed_at", "sequence_time", or "timestamp" + changeTypes map[string]bool + minBalance *int64 + deletedOnly bool + mu sync.RWMutex + stats struct { + ProcessedRecords int64 + FilteredRecords int64 + LastProcessTime time.Time + } +} + +// NewAccountDataFilter creates a new AccountDataFilter processor +func NewAccountDataFilter(config map[string]interface{}) (*AccountDataFilter, error) { + filter := &AccountDataFilter{ + accountIDs: make(map[string]bool), + changeTypes: make(map[string]bool), + dateField: "closed_at", // Default date field + } + + // Handle single account_id (string) + if accountID, ok := config["account_id"].(string); ok && accountID != "" { + filter.accountIDs[accountID] = true + } + + // Handle account_ids (array) + if accountIDsArray, ok := config["account_ids"].([]interface{}); ok { + for _, id := range accountIDsArray { + if strID, ok := id.(string); ok && strID != "" { + filter.accountIDs[strID] = true + } + } + } + + // Handle date range filtering + if startDateStr, ok := config["start_date"].(string); ok && startDateStr != "" { + startDate, err := time.Parse(time.RFC3339, startDateStr) + if err != nil { + return nil, fmt.Errorf("invalid start_date format (expected RFC3339): %w", err) + } + filter.startDate = &startDate + } + + if endDateStr, ok := config["end_date"].(string); ok && endDateStr != "" { + endDate, err := time.Parse(time.RFC3339, endDateStr) + if err != nil { + return nil, fmt.Errorf("invalid end_date format (expected RFC3339): %w", err) + } + filter.endDate = &endDate + } + + // Handle date field selection + if dateField, ok := config["date_field"].(string); ok && dateField != "" { + switch dateField { + case "closed_at", "sequence_time", "timestamp": + filter.dateField = dateField + default: + return nil, fmt.Errorf("invalid date_field: %s (must be 'closed_at', 'sequence_time', or 'timestamp')", dateField) + } + } + + // Handle change type filtering + if changeTypesArray, ok := config["change_types"].([]interface{}); ok { + for _, ct := range changeTypesArray { + if strType, ok := ct.(string); ok && strType != "" { + switch strType { + case "created", "updated", "removed", "state": + filter.changeTypes[strType] = true + default: + return nil, fmt.Errorf("invalid change_type: %s (must be 'created', 'updated', 'removed', or 'state')", strType) + } + } + } + } + + // Handle minimum balance filtering + if minBalanceStr, ok := config["min_balance"].(string); ok && minBalanceStr != "" { + minBalance, err := strconv.ParseInt(minBalanceStr, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid min_balance format: %w", err) + } + filter.minBalance = &minBalance + } + + // Handle deleted_only flag + if deletedOnly, ok := config["deleted_only"].(bool); ok { + filter.deletedOnly = deletedOnly + } + + // Log the filter configuration + log.Printf("AccountDataFilter: Configured with the following filters:") + if len(filter.accountIDs) > 0 { + log.Printf(" - Account IDs: %d accounts", len(filter.accountIDs)) + for id := range filter.accountIDs { + log.Printf(" - %s", id) + } + } + if filter.startDate != nil { + log.Printf(" - Start date: %s", filter.startDate.Format(time.RFC3339)) + } + if filter.endDate != nil { + log.Printf(" - End date: %s", filter.endDate.Format(time.RFC3339)) + } + log.Printf(" - Date field: %s", filter.dateField) + if len(filter.changeTypes) > 0 { + log.Printf(" - Change types: %v", getMapKeys(filter.changeTypes)) + } + if filter.minBalance != nil { + log.Printf(" - Minimum balance: %d stroops", *filter.minBalance) + } + if filter.deletedOnly { + log.Printf(" - Deleted accounts only: true") + } + + return filter, nil +} + +// Subscribe adds a processor to the chain +func (f *AccountDataFilter) Subscribe(processor Processor) { + f.processors = append(f.processors, processor) +} + +// Process processes an AccountRecord message and applies filtering +func (f *AccountDataFilter) Process(ctx context.Context, msg Message) error { + f.mu.Lock() + f.stats.ProcessedRecords++ + f.mu.Unlock() + + // Parse the AccountRecord from JSON + var accountRecord AccountRecord + if jsonBytes, ok := msg.Payload.([]byte); ok { + if err := json.Unmarshal(jsonBytes, &accountRecord); err != nil { + return fmt.Errorf("error unmarshaling AccountRecord: %w", err) + } + } else { + return fmt.Errorf("invalid payload type: expected []byte, got %T", msg.Payload) + } + + // Apply filters + if !f.passesFilters(&accountRecord) { + f.mu.Lock() + f.stats.FilteredRecords++ + f.mu.Unlock() + return nil // Filtered out, don't forward + } + + // Forward to downstream processors + for i, processor := range f.processors { + if err := processor.Process(ctx, msg); err != nil { + return fmt.Errorf("error in downstream processor %d: %w", i, err) + } + } + + f.mu.Lock() + f.stats.LastProcessTime = time.Now() + f.mu.Unlock() + + return nil +} + +// passesFilters checks if an AccountRecord passes all configured filters +func (f *AccountDataFilter) passesFilters(record *AccountRecord) bool { + // Account ID filter + if len(f.accountIDs) > 0 { + if _, exists := f.accountIDs[record.AccountID]; !exists { + return false + } + } + + // Date range filter + if f.startDate != nil || f.endDate != nil { + var recordTime time.Time + switch f.dateField { + case "closed_at": + recordTime = record.ClosedAt + case "sequence_time": + recordTime = record.SequenceTime + case "timestamp": + recordTime = record.Timestamp + } + + if f.startDate != nil && recordTime.Before(*f.startDate) { + return false + } + if f.endDate != nil && recordTime.After(*f.endDate) { + return false + } + } + + // Change type filter + if len(f.changeTypes) > 0 { + changeType := getChangeTypeString(record.LedgerEntryChange) + if _, exists := f.changeTypes[changeType]; !exists { + return false + } + } + + // Minimum balance filter + if f.minBalance != nil && !record.Deleted { + balance, err := strconv.ParseInt(record.Balance, 10, 64) + if err != nil { + log.Printf("Warning: Could not parse balance '%s' for account %s", record.Balance, record.AccountID) + return false + } + if balance < *f.minBalance { + return false + } + } + + // Deleted only filter + if f.deletedOnly && !record.Deleted { + return false + } + + return true +} + +// GetStats returns the current filter statistics +func (f *AccountDataFilter) GetStats() struct { + ProcessedRecords int64 + FilteredRecords int64 + LastProcessTime time.Time +} { + f.mu.RLock() + defer f.mu.RUnlock() + return f.stats +} + +// Helper function to get change type string from uint32 +func getChangeTypeString(changeType uint32) string { + switch changeType { + case 1: + return "created" + case 2: + return "updated" + case 3: + return "state" + default: + return "unknown" + } +} + +// Helper function to get keys from a map +func getMapKeys(m map[string]bool) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + return keys +} \ No newline at end of file diff --git a/processor/processor_contract_invocation.go b/processor/processor_contract_invocation.go index d4d1a6e..4cc21f2 100644 --- a/processor/processor_contract_invocation.go +++ b/processor/processor_contract_invocation.go @@ -16,17 +16,19 @@ import ( // ContractInvocation represents a contract invocation event type ContractInvocation struct { - Timestamp time.Time `json:"timestamp"` - LedgerSequence uint32 `json:"ledger_sequence"` - TransactionHash string `json:"transaction_hash"` - ContractID string `json:"contract_id"` - InvokingAccount string `json:"invoking_account"` - FunctionName string `json:"function_name,omitempty"` - Successful bool `json:"successful"` - DiagnosticEvents []DiagnosticEvent `json:"diagnostic_events,omitempty"` - ContractCalls []ContractCall `json:"contract_calls,omitempty"` - StateChanges []StateChange `json:"state_changes,omitempty"` - TtlExtensions []TtlExtension `json:"ttl_extensions,omitempty"` + Timestamp time.Time `json:"timestamp"` + LedgerSequence uint32 `json:"ledger_sequence"` + TransactionHash string `json:"transaction_hash"` + ContractID string `json:"contract_id"` + InvokingAccount string `json:"invoking_account"` + FunctionName string `json:"function_name,omitempty"` + Arguments []json.RawMessage `json:"arguments,omitempty"` + ArgumentsDecoded map[string]interface{} `json:"arguments_decoded,omitempty"` + Successful bool `json:"successful"` + DiagnosticEvents []DiagnosticEvent `json:"diagnostic_events,omitempty"` + ContractCalls []ContractCall `json:"contract_calls,omitempty"` + StateChanges []StateChange `json:"state_changes,omitempty"` + TtlExtensions []TtlExtension `json:"ttl_extensions,omitempty"` } // DiagnosticEvent represents a diagnostic event emitted during contract execution @@ -199,12 +201,21 @@ func (p *ContractInvocationProcessor) processContractInvocation( Successful: successful, } - // Try to get function name if available + // Extract function name and arguments if function := invokeHostFunction.HostFunction; function.Type == xdr.HostFunctionTypeHostFunctionTypeInvokeContract { - if args := function.MustInvokeContract().Args; len(args) > 0 { - if sym, ok := args[0].GetSym(); ok { - invocation.FunctionName = string(sym) + invokeContract := function.MustInvokeContract() + + // Extract function name using robust method + invocation.FunctionName = extractFunctionName(invokeContract) + + // Extract and convert arguments + if len(invokeContract.Args) > 0 { + rawArgs, decodedArgs, err := extractArguments(invokeContract.Args) + if err != nil { + log.Printf("Error extracting arguments: %v", err) } + invocation.Arguments = rawArgs + invocation.ArgumentsDecoded = decodedArgs } } @@ -413,3 +424,52 @@ func (p *ContractInvocationProcessor) extractTtlExtensions(tx ingest.LedgerTrans return extensions } + +// extractFunctionName extracts the function name from a contract invocation +func extractFunctionName(invokeContract xdr.InvokeContractArgs) string { + // Primary method: Use FunctionName field directly + if len(invokeContract.FunctionName) > 0 { + return string(invokeContract.FunctionName) + } + + // Fallback: Check first argument if it contains function name + if len(invokeContract.Args) > 0 { + functionName := GetFunctionNameFromScVal(invokeContract.Args[0]) + if functionName != "" { + return functionName + } + } + + return "unknown" +} + +// extractArguments extracts and converts all function arguments +func extractArguments(args []xdr.ScVal) ([]json.RawMessage, map[string]interface{}, error) { + rawArgs := make([]json.RawMessage, 0, len(args)) + decodedArgs := make(map[string]interface{}) + + for i, arg := range args { + // Convert ScVal to JSON-serializable format + converted, err := ConvertScValToJSON(arg) + if err != nil { + log.Printf("Error converting argument %d: %v", i, err) + converted = map[string]interface{}{ + "error": err.Error(), + "type": arg.Type.String(), + } + } + + // Store raw JSON + jsonBytes, err := json.Marshal(converted) + if err != nil { + log.Printf("Error marshaling argument %d: %v", i, err) + continue + } + rawArgs = append(rawArgs, jsonBytes) + + // Store in decoded map with index + decodedArgs[fmt.Sprintf("arg_%d", i)] = converted + } + + return rawArgs, decodedArgs, nil +} diff --git a/processor/scval_converter.go b/processor/scval_converter.go new file mode 100644 index 0000000..c85e685 --- /dev/null +++ b/processor/scval_converter.go @@ -0,0 +1,368 @@ +package processor + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "math/big" + + "github.com/stellar/go/strkey" + "github.com/stellar/go/xdr" +) + +// ConvertScValToJSON converts a Soroban ScVal to a JSON-serializable format +func ConvertScValToJSON(val xdr.ScVal) (interface{}, error) { + switch val.Type { + case xdr.ScValTypeScvBool: + if val.B == nil { + return nil, fmt.Errorf("ScvBool has nil value") + } + return *val.B, nil + + case xdr.ScValTypeScvVoid: + return nil, nil + + case xdr.ScValTypeScvU32: + if val.U32 == nil { + return nil, fmt.Errorf("ScvU32 has nil value") + } + return *val.U32, nil + + case xdr.ScValTypeScvI32: + if val.I32 == nil { + return nil, fmt.Errorf("ScvI32 has nil value") + } + return *val.I32, nil + + case xdr.ScValTypeScvU64: + if val.U64 == nil { + return nil, fmt.Errorf("ScvU64 has nil value") + } + return *val.U64, nil + + case xdr.ScValTypeScvI64: + if val.I64 == nil { + return nil, fmt.Errorf("ScvI64 has nil value") + } + return *val.I64, nil + + case xdr.ScValTypeScvTimepoint: + if val.Timepoint == nil { + return nil, fmt.Errorf("ScvTimepoint has nil value") + } + return map[string]interface{}{ + "type": "timepoint", + "value": *val.Timepoint, + }, nil + + case xdr.ScValTypeScvDuration: + if val.Duration == nil { + return nil, fmt.Errorf("ScvDuration has nil value") + } + return map[string]interface{}{ + "type": "duration", + "value": *val.Duration, + }, nil + + case xdr.ScValTypeScvU128: + if val.U128 == nil { + return nil, fmt.Errorf("ScvU128 has nil value") + } + parts := *val.U128 + return map[string]interface{}{ + "type": "u128", + "hi": parts.Hi, + "lo": parts.Lo, + "value": uint128ToString(parts), + }, nil + + case xdr.ScValTypeScvI128: + if val.I128 == nil { + return nil, fmt.Errorf("ScvI128 has nil value") + } + parts := *val.I128 + return map[string]interface{}{ + "type": "i128", + "hi": parts.Hi, + "lo": parts.Lo, + "value": int128ToString(parts), + }, nil + + case xdr.ScValTypeScvU256: + if val.U256 == nil { + return nil, fmt.Errorf("ScvU256 has nil value") + } + parts := *val.U256 + return map[string]interface{}{ + "type": "u256", + "hi_hi": parts.HiHi, + "hi_lo": parts.HiLo, + "lo_hi": parts.LoHi, + "lo_lo": parts.LoLo, + "value": uint256ToString(parts), + "hex": uint256ToHex(parts), + }, nil + + case xdr.ScValTypeScvI256: + if val.I256 == nil { + return nil, fmt.Errorf("ScvI256 has nil value") + } + parts := *val.I256 + return map[string]interface{}{ + "type": "i256", + "hi_hi": parts.HiHi, + "hi_lo": parts.HiLo, + "lo_hi": parts.LoHi, + "lo_lo": parts.LoLo, + "value": int256ToString(parts), + "hex": int256ToHex(parts), + }, nil + + case xdr.ScValTypeScvSymbol: + if val.Sym == nil { + return nil, fmt.Errorf("ScvSymbol has nil value") + } + return string(*val.Sym), nil + + case xdr.ScValTypeScvString: + if val.Str == nil { + return nil, fmt.Errorf("ScvString has nil value") + } + return string(*val.Str), nil + + case xdr.ScValTypeScvBytes: + if val.Bytes == nil { + return nil, fmt.Errorf("ScvBytes has nil value") + } + return map[string]interface{}{ + "type": "bytes", + "hex": hex.EncodeToString(*val.Bytes), + "base64": base64.StdEncoding.EncodeToString(*val.Bytes), + "length": len(*val.Bytes), + }, nil + + case xdr.ScValTypeScvAddress: + if val.Address == nil { + return nil, fmt.Errorf("ScvAddress has nil value") + } + return convertScAddress(*val.Address) + + case xdr.ScValTypeScvVec: + if val.Vec == nil { + return nil, fmt.Errorf("ScvVec has nil value") + } + vec := *val.Vec + result := make([]interface{}, 0, len(*vec)) + for i, item := range *vec { + converted, err := ConvertScValToJSON(item) + if err != nil { + result = append(result, map[string]interface{}{ + "index": i, + "error": err.Error(), + "type": item.Type.String(), + }) + } else { + result = append(result, converted) + } + } + return result, nil + + case xdr.ScValTypeScvMap: + if val.Map == nil { + return nil, fmt.Errorf("ScvMap has nil value") + } + scMap := *val.Map + result := make(map[string]interface{}) + orderedKeys := make([]interface{}, 0, len(*scMap)) + + for _, entry := range *scMap { + key, keyErr := ConvertScValToJSON(entry.Key) + value, valErr := ConvertScValToJSON(entry.Val) + + // Create a string representation of the key for the map + var keyStr string + if keyErr != nil { + keyStr = fmt.Sprintf("error:%s", keyErr.Error()) + } else { + keyStr = fmt.Sprintf("%v", key) + } + + if valErr != nil { + result[keyStr] = map[string]interface{}{ + "error": valErr.Error(), + "type": entry.Val.Type.String(), + } + } else { + result[keyStr] = value + } + + orderedKeys = append(orderedKeys, key) + } + + return map[string]interface{}{ + "type": "map", + "entries": result, + "keys": orderedKeys, + }, nil + + case xdr.ScValTypeScvContractInstance: + if val.Instance == nil { + return nil, fmt.Errorf("ScvContractInstance has nil value") + } + return map[string]interface{}{ + "type": "contract_instance", + "value": "complex_contract_instance", + }, nil + + case xdr.ScValTypeScvLedgerKeyContractInstance: + return map[string]interface{}{ + "type": "ledger_key_contract_instance", + }, nil + + case xdr.ScValTypeScvLedgerKeyNonce: + if val.NonceKey == nil { + return nil, fmt.Errorf("ScvLedgerKeyNonce has nil value") + } + nonceKey := *val.NonceKey + return map[string]interface{}{ + "type": "ledger_key_nonce", + "nonce": nonceKey.Nonce, + }, nil + + default: + return nil, fmt.Errorf("unsupported ScVal type: %s", val.Type.String()) + } +} + +// convertScAddress converts an ScAddress to a JSON-serializable format +func convertScAddress(addr xdr.ScAddress) (interface{}, error) { + switch addr.Type { + case xdr.ScAddressTypeScAddressTypeAccount: + if addr.AccountId == nil { + return nil, fmt.Errorf("ScAddressTypeAccount has nil AccountId") + } + accountID := addr.AccountId.Ed25519 + accountStr, err := strkey.Encode(strkey.VersionByteAccountID, accountID[:]) + if err != nil { + return nil, fmt.Errorf("error encoding account address: %w", err) + } + return map[string]interface{}{ + "type": "account", + "address": accountStr, + }, nil + + case xdr.ScAddressTypeScAddressTypeContract: + if addr.ContractId == nil { + return nil, fmt.Errorf("ScAddressTypeContract has nil ContractId") + } + contractID := *addr.ContractId + contractStr, err := strkey.Encode(strkey.VersionByteContract, contractID[:]) + if err != nil { + return nil, fmt.Errorf("error encoding contract address: %w", err) + } + return map[string]interface{}{ + "type": "contract", + "address": contractStr, + }, nil + + default: + return nil, fmt.Errorf("unknown ScAddress type: %v", addr.Type) + } +} + +// Helper functions for large integer conversions + +func uint128ToString(val xdr.UInt128Parts) string { + hi := big.NewInt(0).SetUint64(uint64(val.Hi)) + lo := big.NewInt(0).SetUint64(uint64(val.Lo)) + hi.Lsh(hi, 64) + hi.Add(hi, lo) + return hi.String() +} + +func int128ToString(val xdr.Int128Parts) string { + // For signed integers, we need to handle the sign bit + hi := big.NewInt(0).SetUint64(uint64(val.Hi)) + lo := big.NewInt(0).SetUint64(uint64(val.Lo)) + + // Check if negative (high bit set) + if uint64(val.Hi)&(uint64(1)<<63) != 0 { + // Two's complement for negative numbers + hi.Sub(hi, big.NewInt(1).Lsh(big.NewInt(1), 64)) + } + + hi.Lsh(hi, 64) + hi.Add(hi, lo) + return hi.String() +} + +func uint256ToString(val xdr.UInt256Parts) string { + hiHi := big.NewInt(0).SetUint64(uint64(val.HiHi)) + hiLo := big.NewInt(0).SetUint64(uint64(val.HiLo)) + loHi := big.NewInt(0).SetUint64(uint64(val.LoHi)) + loLo := big.NewInt(0).SetUint64(uint64(val.LoLo)) + + hiHi.Lsh(hiHi, 192) + hiLo.Lsh(hiLo, 128) + loHi.Lsh(loHi, 64) + + result := big.NewInt(0) + result.Add(result, hiHi) + result.Add(result, hiLo) + result.Add(result, loHi) + result.Add(result, loLo) + + return result.String() +} + +func uint256ToHex(val xdr.UInt256Parts) string { + return fmt.Sprintf("%016x%016x%016x%016x", val.HiHi, val.HiLo, val.LoHi, val.LoLo) +} + +func int256ToString(val xdr.Int256Parts) string { + hiHi := big.NewInt(0).SetUint64(uint64(val.HiHi)) + hiLo := big.NewInt(0).SetUint64(uint64(val.HiLo)) + loHi := big.NewInt(0).SetUint64(uint64(val.LoHi)) + loLo := big.NewInt(0).SetUint64(uint64(val.LoLo)) + + // Check if negative (high bit set in HiHi) + if uint64(val.HiHi)&(uint64(1)<<63) != 0 { + // Two's complement for negative numbers + hiHi.Sub(hiHi, big.NewInt(1).Lsh(big.NewInt(1), 64)) + } + + hiHi.Lsh(hiHi, 192) + hiLo.Lsh(hiLo, 128) + loHi.Lsh(loHi, 64) + + result := big.NewInt(0) + result.Add(result, hiHi) + result.Add(result, hiLo) + result.Add(result, loHi) + result.Add(result, loLo) + + return result.String() +} + +func int256ToHex(val xdr.Int256Parts) string { + return fmt.Sprintf("%016x%016x%016x%016x", val.HiHi, val.HiLo, val.LoHi, val.LoLo) +} + +// GetFunctionNameFromScVal extracts a function name from various ScVal types +func GetFunctionNameFromScVal(val xdr.ScVal) string { + switch val.Type { + case xdr.ScValTypeScvSymbol: + if val.Sym != nil { + return string(*val.Sym) + } + case xdr.ScValTypeScvString: + if val.Str != nil { + return string(*val.Str) + } + case xdr.ScValTypeScvBytes: + if val.Bytes != nil { + return string(*val.Bytes) + } + } + return "" +} \ No newline at end of file