From 506af8e1886950404b268e76375dd90595b139bc Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Thu, 24 Jul 2025 20:01:03 +0000 Subject: [PATCH 1/7] feat: add DIPs (Distributed Indexing Payments) support This squashed commit adds support for DIPs (Distributed Indexing Payments), which allows indexers to receive indexing fees for indexing subgraphs requested via the DIPs system. Key changes: - Add 'dips' as a new IndexingDecisionBasis enum value - Add indexing agreements model and database support - Add DIPs client for interacting with the gateway DIPs service - Support for matching DIPs agreements with allocations - Allow actions on deployments that are not published yet (for DIPs) - Update allocation management to handle DIPs-based allocations - Add proper handling for cancelled agreements Co-authored-by: Multiple contributors from the DIPs development team --- REBASE_STRATEGY.md | 232 +++++++ .../indexer-agent/src/__tests__/indexer.ts | 1 + packages/indexer-agent/src/agent.ts | 166 ++++- packages/indexer-agent/src/commands/start.ts | 36 +- .../19-add-dips-to-decision-basis.ts | 38 ++ packages/indexer-cli/src/__tests__/util.ts | 1 + packages/indexer-common/package.json | 3 + .../src/allocations/__tests__/tap.test.ts | 3 + .../__tests__/validate-queries.test.ts | 3 + .../src/allocations/escrow-accounts.ts | 31 + packages/indexer-common/src/graph-node.ts | 16 + packages/indexer-common/src/index.ts | 2 + .../__tests__/allocations.test.ts | 1 + .../src/indexer-management/__tests__/util.ts | 1 + .../src/indexer-management/allocations.ts | 33 +- .../src/indexer-management/client.ts | 9 +- .../src/indexer-management/models/index.ts | 5 +- .../models/indexing-agreement.ts | 215 +++++++ .../models/indexing-rule.ts | 3 +- .../src/indexer-management/monitor.ts | 11 +- .../resolvers/allocations.ts | 49 +- .../src/indexing-fees/__tests__/dips.test.ts | 599 ++++++++++++++++++ .../indexer-common/src/indexing-fees/dips.ts | 502 +++++++++++++++ .../gateway-dips-service-client.ts | 168 +++++ .../indexer-common/src/indexing-fees/index.ts | 1 + .../src/network-specification.ts | 4 + packages/indexer-common/src/network.ts | 30 +- packages/indexer-common/src/operator.ts | 98 ++- .../indexer-common/src/query-fees/models.ts | 33 +- packages/indexer-common/src/rules.ts | 4 +- packages/indexer-common/src/subgraphs.ts | 9 + 31 files changed, 2226 insertions(+), 81 deletions(-) create mode 100644 REBASE_STRATEGY.md create mode 100644 packages/indexer-agent/src/db/migrations/19-add-dips-to-decision-basis.ts create mode 100644 packages/indexer-common/src/indexer-management/models/indexing-agreement.ts create mode 100644 packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts create mode 100644 packages/indexer-common/src/indexing-fees/dips.ts create mode 100644 packages/indexer-common/src/indexing-fees/gateway-dips-service-client.ts create mode 100644 packages/indexer-common/src/indexing-fees/index.ts diff --git a/REBASE_STRATEGY.md b/REBASE_STRATEGY.md new file mode 100644 index 000000000..eef3a438d --- /dev/null +++ b/REBASE_STRATEGY.md @@ -0,0 +1,232 @@ +# DIPs-Horizon Rebase Strategy + +## Overview +This document tracks the merge conflict resolution strategy for rebasing DIPs (Distributed Indexing Payments) onto the Horizon branch. + +- **DIPs Branch**: Adds distributed indexing payments functionality +- **Horizon Branch**: Adds Graph Horizon protocol upgrade with GraphTally/RAV v2 + +## Conflict Files (8 total) + +### 1. packages/indexer-common/src/network.ts +**Status**: ❌ Unresolved + +**Horizon Changes**: +- Imports `GraphTallyCollector` and `encodeRegistrationData` +- Adds `graphTallyCollector: GraphTallyCollector | undefined` property +- Adds `isHorizon: Eventual` property +- Creates GraphTallyCollector instance for RAV v2 + +**DIPs Changes**: +- Imports `DipsCollector` +- Adds `dipsCollector: DipsCollector | undefined` property +- Adds `queryFeeModels: QueryFeeModels` property +- Adds `managementModels: IndexerManagementModels` property + +**Key Conflicts**: +1. Import section (lines 41-46) +2. Class properties (lines 58-72) +3. Constructor parameters (lines 86-92) +4. Constructor body (lines 106-112) +5. Network instantiation (lines 396-399) + +**Resolution Strategy**: +- [ ] Merge both collectors (GraphTallyCollector AND DipsCollector) +- [ ] Keep all properties from both branches +- [ ] Update constructor to accept all parameters +- [ ] Ensure both collectors can be initialized + +--- + +### 2. packages/indexer-common/src/operator.ts +**Status**: ❌ Unresolved + +**Horizon Changes**: +- `createAllocation` method takes `isHorizon: boolean` parameter +- Uses `isHorizon` to determine `isLegacy` flag on actions +- Passes `isLegacy: !isHorizon` to queueAction +- Also sets `isLegacy: allocation.isLegacy` when closing allocations + +**DIPs Changes**: +- `createAllocation` method takes `forceAction: boolean = false` parameter +- `queueAction` method signature changed to `async queueAction(action: ActionItem, forceAction: boolean = false)` +- Passes forceAction as second parameter to queueAction + +**Key Conflicts**: +1. createAllocation method signature (line 366-370) +2. queueAction calls - Horizon passes object with isLegacy, DIPs passes forceAction as 2nd param +3. closeEligibleAllocations also has forceAction parameter in DIPs +4. refreshExpiredAllocations has similar conflicts + +**Resolution Strategy**: +- [ ] Need both isHorizon AND forceAction parameters in allocation methods +- [ ] Update method signatures: `createAllocation(logger, decision, lastClosed, isHorizon, forceAction = false)` +- [ ] Merge queueAction calls to include both isLegacy (from Horizon) and forceAction (from DIPs) + +--- + +### 3. packages/indexer-common/src/query-fees/models.ts +**Status**: ❌ Unresolved + +**Horizon Changes**: +- Uses simpler Model type: `extends Model` +- id property is `public id!: number` + +**DIPs Changes**: +- Uses Model with creation attributes: `extends Model` +- id property is `public id!: CreationOptional` + +**Key Conflicts**: +- Single conflict at line 28-37 in ScalarTapReceipts class definition + +**Resolution Strategy**: +- [ ] Use DIPs version (more complete typing with CreationOptional) + +--- + +### 4. packages/indexer-common/package.json +**Status**: ❌ Unresolved + +**Horizon Changes**: +- `@graphprotocol/common-ts`: "3.0.1" (newer) +- `@graphprotocol/toolshed`: "0.6.5" +- `@semiotic-labs/tap-contracts-bindings`: "2.0.0" (newer) + +**DIPs Changes**: +- `@graphprotocol/common-ts`: "2.0.11" (older) +- `@semiotic-labs/tap-contracts-bindings`: "^1.2.1" (older) +- Adds DIPs-specific dependencies: + - `@bufbuild/protobuf`: "2.2.3" + - `@graphprotocol/dips-proto`: "0.2.2" + - `@grpc/grpc-js`: "^1.12.6" + +**Key Conflicts**: +- Dependency version mismatches + +**Resolution Strategy**: +- [ ] Use Horizon's newer versions +- [ ] Add DIPs-specific dependencies + +--- + +### 5. packages/indexer-common/src/indexer-management/allocations.ts +**Status**: ❌ Unresolved + +**Horizon Changes**: +- Empty constructor body + +**DIPs Changes**: +- Constructor initializes DipsManager if dipperEndpoint is configured +- Adds `dipsManager: DipsManager | null` property + +**Key Conflicts**: +- Constructor body (lines 131-139) + +**Resolution Strategy**: +- [ ] Keep DIPs initialization logic + +--- + +### 6. packages/indexer-common/src/indexer-management/resolvers/allocations.ts +**Status**: ❌ Unresolved + +**Horizon Changes**: +- Destructures `graphNode` from resolver context + +**DIPs Changes**: +- Destructures `actionManager` from resolver context + +**Key Conflicts**: +- reallocateAllocation resolver context destructuring (lines 1720-1724) + +**Resolution Strategy**: +- [ ] Include BOTH in destructuring: `{ logger, models, multiNetworks, graphNode, actionManager }` +- [ ] The IndexerManagementResolverContext interface already has both properties + +--- + +### 7. packages/indexer-agent/src/agent.ts +**Status**: ❌ Unresolved + +**Horizon Changes**: +- Passes `isHorizon` to createAllocation + +**DIPs Changes**: +- Passes `forceAction` to createAllocation + +**Key Conflicts**: +- createAllocation call (lines 1243-1247) + +**Resolution Strategy**: +- [ ] Pass both parameters: `createAllocation(logger, decision, lastClosed, isHorizon, forceAction)` + +--- + +### 8. yarn.lock +**Status**: ❌ Unresolved + +**Resolution Strategy**: +- [ ] Will regenerate after resolving package.json conflicts + +--- + +## General Notes +- Both branches introduce different payment/collection systems that need to coexist +- Horizon introduces protocol upgrade detection and legacy/horizon mode switching +- DIPs introduces indexing agreements and gateway payment integration + +## Important Context +- **Current branch**: dips-horizon-rebase +- **Base commit**: Squashed DIPs changes into single commit (35ceac2a) on top of 32d8f174 +- **Rebase status**: `git rebase origin/horizon` in progress with conflicts +- **To continue rebase**: After resolving conflicts, use `git add ` then `git rebase --continue` +- **To abort**: `git rebase --abort` if needed + +## Key Files/Imports Added by Each Branch +**Horizon**: +- `GraphTallyCollector` from './allocations/graph-tally-collector' +- `encodeRegistrationData` from '@graphprotocol/toolshed' +- `isHorizon` property for protocol upgrade detection +- `isLegacy` flag on actions + +**DIPs**: +- `DipsCollector` from './indexing-fees/dips' +- `DipsManager` class +- DIPs-specific dependencies in package.json +- `forceAction` parameter for manual allocation management +- New directory: `indexing-fees/` with DIPs implementation + +## Important Note: Method Call Analysis + +**Call sites found for modified methods:** +- `createAllocation`: Only 1 call in agent.ts (already in conflict) +- `closeEligibleAllocations`: 2 calls in agent.ts (already have forceAction parameter) +- `refreshExpiredAllocations`: 1 call in agent.ts (already has forceAction parameter) +- `queueAction`: 5 calls in operator.ts (all in conflicts) + +**Good news**: All method calls appear to be either: +1. Already in merge conflicts (so we'll handle them) +2. Already updated with the DIPs parameters (forceAction) + +**Action needed**: When resolving conflicts, ensure we add BOTH parameters where needed. + +## Resolution Summary + +### High Priority Decisions Needed: +1. **Method Signatures**: Most conflicts are about method parameters. We need both `isHorizon` (from Horizon) AND `forceAction` (from DIPs) +2. **Collectors**: We need both GraphTallyCollector (Horizon) and DipsCollector (DIPs) to coexist +3. **Dependencies**: Use Horizon's newer versions but add DIPs-specific dependencies + +### Recommended Approach: +1. Start with package.json - merge dependencies +2. Fix network.ts - ensure both collectors can exist +3. Fix operator.ts - update method signatures to accept both parameters +4. Fix agent.ts - pass both parameters +5. Fix remaining files with minor conflicts +6. Regenerate yarn.lock + +### Key Principle: +Both payment systems (Horizon's GraphTally and DIPs) should coexist. The system should support: +- Legacy allocations (pre-Horizon) +- Horizon allocations (with GraphTally/RAV v2) +- DIPs agreements (with distributed indexing payments) \ No newline at end of file diff --git a/packages/indexer-agent/src/__tests__/indexer.ts b/packages/indexer-agent/src/__tests__/indexer.ts index c6b1ce463..d417b46ba 100644 --- a/packages/indexer-agent/src/__tests__/indexer.ts +++ b/packages/indexer-agent/src/__tests__/indexer.ts @@ -148,6 +148,7 @@ const setup = async () => { const network = await Network.create( logger, networkSpecification, + models, queryFeeModels, graphNode, metrics, diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index c0a41a2f5..6ff9320db 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -291,6 +291,20 @@ export class Agent { { logger, milliseconds: requestIntervalSmall }, async () => { return this.multiNetworks.map(async ({ network, operator }) => { + if (network.specification.indexerOptions.enableDips) { + // There should be a DipsManager in the operator + if (!operator.dipsManager) { + throw new Error('DipsManager is not available') + } + logger.debug('Ensuring indexing rules for DIPs', { + protocolNetwork: network.specification.networkIdentifier, + }) + await operator.dipsManager.ensureAgreementRules() + } else { + logger.debug( + 'DIPs is disabled, skipping indexing rule enforcement', + ) + } logger.trace('Fetching indexing rules', { protocolNetwork: network.specification.networkIdentifier, }) @@ -324,12 +338,21 @@ export class Agent { }, ) - // Skip fetching active deployments if the deployment management mode is manual and POI tracking is disabled + // Skip fetching active deployments if the deployment management mode is manual, DIPs is disabled, and POI tracking is disabled const activeDeployments: Eventual = sequentialTimerMap( { logger, milliseconds: requestIntervalLarge }, async () => { - if (this.deploymentManagement === DeploymentManagementMode.AUTO) { + let dipsEnabled = false + await this.multiNetworks.map(async ({ network }) => { + if (network.specification.indexerOptions.enableDips) { + dipsEnabled = true + } + }) + if ( + this.deploymentManagement === DeploymentManagementMode.AUTO || + dipsEnabled + ) { logger.debug('Fetching active deployments') const assignments = await this.graphNode.subgraphDeploymentsAssignments( @@ -338,7 +361,7 @@ export class Agent { return assignments.map(assignment => assignment.id) } else { logger.info( - "Skipping fetching active deployments fetch since DeploymentManagementMode = 'manual' and POI tracking is disabled", + "Skipping fetching active deployments fetch since DeploymentManagementMode = 'manual' and DIPs is disabled", ) return [] } @@ -351,37 +374,63 @@ export class Agent { }, ) - const networkDeployments: Eventual> = - sequentialTimerMap( - { logger, milliseconds: requestIntervalSmall }, - async () => - await this.multiNetworks.map(({ network }) => { - logger.trace('Fetching network deployments', { - protocolNetwork: network.specification.networkIdentifier, - }) - return network.networkMonitor.subgraphDeployments() - }), - { - onError: error => - logger.warn( - `Failed to obtain network deployments, trying again later`, - { error }, - ), - }, - ) + const networkAndDipsDeployments: Eventual< + NetworkMapped + > = sequentialTimerMap( + { logger, milliseconds: requestIntervalSmall }, + async () => + await this.multiNetworks.map(async ({ network, operator }) => { + logger.trace('Fetching network deployments', { + protocolNetwork: network.specification.networkIdentifier, + }) + const deployments = network.networkMonitor.subgraphDeployments() + if (network.specification.indexerOptions.enableDips) { + if (!operator.dipsManager) { + throw new Error('DipsManager is not available') + } + const resolvedDeployments = await deployments + const dipsDeployments = await Promise.all( + (await operator.dipsManager.getActiveDipsDeployments()).map( + deployment => + network.networkMonitor.subgraphDeployment( + deployment.ipfsHash, + ), + ), + ) + for (const deployment of dipsDeployments) { + if ( + resolvedDeployments.find( + d => d.id.bytes32 === deployment.id.bytes32, + ) == null + ) { + resolvedDeployments.push(deployment) + } + } + return resolvedDeployments + } + return deployments + }), + { + onError: error => + logger.warn( + `Failed to obtain network deployments, trying again later`, + { error }, + ), + }, + ) const networkDeploymentAllocationDecisions: Eventual< NetworkMapped > = join({ - networkDeployments, + networkAndDipsDeployments, indexingRules, }).tryMap( - async ({ indexingRules, networkDeployments }) => { + async ({ indexingRules, networkAndDipsDeployments }) => { return this.multiNetworks.mapNetworkMapped( - this.multiNetworks.zip(indexingRules, networkDeployments), + this.multiNetworks.zip(indexingRules, networkAndDipsDeployments), async ( { network }: NetworkAndOperator, - [indexingRules, networkDeployments]: [ + [indexingRules, networkAndDipsDeployments]: [ IndexingRuleAttributes[], SubgraphDeployment[], ], @@ -405,7 +454,11 @@ export class Agent { logger.trace('Evaluating which deployments are worth allocating to') return indexingRules.length === 0 ? [] - : evaluateDeployments(logger, networkDeployments, indexingRules) + : evaluateDeployments( + logger, + networkAndDipsDeployments, + indexingRules, + ) }, ) }, @@ -599,9 +652,42 @@ export class Agent { } break case DeploymentManagementMode.MANUAL: - this.logger.debug( - `Skipping subgraph deployment reconciliation since DeploymentManagementMode = 'manual'`, - ) + await this.multiNetworks.map(async ({ network, operator }) => { + if (network.specification.indexerOptions.enableDips) { + // Reconcile DIPs deployments anyways + this.logger.warn( + `Deployment management is manual, but DIPs is enabled. Reconciling DIPs deployments anyways.`, + ) + if (!operator.dipsManager) { + throw new Error('DipsManager is not available') + } + const dipsDeployments = + await operator.dipsManager.getActiveDipsDeployments() + const newTargetDeployments = new Set([ + ...activeDeployments, + ...dipsDeployments, + ]) + try { + await this.reconcileDeployments( + activeDeployments, + Array.from(newTargetDeployments), + eligibleAllocations, + ) + } catch (err) { + logger.warn( + `Exited early while reconciling deployments. Skipped reconciling actions.`, + { + err: indexerError(IndexerErrorCode.IE005, err), + }, + ) + return + } + } else { + this.logger.debug( + `Skipping subgraph deployment reconciliation since DeploymentManagementMode = 'manual'`, + ) + } + }) break default: throw new Error( @@ -622,6 +708,23 @@ export class Agent { }) return } + + await this.multiNetworks.mapNetworkMapped( + activeAllocations, + async ({ network, operator }, activeAllocations: Allocation[]) => { + if (network.specification.indexerOptions.enableDips) { + if (!operator.dipsManager) { + throw new Error('DipsManager is not available') + } + this.logger.debug( + `Matching agreement allocations for network ${network.specification.networkIdentifier}`, + ) + await operator.dipsManager.matchAgreementAllocations( + activeAllocations, + ) + } + }, + ) }, ) } @@ -948,6 +1051,7 @@ export class Agent { maxAllocationDuration: HorizonTransitionValue, network: Network, operator: Operator, + forceAction: boolean = false, ): Promise { const logger = this.logger.child({ deployment: deploymentAllocationDecision.deployment.ipfsHash, @@ -971,6 +1075,7 @@ export class Agent { logger, deploymentAllocationDecision, activeDeploymentAllocations, + forceAction, ) case true: { // If no active allocations and subgraph health passes safety check, create one @@ -1008,6 +1113,7 @@ export class Agent { deploymentAllocationDecision, mostRecentlyClosedAllocation, isHorizon, + forceAction, ) } } else if (activeDeploymentAllocations.length > 0) { @@ -1016,6 +1122,7 @@ export class Agent { logger, deploymentAllocationDecision, activeDeploymentAllocations, + forceAction, ) } else { // Refresh any expiring allocations @@ -1032,6 +1139,7 @@ export class Agent { logger, deploymentAllocationDecision, expiringAllocations, + forceAction, ) } } diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts index f06492976..655582d48 100644 --- a/packages/indexer-agent/src/commands/start.ts +++ b/packages/indexer-agent/src/commands/start.ts @@ -361,6 +361,26 @@ export const start = { required: false, group: 'Indexer Infrastructure', }) + .option('enable-dips', { + description: 'Whether to enable Indexing Fees (DIPs)', + type: 'boolean', + default: false, + group: 'Indexing Fees ("DIPs")', + }) + .option('dipper-endpoint', { + description: 'Gateway endpoint for DIPs receipts', + type: 'string', + array: false, + required: false, + group: 'Indexing Fees ("DIPs")', + }) + .option('dips-allocation-amount', { + description: 'Amount of GRT to allocate for DIPs', + type: 'number', + default: 1, + required: false, + group: 'Indexing Fees ("DIPs")', + }) .check(argv => { if ( !argv['network-subgraph-endpoint'] && @@ -388,6 +408,9 @@ export const start = { ) { return 'Invalid --rebate-claim-max-batch-size provided. Must be > 0 and an integer.' } + if (argv['enable-dips'] && !argv['dipper-endpoint']) { + return 'Invalid --dipper-endpoint provided. Must be provided when --enable-dips is true.' + } return true }) }, @@ -428,6 +451,10 @@ export async function createNetworkSpecification( maxProvisionInitialSize: argv.maxProvisionInitialSize, finalityTime: argv.chainFinalizeTime, legacyMnemonics: argv.legacyMnemonics, + enableDips: argv.enableDips, + dipperEndpoint: argv.dipperEndpoint, + dipsAllocationAmount: argv.dipsAllocationAmount, + dipsEpochsMargin: argv.dipsEpochsMargin, } const transactionMonitoring = { @@ -653,7 +680,14 @@ export async function run( const networks: Network[] = await pMap( networkSpecifications, async (spec: NetworkSpecification) => - Network.create(logger, spec, queryFeeModels, graphNode, metrics), + Network.create( + logger, + spec, + managementModels, + queryFeeModels, + graphNode, + metrics, + ), ) // -------------------------------------------------------------------------------- diff --git a/packages/indexer-agent/src/db/migrations/19-add-dips-to-decision-basis.ts b/packages/indexer-agent/src/db/migrations/19-add-dips-to-decision-basis.ts new file mode 100644 index 000000000..793ef3ab3 --- /dev/null +++ b/packages/indexer-agent/src/db/migrations/19-add-dips-to-decision-basis.ts @@ -0,0 +1,38 @@ +import type { Logger } from '@graphprotocol/common-ts' +import type { QueryInterface } from 'sequelize' + +interface MigrationContext { + queryInterface: QueryInterface + logger: Logger +} + +interface Context { + context: MigrationContext +} + +export async function up({ context }: Context): Promise { + const { queryInterface, logger } = context + + if (await queryInterface.tableExists('IndexingRules')) { + logger.debug('Adding dips to decision basis') + + await queryInterface.sequelize.query( + `ALTER TYPE "enum_IndexingRules_decisionBasis" ADD VALUE 'dips'`, + ) + } else { + logger.debug('IndexingRules table does not exist, skipping migration') + } + + logger.info('Migration completed') +} + +export async function down({ context }: Context): Promise { + const { queryInterface, logger } = context + + logger.info('Removing dips from decision basis') + await queryInterface.sequelize.query( + `ALTER TYPE "enum_IndexingRules_decisionBasis" DROP VALUE 'dips'`, + ) + + logger.info('Migration completed') +} diff --git a/packages/indexer-cli/src/__tests__/util.ts b/packages/indexer-cli/src/__tests__/util.ts index 41df97e13..51dc630bf 100644 --- a/packages/indexer-cli/src/__tests__/util.ts +++ b/packages/indexer-cli/src/__tests__/util.ts @@ -108,6 +108,7 @@ export const setup = async (multiNetworksEnabled: boolean) => { const network = await Network.create( logger, testNetworkSpecification, + models, queryFeeModels, graphNode, metrics, diff --git a/packages/indexer-common/package.json b/packages/indexer-common/package.json index e591bb5cc..f99e829b6 100644 --- a/packages/indexer-common/package.json +++ b/packages/indexer-common/package.json @@ -23,8 +23,11 @@ }, "dependencies": { "@pinax/graph-networks-registry": "0.6.7", + "@bufbuild/protobuf": "2.2.3", "@graphprotocol/common-ts": "3.0.1", + "@graphprotocol/dips-proto": "0.2.2", "@graphprotocol/toolshed": "1.1.1", + "@grpc/grpc-js": "^1.12.6", "@semiotic-labs/tap-contracts-bindings": "2.0.0", "@thi.ng/heaps": "1.2.38", "@types/lodash.clonedeep": "^4.5.7", diff --git a/packages/indexer-common/src/allocations/__tests__/tap.test.ts b/packages/indexer-common/src/allocations/__tests__/tap.test.ts index 8a748d89a..fca726220 100644 --- a/packages/indexer-common/src/allocations/__tests__/tap.test.ts +++ b/packages/indexer-common/src/allocations/__tests__/tap.test.ts @@ -7,6 +7,7 @@ import { TapSubgraphResponse, TapCollector, Allocation, + defineIndexerManagementModels, } from '@graphprotocol/indexer-common' import { Address, @@ -43,6 +44,7 @@ const setup = async () => { // Clearing the registry prevents duplicate metric registration in the default registry. metrics.registry.clear() sequelize = await connectDatabase(__DATABASE__) + const models = defineIndexerManagementModels(sequelize) queryFeeModels = defineQueryFeeModels(sequelize) sequelize = await sequelize.sync({ force: true }) @@ -58,6 +60,7 @@ const setup = async () => { const network = await Network.create( logger, testNetworkSpecification, + models, queryFeeModels, graphNode, metrics, diff --git a/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts b/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts index 85c76fdc1..dbeeb9968 100644 --- a/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts +++ b/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts @@ -1,4 +1,5 @@ import { + defineIndexerManagementModels, defineQueryFeeModels, GraphNode, Network, @@ -36,6 +37,7 @@ const setup = async () => { // Clearing the registry prevents duplicate metric registration in the default registry. metrics.registry.clear() sequelize = await connectDatabase(__DATABASE__) + const models = defineIndexerManagementModels(sequelize) queryFeeModels = defineQueryFeeModels(sequelize) sequelize = await sequelize.sync({ force: true }) @@ -51,6 +53,7 @@ const setup = async () => { const network = await Network.create( logger, testNetworkSpecification, + models, queryFeeModels, graphNode, metrics, diff --git a/packages/indexer-common/src/allocations/escrow-accounts.ts b/packages/indexer-common/src/allocations/escrow-accounts.ts index 1d126fd94..45402171c 100644 --- a/packages/indexer-common/src/allocations/escrow-accounts.ts +++ b/packages/indexer-common/src/allocations/escrow-accounts.ts @@ -13,6 +13,14 @@ export type EscrowAccountResponse = { }[] } +export type EscrowSenderResponse = { + signer: { + sender: { + id: string + } + } +} + export class EscrowAccounts { constructor(private sendersBalances: Map) {} @@ -65,3 +73,26 @@ export const getEscrowAccounts = async ( } return EscrowAccounts.fromResponse(result.data) } + +export const getEscrowSenderForSigner = async ( + tapSubgraph: SubgraphClient, + signer: Address, +): Promise
=> { + const signerLower = signer.toLowerCase() + const result = await tapSubgraph.query( + gql` + query EscrowAccountQuery($signer: ID!) { + signer(id: $signer) { + sender { + id + } + } + } + `, + { signer: signerLower }, + ) + if (!result.data) { + throw `There was an error while querying Tap Subgraph. Errors: ${result.error}` + } + return toAddress(result.data.signer.sender.id) +} diff --git a/packages/indexer-common/src/graph-node.ts b/packages/indexer-common/src/graph-node.ts index e06b2ff81..836985514 100644 --- a/packages/indexer-common/src/graph-node.ts +++ b/packages/indexer-common/src/graph-node.ts @@ -1095,6 +1095,22 @@ export class GraphNode { } } + public async entityCount(deployments: SubgraphDeploymentID[]): Promise { + // Query the entity count for each deployment using the indexingStatuses query + const query = ` + query entityCounts($deployments: [String!]!) { + indexingStatuses(subgraphs: $deployments) { + entityCount + } + } + ` + const result = await this.status + .query(query, { deployments: deployments.map((id) => id.ipfsHash) }) + .toPromise() + + return result.data.indexingStatuses.map((status) => status.entityCount) as number[] + } + public async proofOfIndexing( deployment: SubgraphDeploymentID, block: BlockPointer, diff --git a/packages/indexer-common/src/index.ts b/packages/indexer-common/src/index.ts index ab3eedd97..a74020745 100644 --- a/packages/indexer-common/src/index.ts +++ b/packages/indexer-common/src/index.ts @@ -3,6 +3,7 @@ export * from './allocations' export * from './async-cache' export * from './errors' export * from './indexer-management' +export * from './indexing-fees' export * from './graph-node' export * from './operator' export * from './network' @@ -17,3 +18,4 @@ export * from './parsers' export * as specification from './network-specification' export * from './multi-networks' export * from './sequential-timer' +export * from './indexing-fees' diff --git a/packages/indexer-common/src/indexer-management/__tests__/allocations.test.ts b/packages/indexer-common/src/indexer-management/__tests__/allocations.test.ts index e6d2ce5a9..f94e77d7f 100644 --- a/packages/indexer-common/src/indexer-management/__tests__/allocations.test.ts +++ b/packages/indexer-common/src/indexer-management/__tests__/allocations.test.ts @@ -64,6 +64,7 @@ const setup = async () => { const network = await Network.create( logger, testNetworkSpecification, + managementModels, queryFeeModels, graphNode, metrics, diff --git a/packages/indexer-common/src/indexer-management/__tests__/util.ts b/packages/indexer-common/src/indexer-management/__tests__/util.ts index 0c832674e..b25ef6c88 100644 --- a/packages/indexer-common/src/indexer-management/__tests__/util.ts +++ b/packages/indexer-common/src/indexer-management/__tests__/util.ts @@ -59,6 +59,7 @@ export const createTestManagementClient = async ( const network = await Network.create( logger, networkSpecification, + managementModels, queryFeeModels, graphNode, metrics, diff --git a/packages/indexer-common/src/indexer-management/allocations.ts b/packages/indexer-common/src/indexer-management/allocations.ts index 5a363ceba..6be66c0c5 100644 --- a/packages/indexer-common/src/indexer-management/allocations.ts +++ b/packages/indexer-common/src/indexer-management/allocations.ts @@ -15,6 +15,7 @@ import { AllocationStatus, CloseAllocationResult, CreateAllocationResult, + DipsManager, fetchIndexingRules, GraphNode, indexerError, @@ -159,12 +160,17 @@ export function encodeCollectData(allocationId: string, poiData: POIData): strin } export class AllocationManager { + declare dipsManager: DipsManager | null constructor( private logger: Logger, private models: IndexerManagementModels, private graphNode: GraphNode, private network: Network, - ) {} + ) { + if (this.network.specification.indexerOptions.dipperEndpoint) { + this.dipsManager = new DipsManager(this.logger, this.models, this.network, this) + } + } async executeBatch( actions: Action[], @@ -943,6 +949,14 @@ export class AllocationManager { await upsertIndexingRule(logger, this.models, indexingRule) } + if (this.dipsManager) { + await this.dipsManager.tryUpdateAgreementAllocation( + deployment, + null, + toAddress(createAllocationEventLogs.allocationID), + ) + } + return { actionID, type: 'allocate', @@ -1187,6 +1201,15 @@ export class AllocationManager { await upsertIndexingRule(logger, this.models, neverIndexingRule) + if (this.dipsManager) { + await this.dipsManager.tryCancelAgreement(allocationID) + await this.dipsManager.tryUpdateAgreementAllocation( + allocation.subgraphDeployment.id.toString(), + toAddress(allocationID), + null, + ) + } + return { actionID, type: 'unallocate', @@ -1979,6 +2002,14 @@ export class AllocationManager { await upsertIndexingRule(logger, this.models, indexingRule) } + if (this.dipsManager) { + await this.dipsManager.tryUpdateAgreementAllocation( + subgraphDeploymentID.toString(), + toAddress(allocationID), + toAddress(createAllocationEventLogs.allocationID), + ) + } + return { actionID, type: 'reallocate', diff --git a/packages/indexer-common/src/indexer-management/client.ts b/packages/indexer-common/src/indexer-management/client.ts index e6ad67708..041f7d65f 100644 --- a/packages/indexer-common/src/indexer-management/client.ts +++ b/packages/indexer-common/src/indexer-management/client.ts @@ -44,6 +44,7 @@ const SCHEMA_SDL = gql` never always offchain + dips } enum IdentifierType { @@ -572,9 +573,11 @@ export interface IndexerManagementClientOptions { graphNode: GraphNode multiNetworks: MultiNetworks | undefined defaults: IndexerManagementDefaults + actionManager?: ActionManager | undefined } export class IndexerManagementClient extends Client { + declare actionManager: ActionManager | undefined private logger?: Logger private models: IndexerManagementModels @@ -583,6 +586,7 @@ export class IndexerManagementClient extends Client { this.logger = options.logger this.models = options.models + this.actionManager = options.actionManager } } @@ -627,5 +631,8 @@ export const createIndexerManagementClient = async ( context, }) - return new IndexerManagementClient({ url: 'no-op', exchanges: [exchange] }, options) + return new IndexerManagementClient( + { url: 'no-op', exchanges: [exchange] }, + { ...options, actionManager }, + ) } diff --git a/packages/indexer-common/src/indexer-management/models/index.ts b/packages/indexer-common/src/indexer-management/models/index.ts index 8d5ec55af..81a59f4d3 100644 --- a/packages/indexer-common/src/indexer-management/models/index.ts +++ b/packages/indexer-common/src/indexer-management/models/index.ts @@ -4,6 +4,7 @@ import { IndexingRuleModels, defineIndexingRuleModels } from './indexing-rule' import { CostModelModels, defineCostModelModels } from './cost-model' import { POIDisputeModels, definePOIDisputeModels } from './poi-dispute' import { ActionModels, defineActionModels } from './action' +import { defineIndexingFeesModels, IndexingFeesModels } from './indexing-agreement' export * from './cost-model' export * from './indexing-rule' @@ -13,7 +14,8 @@ export * from './action' export type IndexerManagementModels = IndexingRuleModels & CostModelModels & POIDisputeModels & - ActionModels + ActionModels & + IndexingFeesModels export const defineIndexerManagementModels = ( sequelize: Sequelize, @@ -24,4 +26,5 @@ export const defineIndexerManagementModels = ( defineIndexingRuleModels(sequelize), definePOIDisputeModels(sequelize), defineActionModels(sequelize), + defineIndexingFeesModels(sequelize), ) diff --git a/packages/indexer-common/src/indexer-management/models/indexing-agreement.ts b/packages/indexer-common/src/indexer-management/models/indexing-agreement.ts new file mode 100644 index 000000000..462456a7c --- /dev/null +++ b/packages/indexer-common/src/indexer-management/models/indexing-agreement.ts @@ -0,0 +1,215 @@ +import { toAddress, Address } from '@graphprotocol/common-ts' +import { + DataTypes, + Sequelize, + Model, + CreationOptional, + InferCreationAttributes, + InferAttributes, +} from 'sequelize' + +// Indexing Fees AKA "DIPs" + +export class IndexingAgreement extends Model< + InferAttributes, + InferCreationAttributes +> { + declare id: CreationOptional + declare signature: Buffer + declare signed_payload: Buffer + declare protocol_network: string + declare chain_id: string + declare base_price_per_epoch: string + declare price_per_entity: string + declare subgraph_deployment_id: string + declare service: string + declare payee: string + declare payer: string + declare deadline: Date + declare duration_epochs: bigint + declare max_initial_amount: string + declare max_ongoing_amount_per_epoch: string + declare min_epochs_per_collection: bigint + declare max_epochs_per_collection: bigint + declare created_at: Date + declare updated_at: Date + declare cancelled_at: Date | null + declare signed_cancellation_payload: Buffer | null + declare current_allocation_id: string | null + declare last_allocation_id: string | null + declare last_payment_collected_at: Date | null +} + +export interface IndexingFeesModels { + IndexingAgreement: typeof IndexingAgreement +} + +export const defineIndexingFeesModels = (sequelize: Sequelize): IndexingFeesModels => { + IndexingAgreement.init( + { + id: { + type: DataTypes.UUID, + primaryKey: true, + }, + signature: { + type: DataTypes.BLOB, + allowNull: false, + unique: true, + }, + signed_payload: { + type: DataTypes.BLOB, + allowNull: false, + }, + protocol_network: { + type: DataTypes.STRING(255), + allowNull: false, + }, + chain_id: { + type: DataTypes.STRING(255), + allowNull: false, + }, + base_price_per_epoch: { + type: DataTypes.DECIMAL(39), + allowNull: false, + }, + price_per_entity: { + type: DataTypes.DECIMAL(39), + allowNull: false, + }, + subgraph_deployment_id: { + type: DataTypes.STRING(255), + allowNull: false, + }, + service: { + type: DataTypes.CHAR(40), + allowNull: false, + get() { + const rawValue = this.getDataValue('service') + return toAddress(rawValue) + }, + set(value: Address) { + const addressWithoutPrefix = value.toLowerCase().replace('0x', '') + this.setDataValue('service', addressWithoutPrefix) + }, + }, + payee: { + type: DataTypes.CHAR(40), + allowNull: false, + get() { + const rawValue = this.getDataValue('payee') + return toAddress(rawValue) + }, + set(value: Address) { + const addressWithoutPrefix = value.toLowerCase().replace('0x', '') + this.setDataValue('payee', addressWithoutPrefix) + }, + }, + payer: { + type: DataTypes.CHAR(40), + allowNull: false, + get() { + const rawValue = this.getDataValue('payer') + return toAddress(rawValue) + }, + set(value: Address) { + const addressWithoutPrefix = value.toLowerCase().replace('0x', '') + this.setDataValue('payer', addressWithoutPrefix) + }, + }, + deadline: { + type: DataTypes.DATE, + allowNull: false, + }, + duration_epochs: { + type: DataTypes.BIGINT, + allowNull: false, + }, + max_initial_amount: { + type: DataTypes.DECIMAL(39), + allowNull: false, + }, + max_ongoing_amount_per_epoch: { + type: DataTypes.DECIMAL(39), + allowNull: false, + }, + min_epochs_per_collection: { + type: DataTypes.BIGINT, + allowNull: false, + }, + max_epochs_per_collection: { + type: DataTypes.BIGINT, + allowNull: false, + }, + created_at: { + type: DataTypes.DATE, + allowNull: false, + }, + updated_at: { + type: DataTypes.DATE, + allowNull: false, + }, + cancelled_at: { + type: DataTypes.DATE, + allowNull: true, + }, + signed_cancellation_payload: { + type: DataTypes.BLOB, + allowNull: true, + }, + current_allocation_id: { + type: DataTypes.CHAR(40), + allowNull: true, + get() { + const rawValue = this.getDataValue('current_allocation_id') + if (!rawValue) { + return null + } + return toAddress(rawValue) + }, + set(value: Address | null) { + if (!value) { + this.setDataValue('current_allocation_id', null) + } else { + const addressWithoutPrefix = value.toLowerCase().replace('0x', '') + this.setDataValue('current_allocation_id', addressWithoutPrefix) + } + }, + }, + last_allocation_id: { + type: DataTypes.CHAR(40), + allowNull: true, + get() { + const rawValue = this.getDataValue('last_allocation_id') + if (!rawValue) { + return null + } + return toAddress(rawValue) + }, + set(value: Address | null) { + if (!value) { + this.setDataValue('last_allocation_id', null) + } else { + const addressWithoutPrefix = value.toLowerCase().replace('0x', '') + this.setDataValue('last_allocation_id', addressWithoutPrefix) + } + }, + }, + last_payment_collected_at: { + type: DataTypes.DATE, + allowNull: true, + }, + }, + { + modelName: 'IndexingAgreement', + sequelize, + tableName: 'indexing_agreements', + timestamps: true, + createdAt: 'created_at', + updatedAt: 'updated_at', + }, + ) + + return { + ['IndexingAgreement']: IndexingAgreement, + } +} diff --git a/packages/indexer-common/src/indexer-management/models/indexing-rule.ts b/packages/indexer-common/src/indexer-management/models/indexing-rule.ts index 594191537..6ee5fca39 100644 --- a/packages/indexer-common/src/indexer-management/models/indexing-rule.ts +++ b/packages/indexer-common/src/indexer-management/models/indexing-rule.ts @@ -9,6 +9,7 @@ export enum IndexingDecisionBasis { NEVER = 'never', ALWAYS = 'always', OFFCHAIN = 'offchain', + DIPS = 'dips', } export const INDEXING_RULE_GLOBAL = 'global' @@ -245,7 +246,7 @@ export const defineIndexingRuleModels = (sequelize: Sequelize): IndexingRuleMode allowNull: true, }, decisionBasis: { - type: DataTypes.ENUM('rules', 'never', 'always', 'offchain'), + type: DataTypes.ENUM('rules', 'never', 'always', 'offchain', 'dips'), allowNull: false, defaultValue: 'rules', }, diff --git a/packages/indexer-common/src/indexer-management/monitor.ts b/packages/indexer-common/src/indexer-management/monitor.ts index eb39634d5..4af48d55c 100644 --- a/packages/indexer-common/src/indexer-management/monitor.ts +++ b/packages/indexer-common/src/indexer-management/monitor.ts @@ -647,7 +647,7 @@ export class NetworkMonitor { return subgraphs } - async subgraphDeployment(ipfsHash: string): Promise { + async subgraphDeployment(ipfsHash: string): Promise { try { const result = await this.networkSubgraph.checkedQuery( gql` @@ -684,7 +684,14 @@ export class NetworkMonitor { this.logger.warn( `SubgraphDeployment with ipfsHash = ${ipfsHash} not found on chain`, ) - return undefined + return { + id: new SubgraphDeploymentID(ipfsHash), + deniedAt: 1, // We assume the deployment won't be eligible for rewards if it's not found + stakedTokens: BigNumber.from(0), + signalledTokens: BigNumber.from(0), + queryFeesAmount: BigNumber.from(0), + protocolNetwork: this.networkCAIPID, + } } return parseGraphQLSubgraphDeployment( diff --git a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts index b35e81cc5..77cb2d812 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts @@ -1529,7 +1529,13 @@ export default { amount: string protocolNetwork: string }, - { multiNetworks, graphNode, logger, models }: IndexerManagementResolverContext, + { + multiNetworks, + graphNode, + logger, + models, + actionManager, + }: IndexerManagementResolverContext, ): Promise => { logger.debug('Execute createAllocation() mutation', { deployment, @@ -1626,6 +1632,16 @@ export default { await models.IndexingRule.upsert(indexingRule) + const allocationManager = + actionManager?.allocationManagers[network.specification.networkIdentifier] + if (allocationManager?.dipsManager) { + await allocationManager.dipsManager.tryUpdateAgreementAllocation( + deployment, + null, + toAddress(createAllocationEventLogs.allocationID), + ) + } + // Since upsert succeeded, we _must_ have a rule const updatedRule = await models.IndexingRule.findOne({ where: { identifier: indexingRule.identifier }, @@ -1670,7 +1686,7 @@ export default { force: boolean protocolNetwork: string }, - { logger, models, multiNetworks }: IndexerManagementResolverContext, + { logger, models, multiNetworks, actionManager }: IndexerManagementResolverContext, ): Promise => { logger.debug('Execute closeAllocation() mutation', { allocationID: allocation, @@ -1746,6 +1762,17 @@ export default { await models.IndexingRule.upsert(offchainIndexingRule) + const allocationManager = + actionManager?.allocationManagers[network.specification.networkIdentifier] + if (allocationManager?.dipsManager) { + await allocationManager.dipsManager.tryCancelAgreement(allocation) + await allocationManager.dipsManager.tryUpdateAgreementAllocation( + allocationData.subgraphDeployment.id.toString(), + toAddress(allocation), + null, + ) + } + // Since upsert succeeded, we _must_ have a rule const updatedRule = await models.IndexingRule.findOne({ where: { identifier: offchainIndexingRule.identifier }, @@ -1789,7 +1816,13 @@ export default { force: boolean protocolNetwork: string }, - { logger, models, multiNetworks, graphNode }: IndexerManagementResolverContext, + { + logger, + models, + multiNetworks, + graphNode, + actionManager, + }: IndexerManagementResolverContext, ): Promise => { logger = logger.child({ component: 'reallocateAllocationResolver', @@ -1912,6 +1945,16 @@ export default { await models.IndexingRule.upsert(indexingRule) + const allocationManager = + actionManager?.allocationManagers[network.specification.networkIdentifier] + if (allocationManager?.dipsManager) { + await allocationManager.dipsManager.tryUpdateAgreementAllocation( + allocationData.subgraphDeployment.id.toString(), + toAddress(allocation), + toAddress(createAllocationEventLogs.allocationID), + ) + } + // Since upsert succeeded, we _must_ have a rule const updatedRule = await models.IndexingRule.findOne({ where: { identifier: indexingRule.identifier }, diff --git a/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts b/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts new file mode 100644 index 000000000..be56353f9 --- /dev/null +++ b/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts @@ -0,0 +1,599 @@ +import { + DipsManager, + GraphNode, + IndexerManagementModels, + Network, + QueryFeeModels, + defineIndexerManagementModels, + defineQueryFeeModels, + SubgraphIdentifierType, + IndexingDecisionBasis, + AllocationManager, + DipsCollector, + TapCollector, + createIndexerManagementClient, + Operator, + ActionManager, + IndexerManagementClient, + MultiNetworks, +} from '@graphprotocol/indexer-common' +import { + connectDatabase, + createLogger, + createMetrics, + Logger, + Metrics, + parseGRT, + SubgraphDeploymentID, + toAddress, +} from '@graphprotocol/common-ts' +import { Sequelize } from 'sequelize' +import { testNetworkSpecification } from '../../indexer-management/__tests__/util' +import { BigNumber } from 'ethers' +import { CollectPaymentStatus } from '@graphprotocol/dips-proto/generated/gateway' + +// Make global Jest variables available +// eslint-disable-next-line @typescript-eslint/no-explicit-any +declare const __DATABASE__: any +declare const __LOG_LEVEL__: never + +// Add these type declarations after the existing imports +let sequelize: Sequelize +let logger: Logger +let metrics: Metrics +let graphNode: GraphNode +let managementModels: IndexerManagementModels +let queryFeeModels: QueryFeeModels +let network: Network +let multiNetworks: MultiNetworks +let dipsCollector: DipsCollector +let indexerManagementClient: IndexerManagementClient +let operator: Operator +const networkSpecWithDips = { + ...testNetworkSpecification, + indexerOptions: { + ...testNetworkSpecification.indexerOptions, + enableDips: true, + dipperEndpoint: 'https://test-dipper-endpoint.xyz', + dipsAllocationAmount: parseGRT('1.0'), // Amount of GRT to allocate for DIPs + dipsEpochsMargin: 1, // Optional: Number of epochs margin for DIPs + }, +} + +const mockSubgraphDeployment = (id: string) => { + return { + id: new SubgraphDeploymentID(id), + ipfsHash: id, + deniedAt: null, + stakedTokens: BigNumber.from('1000'), + signalledTokens: BigNumber.from('1000'), + queryFeesAmount: BigNumber.from('0'), + protocolNetwork: 'eip155:421614', + } +} + +jest.spyOn(TapCollector.prototype, 'startRAVProcessing').mockImplementation(() => {}) +const startCollectionLoop = jest + .spyOn(DipsCollector.prototype, 'startCollectionLoop') + .mockImplementation(() => {}) +jest.spyOn(ActionManager.prototype, 'monitorQueue').mockImplementation(async () => {}) +const setup = async () => { + logger = createLogger({ + name: 'DIPs Test Logger', + async: false, + level: __LOG_LEVEL__ ?? 'error', + }) + metrics = createMetrics() + // Clearing the registry prevents duplicate metric registration in the default registry. + metrics.registry.clear() + + graphNode = new GraphNode( + logger, + 'https://test-admin-endpoint.xyz', + 'https://test-query-endpoint.xyz', + 'https://test-status-endpoint.xyz', + 'https://test-ipfs-endpoint.xyz', + ) + + sequelize = await connectDatabase(__DATABASE__) + managementModels = defineIndexerManagementModels(sequelize) + queryFeeModels = defineQueryFeeModels(sequelize) + sequelize = await sequelize.sync({ force: true }) + + network = await Network.create( + logger, + networkSpecWithDips, + managementModels, + queryFeeModels, + graphNode, + metrics, + ) + + multiNetworks = new MultiNetworks( + [network], + (n: Network) => n.specification.networkIdentifier, + ) + + dipsCollector = network.dipsCollector! + indexerManagementClient = await createIndexerManagementClient({ + models: managementModels, + graphNode, + logger, + defaults: { + globalIndexingRule: { + allocationAmount: parseGRT('1000'), + parallelAllocations: 1, + }, + }, + multiNetworks, + }) + + operator = new Operator(logger, indexerManagementClient, networkSpecWithDips) +} + +const ensureGlobalIndexingRule = async () => { + await operator.ensureGlobalIndexingRule() + logger.debug('Ensured global indexing rule') +} + +const setupEach = async () => { + sequelize = await sequelize.sync({ force: true }) + await ensureGlobalIndexingRule() +} + +const teardownEach = async () => { + // Clear out query fee model tables + await queryFeeModels.allocationReceipts.truncate({ cascade: true }) + await queryFeeModels.vouchers.truncate({ cascade: true }) + await queryFeeModels.transferReceipts.truncate({ cascade: true }) + await queryFeeModels.transfers.truncate({ cascade: true }) + await queryFeeModels.allocationSummaries.truncate({ cascade: true }) + await queryFeeModels.scalarTapReceipts.truncate({ cascade: true }) + + // Clear out indexer management models + await managementModels.Action.truncate({ cascade: true }) + await managementModels.CostModel.truncate({ cascade: true }) + await managementModels.IndexingRule.truncate({ cascade: true }) + await managementModels.POIDispute.truncate({ cascade: true }) + + // Clear out indexing agreement model + await managementModels.IndexingAgreement.truncate({ cascade: true }) +} + +const teardownAll = async () => { + await sequelize.drop({}) +} + +describe('DipsManager', () => { + beforeAll(setup) + beforeEach(setupEach) + afterEach(teardownEach) + afterAll(teardownAll) + + // We have been rate-limited on CI as this test uses RPC providers, + // so we set its timeout to a higher value than usual. + jest.setTimeout(30_000) + + describe('initialization', () => { + test('creates DipsManager when dipperEndpoint is configured', () => { + const dipsManager = new DipsManager(logger, managementModels, network, null) + expect(dipsManager).toBeDefined() + }) + + test('throws error when dipperEndpoint is not configured', async () => { + const specWithoutDipper = { + ...testNetworkSpecification, + indexerOptions: { + ...testNetworkSpecification.indexerOptions, + dipperEndpoint: undefined, + }, + } + + metrics.registry.clear() + const networkWithoutDipper = await Network.create( + logger, + specWithoutDipper, + managementModels, + queryFeeModels, + graphNode, + metrics, + ) + expect( + () => new DipsManager(logger, managementModels, networkWithoutDipper, null), + ).toThrow('dipperEndpoint is not set') + }) + }) + + describe('agreement management', () => { + let dipsManager: DipsManager + const testDeploymentId = 'QmTZ8ejXJxRo7vDBS4uwqBeGoxLSWbhaA7oXa1RvxunLy7' + const testAllocationId = 'abcd47df40c29949a75a6693c77834c00b8ad626' + const testAgreementId = '123e4567-e89b-12d3-a456-426614174000' + + beforeEach(async () => { + // Clear mock calls between tests + jest.clearAllMocks() + + const allocationManager = new AllocationManager( + logger, + managementModels, + graphNode, + network, + ) + + dipsManager = new DipsManager(logger, managementModels, network, allocationManager) + + // Create a test agreement + await managementModels.IndexingAgreement.create({ + id: testAgreementId, + subgraph_deployment_id: testDeploymentId, + current_allocation_id: testAllocationId, + last_allocation_id: null, + last_payment_collected_at: null, + cancelled_at: null, + min_epochs_per_collection: BigInt(1), + max_epochs_per_collection: BigInt(5), + payer: '123456df40c29949a75a6693c77834c00b8a5678', + signature: Buffer.from('1234', 'hex'), + signed_payload: Buffer.from('5678', 'hex'), + protocol_network: 'arbitrum-sepolia', + chain_id: 'eip155:1', + base_price_per_epoch: '100', + price_per_entity: '1', + service: 'deadbedf40c29949a75a2293c11834c00b8a1234', + payee: '1212564f40c29949a75a3423c11834c00b8aaaaa', + deadline: new Date(Date.now() + 86400000), // 1 day from now + duration_epochs: BigInt(10), + max_initial_amount: '1000', + max_ongoing_amount_per_epoch: '100', + created_at: new Date(), + updated_at: new Date(), + signed_cancellation_payload: null, + }) + }) + + test('cancels agreement when allocation is closed', async () => { + const client = dipsManager.gatewayDipsServiceClient + + client.CancelAgreement = jest.fn().mockResolvedValue({}) + + await dipsManager.tryCancelAgreement(testAllocationId) + + // Verify the client was called with correct parameters + expect((client.CancelAgreement as jest.Mock).mock.calls.length).toBe(1) + // TODO: Check the signed cancellation payload + expect((client.CancelAgreement as jest.Mock).mock.calls[0][0]).toEqual({ + version: 1, + signedCancellation: expect.any(Uint8Array), + }) + + const agreement = await managementModels.IndexingAgreement.findOne({ + where: { id: testAgreementId }, + }) + expect(agreement?.cancelled_at).toBeDefined() + }) + + test('handles errors when cancelling agreement', async () => { + const client = dipsManager.gatewayDipsServiceClient + client.CancelAgreement = jest + .fn() + .mockRejectedValueOnce(new Error('Failed to cancel')) + + await dipsManager.tryCancelAgreement(testAllocationId) + + const agreement = await managementModels.IndexingAgreement.findOne({ + where: { id: testAgreementId }, + }) + expect(agreement?.cancelled_at).toBeNull() + }) + + test('updates agreement allocation IDs during reallocation', async () => { + const newAllocationId = '5678bedf40c29945678a2293c15678c00b8a5678' + + await dipsManager.tryUpdateAgreementAllocation( + testDeploymentId, + toAddress(testAllocationId), + toAddress(newAllocationId), + ) + + const agreement = await managementModels.IndexingAgreement.findOne({ + where: { id: testAgreementId }, + }) + expect(agreement?.current_allocation_id).toBe(toAddress(newAllocationId)) + expect(agreement?.last_allocation_id).toBe(toAddress(testAllocationId)) + expect(agreement?.last_payment_collected_at).toBeNull() + }) + + test('creates indexing rules for active agreements', async () => { + // Mock fetch the subgraph deployment from the network subgraph + network.networkMonitor.subgraphDeployment = jest + .fn() + .mockResolvedValue(mockSubgraphDeployment(testDeploymentId)) + + await dipsManager.ensureAgreementRules() + + const rules = await managementModels.IndexingRule.findAll({ + where: { + identifier: testDeploymentId, + }, + }) + + expect(rules).toHaveLength(1) + expect(rules[0]).toMatchObject({ + identifier: testDeploymentId, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.DIPS, + allocationAmount: + network.specification.indexerOptions.dipsAllocationAmount.toString(), + autoRenewal: true, + allocationLifetime: 4, // max_epochs_per_collection - dipsEpochsMargin + }) + }) + + test('does not create or modify an indexing rule if it already exists', async () => { + // Create an indexing rule with the same identifier + await managementModels.IndexingRule.create({ + identifier: testDeploymentId, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.ALWAYS, + allocationLifetime: 16, + requireSupported: true, + safety: true, + protocolNetwork: 'eip155:421614', + allocationAmount: '1030', + }) + + // Mock fetch the subgraph deployment from the network subgraph + network.networkMonitor.subgraphDeployment = jest + .fn() + .mockResolvedValue(mockSubgraphDeployment(testDeploymentId)) + + await dipsManager.ensureAgreementRules() + + const rules = await managementModels.IndexingRule.findAll({ + where: { identifier: testDeploymentId }, + }) + expect(rules).toHaveLength(1) + expect(rules[0]).toMatchObject({ + identifier: testDeploymentId, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.ALWAYS, + allocationLifetime: 16, + requireSupported: true, + safety: true, + protocolNetwork: 'eip155:421614', + allocationAmount: '1030', + }) + }) + + test('removes DIPs indexing rule for cancelled agreement', async () => { + await dipsManager.ensureAgreementRules() + const rule = await managementModels.IndexingRule.findOne({ + where: { + identifier: testDeploymentId, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.DIPS, + }, + }) + expect(rule).toBeDefined() + await managementModels.IndexingAgreement.update( + { + cancelled_at: new Date(), + }, + { + where: { id: testAgreementId }, + }, + ) + await dipsManager.ensureAgreementRules() + const ruleAfter = await managementModels.IndexingRule.findOne({ + where: { + identifier: testDeploymentId, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.DIPS, + }, + }) + expect(ruleAfter).toBeNull() + }) + + test('does not remove pre-existing non-DIPS indexing rule', async () => { + // Create an indexing rule with the same identifier + await managementModels.IndexingRule.create({ + identifier: testDeploymentId, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.ALWAYS, + allocationLifetime: 16, + requireSupported: true, + safety: true, + protocolNetwork: 'eip155:421614', + allocationAmount: '1030', + }) + await dipsManager.ensureAgreementRules() + const ruleBefore = await managementModels.IndexingRule.findOne({ + where: { + identifier: testDeploymentId, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.ALWAYS, + }, + }) + expect(ruleBefore).toBeDefined() + await managementModels.IndexingAgreement.update( + { + cancelled_at: new Date(), + }, + { + where: { id: testAgreementId }, + }, + ) + await dipsManager.ensureAgreementRules() + const ruleAfter = await managementModels.IndexingRule.findOne({ + where: { + identifier: testDeploymentId, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.ALWAYS, + }, + }) + expect(ruleAfter).toBeDefined() + }) + + test('returns active DIPs deployments', async () => { + const deployments = await dipsManager.getActiveDipsDeployments() + + expect(deployments).toHaveLength(1) + expect(deployments[0].ipfsHash).toBe(testDeploymentId) + }) + }) +}) + +describe('DipsCollector', () => { + beforeAll(setup) + beforeEach(setupEach) + afterEach(teardownEach) + afterAll(teardownAll) + + describe('initialization', () => { + test('creates DipsCollector when dipperEndpoint is configured', () => { + const dipsCollector = new DipsCollector( + logger, + managementModels, + queryFeeModels, + networkSpecWithDips, + network.tapCollector!, + network.wallet, + graphNode, + jest.fn(), + ) + expect(dipsCollector).toBeDefined() + }) + test('starts payment collection loop', () => { + const dipsCollector = new DipsCollector( + logger, + managementModels, + queryFeeModels, + networkSpecWithDips, + network.tapCollector!, + network.wallet, + graphNode, + jest.fn(), + ) + expect(dipsCollector).toBeDefined() + expect(startCollectionLoop).toHaveBeenCalled() + }) + test('throws error when dipperEndpoint is not configured', () => { + const specWithoutDipper = { + ...testNetworkSpecification, + indexerOptions: { + ...testNetworkSpecification.indexerOptions, + dipperEndpoint: undefined, + }, + } + expect( + () => + new DipsCollector( + logger, + managementModels, + queryFeeModels, + specWithoutDipper, + network.tapCollector!, + network.wallet, + graphNode, + jest.fn(), + ), + ).toThrow('dipperEndpoint is not set') + }) + }) + + describe('payment collection', () => { + const testDeploymentId = 'QmTZ8ejXJxRo7vDBS4uwqBeGoxLSWbhaA7oXa1RvxunLy7' + const testAllocationId = 'abcd47df40c29949a75a6693c77834c00b8ad626' + const testAgreementId = '123e4567-e89b-12d3-a456-426614174000' + + beforeEach(async () => { + // Clear mock calls between tests + jest.clearAllMocks() + + // Create a test agreement + // Note last_allocation_id is set to the testAllocationId + // current_allocation_id is set to null so that we can collect payment + // (also last_payment_collected_at is set to null) + await managementModels.IndexingAgreement.create({ + id: testAgreementId, + subgraph_deployment_id: testDeploymentId, + current_allocation_id: null, + last_allocation_id: testAllocationId, + last_payment_collected_at: null, + cancelled_at: null, + min_epochs_per_collection: BigInt(1), + max_epochs_per_collection: BigInt(5), + payer: '123456df40c29949a75a6693c77834c00b8a5678', + signature: Buffer.from('1234', 'hex'), + signed_payload: Buffer.from('5678', 'hex'), + protocol_network: 'arbitrum-sepolia', + chain_id: 'eip155:1', + base_price_per_epoch: '100', + price_per_entity: '1', + service: 'deadbedf40c29949a75a2293c11834c00b8a1234', + payee: '1212564f40c29949a75a3423c11834c00b8aaaaa', + deadline: new Date(Date.now() + 86400000), // 1 day from now + duration_epochs: BigInt(10), + max_initial_amount: '1000', + max_ongoing_amount_per_epoch: '100', + created_at: new Date(), + updated_at: new Date(), + signed_cancellation_payload: null, + }) + graphNode.entityCount = jest.fn().mockResolvedValue([250000]) + }) + test('collects payment for a specific agreement', async () => { + const agreement = await managementModels.IndexingAgreement.findOne({ + where: { id: testAgreementId }, + }) + if (!agreement) { + throw new Error('Agreement not found') + } + + const client = dipsCollector.gatewayDipsServiceClient + + client.CollectPayment = jest.fn().mockResolvedValue({ + version: 1, + status: CollectPaymentStatus.ACCEPT, + tapReceipt: Buffer.from('1234', 'hex'), + }) + dipsCollector.gatewayDipsServiceMessagesCodec.decodeTapReceipt = jest + .fn() + .mockImplementation(() => { + logger.info('MOCK Decoding TAP receipt') + return { + allocation_id: toAddress(testAllocationId), + signer_address: toAddress('0xabcd56df41234949a75a6693c77834c00b8abbbb'), + signature: Buffer.from('1234', 'hex'), + timestamp_ns: 1234567890, + nonce: 1, + value: '1000', + } + }) + dipsCollector.escrowSenderGetter = jest.fn().mockImplementation(() => { + logger.info('MOCK Getting escrow sender for signer') + return toAddress('0x123456df40c29949a75a6693c77834c00b8a5678') + }) + + await dipsCollector.tryCollectPayment(agreement) + + expect(client.CollectPayment).toHaveBeenCalledWith({ + version: 1, + signedCollection: expect.any(Uint8Array), + }) + expect(agreement.last_payment_collected_at).not.toBeNull() + + const receipt = await queryFeeModels.scalarTapReceipts.findOne({ + where: { + allocation_id: testAllocationId, + }, + }) + expect(receipt).not.toBeNull() + expect(receipt?.signer_address).toBe( + toAddress('0xabcd56df41234949a75a6693c77834c00b8abbbb'), + ) + expect(receipt?.value).toBe('1000') + }) + }) +}) diff --git a/packages/indexer-common/src/indexing-fees/dips.ts b/packages/indexer-common/src/indexing-fees/dips.ts new file mode 100644 index 000000000..dc939ba64 --- /dev/null +++ b/packages/indexer-common/src/indexing-fees/dips.ts @@ -0,0 +1,502 @@ +import { + Address, + formatGRT, + Logger, + SubgraphDeploymentID, + toAddress, +} from '@graphprotocol/common-ts' +import { + ActionStatus, + Allocation, + AllocationManager, + getEscrowSenderForSigner, + GraphNode, + IndexerManagementModels, + IndexingDecisionBasis, + IndexingRuleAttributes, + Network, + QueryFeeModels, + sequentialTimerMap, + SubgraphClient, + SubgraphIdentifierType, + TapCollector, + upsertIndexingRule, +} from '@graphprotocol/indexer-common' +import { Op } from 'sequelize' + +import { + createGatewayDipsServiceClient, + GatewayDipsServiceMessagesCodec, +} from './gateway-dips-service-client' +import { + CollectPaymentStatus, + GatewayDipsServiceClientImpl, +} from '@graphprotocol/dips-proto/generated/gateway' +import { IndexingAgreement } from '../indexer-management/models/indexing-agreement' +import { NetworkSpecification } from '../network-specification' +import { Wallet } from 'ethers' + +const DIPS_COLLECTION_INTERVAL = 60_000 + +const uuidToHex = (uuid: string) => { + return `0x${uuid.replace(/-/g, '')}` +} + +const normalizeAddressForDB = (address: string) => { + return toAddress(address).toLowerCase().replace('0x', '') +} + +type GetEscrowSenderForSigner = ( + tapSubgraph: SubgraphClient, + signer: Address, +) => Promise
+export class DipsManager { + declare gatewayDipsServiceClient: GatewayDipsServiceClientImpl + declare gatewayDipsServiceMessagesCodec: GatewayDipsServiceMessagesCodec + constructor( + private logger: Logger, + private models: IndexerManagementModels, + private network: Network, + private parent: AllocationManager | null, + ) { + if (!this.network.specification.indexerOptions.dipperEndpoint) { + throw new Error('dipperEndpoint is not set') + } + this.gatewayDipsServiceClient = createGatewayDipsServiceClient( + this.network.specification.indexerOptions.dipperEndpoint, + ) + this.gatewayDipsServiceMessagesCodec = new GatewayDipsServiceMessagesCodec() + } + // Cancel an agreement associated to an allocation if it exists + async tryCancelAgreement(allocationId: string) { + const normalizedAllocationId = normalizeAddressForDB(allocationId) + const agreement = await this.models.IndexingAgreement.findOne({ + where: { + current_allocation_id: normalizedAllocationId, + cancelled_at: null, + }, + }) + if (agreement) { + try { + await this._tryCancelAgreement(agreement) + } catch (error) { + this.logger.error(`Error cancelling agreement ${agreement.id}`, { error }) + } + } + } + async _tryCancelAgreement(agreement: IndexingAgreement) { + try { + const cancellation = + await this.gatewayDipsServiceMessagesCodec.createSignedCancellationRequest( + uuidToHex(agreement.id), + this.network.wallet, + ) + await this.gatewayDipsServiceClient.CancelAgreement({ + version: 1, + signedCancellation: cancellation, + }) + agreement.cancelled_at = new Date() + agreement.updated_at = new Date() + await agreement.save() + } catch (error) { + this.logger.error(`Error cancelling agreement ${agreement.id}`, { error }) + } + } + // Update the current and last allocation ids for an agreement if it exists + async tryUpdateAgreementAllocation( + deploymentId: string, + oldAllocationId: Address | null, + newAllocationId: Address | null, + ) { + const agreement = await this.models.IndexingAgreement.findOne({ + where: { + subgraph_deployment_id: deploymentId, + }, + }) + if (agreement) { + agreement.current_allocation_id = newAllocationId + agreement.last_allocation_id = oldAllocationId + agreement.last_payment_collected_at = null + agreement.updated_at = new Date() + await agreement.save() + } + } + async ensureAgreementRules() { + if (!this.parent) { + this.logger.error( + 'DipsManager has no parent AllocationManager, cannot ensure agreement rules', + ) + return + } + // Get all the indexing agreements that are not cancelled + const indexingAgreements = await this.models.IndexingAgreement.findAll({ + where: { + cancelled_at: null, + }, + }) + this.logger.debug( + `Ensuring indexing rules for ${indexingAgreements.length} active agreement${ + indexingAgreements.length === 1 ? '' : 's' + }`, + ) + // For each agreement, check that there is an indexing rule to always + // allocate to the agreement's subgraphDeploymentId, and if not, create one + for (const agreement of indexingAgreements) { + const subgraphDeploymentID = new SubgraphDeploymentID( + agreement.subgraph_deployment_id, + ) + this.logger.info( + `Checking if indexing rule exists for agreement ${ + agreement.id + }, deployment ${subgraphDeploymentID.toString()}`, + ) + // If there is not yet an indexingRule that deems this deployment worth allocating to, make one + const ruleExists = await this.parent.matchingRuleExists( + this.logger, + subgraphDeploymentID, + ) + // Check if there is an indexing rule saying we should NEVER allocate to this one, consider it blocklisted + const allDeploymentRules = await this.models.IndexingRule.findAll({ + where: { + identifierType: SubgraphIdentifierType.DEPLOYMENT, + }, + }) + const blocklistedRule = allDeploymentRules.find( + (rule) => + new SubgraphDeploymentID(rule.identifier).bytes32 === + subgraphDeploymentID.bytes32 && + rule.decisionBasis === IndexingDecisionBasis.NEVER, + ) + if (blocklistedRule) { + this.logger.info( + `Blocklisted deployment ${subgraphDeploymentID.toString()}, skipping indexing rule creation`, + ) + await this._tryCancelAgreement(agreement) + } else if (!ruleExists) { + this.logger.info( + `Creating indexing rule for agreement ${agreement.id}, deployment ${agreement.subgraph_deployment_id}`, + ) + const indexingRule = { + identifier: agreement.subgraph_deployment_id, + allocationAmount: formatGRT( + this.network.specification.indexerOptions.dipsAllocationAmount, + ), + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.DIPS, + protocolNetwork: this.network.specification.networkIdentifier, + autoRenewal: true, + allocationLifetime: Math.max( + Number(agreement.min_epochs_per_collection), + Number(agreement.max_epochs_per_collection) - + this.network.specification.indexerOptions.dipsEpochsMargin, + ), + requireSupported: false, + } as Partial + + await upsertIndexingRule(this.logger, this.models, indexingRule) + } + } + + const cancelledAgreements = await this.models.IndexingAgreement.findAll({ + where: { + cancelled_at: { + [Op.ne]: null, + }, + }, + }) + this.logger.debug( + `Ensuring no DIPs indexing rules for ${ + cancelledAgreements.length + } cancelled agreement${cancelledAgreements.length === 1 ? '' : 's'}`, + ) + for (const agreement of cancelledAgreements) { + this.logger.info( + `Checking if indexing rule exists for cancelled agreement ${agreement.id}, deployment ${agreement.subgraph_deployment_id}`, + ) + // First check if there is another agreement that is not cancelled that has the same deployment id + const otherAgreement = indexingAgreements.find( + (a) => + a.subgraph_deployment_id === agreement.subgraph_deployment_id && + a.id !== agreement.id, + ) + if (otherAgreement) { + this.logger.info( + `Another agreement ${otherAgreement.id} exists for deployment ${agreement.subgraph_deployment_id}, skipping removal of DIPs indexing rule`, + ) + continue + } + const rule = await this.models.IndexingRule.findOne({ + where: { + identifier: agreement.subgraph_deployment_id, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.DIPS, + }, + }) + if (rule) { + this.logger.info( + `Removing DIPs indexing rule for cancelled agreement ${agreement.id}, deployment ${agreement.subgraph_deployment_id}`, + ) + await this.models.IndexingRule.destroy({ + where: { id: rule.id }, + }) + } + } + } + async getActiveDipsDeployments(): Promise { + // Get all the indexing agreements that are not cancelled + const indexingAgreements = await this.models.IndexingAgreement.findAll({ + where: { + cancelled_at: null, + }, + }) + return indexingAgreements.map( + (agreement) => new SubgraphDeploymentID(agreement.subgraph_deployment_id), + ) + } + async matchAgreementAllocations(allocations: Allocation[]) { + const indexingAgreements = await this.models.IndexingAgreement.findAll({ + where: { + cancelled_at: null, + }, + }) + for (const agreement of indexingAgreements) { + this.logger.trace(`Matching active agreement ${agreement.id}`) + const allocation = allocations.find( + (allocation) => + allocation.subgraphDeployment.id.bytes32 === + new SubgraphDeploymentID(agreement.subgraph_deployment_id).bytes32, + ) + const actions = await this.models.Action.findAll({ + where: { + deploymentID: agreement.subgraph_deployment_id, + status: { + [Op.or]: [ + ActionStatus.PENDING, + ActionStatus.QUEUED, + ActionStatus.APPROVED, + ActionStatus.DEPLOYING, + ], + }, + }, + }) + this.logger.trace(`Found ${actions.length} actions for agreement ${agreement.id}`) + if (allocation && actions.length === 0) { + const currentAllocationId = + agreement.current_allocation_id != null + ? toAddress(agreement.current_allocation_id) + : null + this.logger.trace( + `Current allocation id for agreement ${agreement.id} is ${currentAllocationId}`, + { + currentAllocationId, + allocation, + }, + ) + if (currentAllocationId !== allocation.id) { + this.logger.warn( + `Found mismatched allocation for agreement ${agreement.id}, updating from ${currentAllocationId} to ${allocation.id}`, + ) + await this.tryUpdateAgreementAllocation( + agreement.subgraph_deployment_id, + currentAllocationId, + allocation.id, + ) + } + } + } + // Now we find the cancelled agreements and check if their allocation is still active + const cancelledAgreements = await this.models.IndexingAgreement.findAll({ + where: { + cancelled_at: { + [Op.ne]: null, + }, + current_allocation_id: { + [Op.ne]: null, + }, + }, + }) + for (const agreement of cancelledAgreements) { + this.logger.trace(`Matching cancelled agreement ${agreement.id}`) + const allocation = allocations.find( + (allocation) => + allocation.subgraphDeployment.id.bytes32 === + new SubgraphDeploymentID(agreement.subgraph_deployment_id).bytes32, + ) + if (allocation == null && agreement.current_allocation_id != null) { + const actions = await this.models.Action.findAll({ + where: { + deploymentID: agreement.subgraph_deployment_id, + status: { + [Op.or]: [ + ActionStatus.PENDING, + ActionStatus.QUEUED, + ActionStatus.APPROVED, + ActionStatus.DEPLOYING, + ], + }, + }, + }) + if (actions.length > 0) { + this.logger.warn( + `Found active actions for cancelled agreement ${agreement.id}, deployment ${agreement.subgraph_deployment_id}, skipping matching allocation`, + ) + continue + } + this.logger.info( + `Updating last allocation id for cancelled agreement ${agreement.id}, deployment ${agreement.subgraph_deployment_id}`, + ) + await this.tryUpdateAgreementAllocation( + agreement.subgraph_deployment_id, + toAddress(agreement.current_allocation_id), + null, + ) + } + } + } +} + +export class DipsCollector { + declare gatewayDipsServiceClient: GatewayDipsServiceClientImpl + declare gatewayDipsServiceMessagesCodec: GatewayDipsServiceMessagesCodec + constructor( + private logger: Logger, + private managementModels: IndexerManagementModels, + private queryFeeModels: QueryFeeModels, + private specification: NetworkSpecification, + private tapCollector: TapCollector, + private wallet: Wallet, + private graphNode: GraphNode, + public escrowSenderGetter: GetEscrowSenderForSigner, + ) { + if (!this.specification.indexerOptions.dipperEndpoint) { + throw new Error('dipperEndpoint is not set') + } + this.gatewayDipsServiceClient = createGatewayDipsServiceClient( + this.specification.indexerOptions.dipperEndpoint, + ) + this.gatewayDipsServiceMessagesCodec = new GatewayDipsServiceMessagesCodec() + } + + static create( + logger: Logger, + managementModels: IndexerManagementModels, + queryFeeModels: QueryFeeModels, + specification: NetworkSpecification, + tapCollector: TapCollector, + wallet: Wallet, + graphNode: GraphNode, + escrowSenderGetter?: GetEscrowSenderForSigner, + ) { + const collector = new DipsCollector( + logger, + managementModels, + queryFeeModels, + specification, + tapCollector, + wallet, + graphNode, + escrowSenderGetter ?? getEscrowSenderForSigner, + ) + collector.startCollectionLoop() + return collector + } + + startCollectionLoop() { + sequentialTimerMap( + { + logger: this.logger, + milliseconds: DIPS_COLLECTION_INTERVAL, + }, + async () => { + this.logger.debug('Running DIPs payment collection loop') + await this.collectAllPayments() + }, + { + onError: (err) => { + this.logger.error('Failed to collect DIPs payments', { err }) + }, + }, + ) + } + + // Collect payments for all outstanding agreements + async collectAllPayments() { + const outstandingAgreements = await this.managementModels.IndexingAgreement.findAll({ + where: { + last_payment_collected_at: null, + last_allocation_id: { + [Op.ne]: null, + }, + }, + }) + for (const agreement of outstandingAgreements) { + await this.tryCollectPayment(agreement) + } + } + async tryCollectPayment(agreement: IndexingAgreement) { + if (!agreement.last_allocation_id) { + this.logger.error(`Agreement ${agreement.id} has no last allocation id`) + return + } + const entityCounts = await this.graphNode.entityCount([ + new SubgraphDeploymentID(agreement.subgraph_deployment_id), + ]) + if (entityCounts.length === 0) { + this.logger.error(`Agreement ${agreement.id} has no entity count`) + return + } + const entityCount = entityCounts[0] + const collection = + await this.gatewayDipsServiceMessagesCodec.createSignedCollectionRequest( + uuidToHex(agreement.id), + agreement.last_allocation_id, + entityCount, + this.wallet, + ) + try { + this.logger.info(`Collecting payment for agreement ${agreement.id}`) + const response = await this.gatewayDipsServiceClient.CollectPayment({ + version: 1, + signedCollection: collection, + }) + if (response.status === CollectPaymentStatus.ACCEPT) { + if (!this.tapCollector) { + throw new Error('TapCollector not initialized') + } + // Store the tap receipt in the database + this.logger.info('Decoding TAP receipt for agreement') + const tapReceipt = this.gatewayDipsServiceMessagesCodec.decodeTapReceipt( + response.tapReceipt, + this.tapCollector?.tapContracts.tapVerifier.address, + ) + // Check that the signer of the TAP receipt is a signer + // on the corresponding escrow account for the payer (sender) of the + // indexing agreement + const escrowSender = await this.escrowSenderGetter( + this.tapCollector?.tapSubgraph, + tapReceipt.signer_address, + ) + if (escrowSender !== toAddress(agreement.payer)) { + // TODO: should we cancel the agreement here? + throw new Error( + 'Signer of TAP receipt is not a signer on the indexing agreement', + ) + } + if (tapReceipt.allocation_id !== toAddress(agreement.last_allocation_id)) { + throw new Error('Allocation ID mismatch') + } + await this.queryFeeModels.scalarTapReceipts.create(tapReceipt) + // Mark the agreement as having had a payment collected + agreement.last_payment_collected_at = new Date() + agreement.updated_at = new Date() + await agreement.save() + } else { + throw new Error(`Payment request not accepted: ${response.status}`) + } + } catch (error) { + this.logger.error(`Error collecting payment for agreement ${agreement.id}`, { + error, + }) + } + } +} diff --git a/packages/indexer-common/src/indexing-fees/gateway-dips-service-client.ts b/packages/indexer-common/src/indexing-fees/gateway-dips-service-client.ts new file mode 100644 index 000000000..1bfb832a5 --- /dev/null +++ b/packages/indexer-common/src/indexing-fees/gateway-dips-service-client.ts @@ -0,0 +1,168 @@ +import { Client, credentials } from '@grpc/grpc-js' +import { UnaryCallback } from '@grpc/grpc-js/build/src/client' +import { GatewayDipsServiceClientImpl } from '@graphprotocol/dips-proto/generated/gateway' +import { Wallet } from 'ethers' +import { + _TypedDataEncoder, + arrayify, + defaultAbiCoder, + recoverAddress, +} from 'ethers/lib/utils' +import { toAddress } from '@graphprotocol/common-ts' + +type RpcImpl = (service: string, method: string, data: Uint8Array) => Promise + +interface Rpc { + request: RpcImpl +} + +export const domainSalt = + '0xb4632c657c26dce5d4d7da1d65bda185b14ff8f905ddbb03ea0382ed06c5ef28' +export const chainId = 0xa4b1 // 42161 +export const cancelAgreementDomain = { + name: 'Graph Protocol Indexing Agreement Cancellation', + version: '0', + chainId: chainId, + salt: domainSalt, +} +export const cancelAgreementTypes = { + CancellationRequest: [{ name: 'agreement_id', type: 'bytes16' }], +} + +export const collectPaymentsDomain = { + name: 'Graph Protocol Indexing Agreement Collection', + version: '0', + chainId: chainId, + salt: domainSalt, +} +export const collectPaymentsTypes = { + CollectionRequest: [ + { name: 'agreement_id', type: 'bytes16' }, + { name: 'allocation_id', type: 'address' }, + { name: 'entity_count', type: 'uint64' }, + ], +} + +export class GatewayDipsServiceMessagesCodec { + async createSignedCancellationRequest( + agreementId: string, + wallet: Wallet, + ): Promise { + const signature = await wallet._signTypedData( + cancelAgreementDomain, + cancelAgreementTypes, + { agreement_id: agreementId }, + ) + return arrayify( + defaultAbiCoder.encode(['tuple(bytes16)', 'bytes'], [[agreementId], signature]), + ) + } + + async createSignedCollectionRequest( + agreementId: string, + allocationId: string, + entityCount: number, + wallet: Wallet, + ): Promise { + const signature = await wallet._signTypedData( + collectPaymentsDomain, + collectPaymentsTypes, + { + agreement_id: agreementId, + allocation_id: toAddress(allocationId), + entity_count: entityCount, + }, + ) + return arrayify( + defaultAbiCoder.encode( + ['tuple(bytes16, address, uint64)', 'bytes'], + [[agreementId, toAddress(allocationId), entityCount], signature], + ), + ) + } + + decodeTapReceipt(receipt: Uint8Array, verifyingContract: string) { + const [message, signature] = defaultAbiCoder.decode( + ['tuple(address,uint64,uint64,uint128)', 'bytes'], + receipt, + ) + + const [allocationId, timestampNs, nonce, value] = message + + // Recover the signer address from the signature + // compute the EIP-712 digest of the message + const domain = { + name: 'TAP', + version: '1', + chainId: chainId, + verifyingContract, + } + + const types = { + Receipt: [ + { name: 'allocation_id', type: 'address' }, + { name: 'timestamp_ns', type: 'uint64' }, + { name: 'nonce', type: 'uint64' }, + { name: 'value', type: 'uint128' }, + ], + } + + const digest = _TypedDataEncoder.hash(domain, types, { + allocation_id: allocationId, + timestamp_ns: timestampNs, + nonce: nonce, + value: value, + }) + const signerAddress = recoverAddress(digest, signature) + return { + allocation_id: toAddress(allocationId), + signer_address: toAddress(signerAddress), + signature: signature, + timestamp_ns: timestampNs, + nonce: nonce, + value: value, + } + } +} + +export const createRpc = (url: string): Rpc => { + const client = new Client(url, credentials.createInsecure()) + const request: RpcImpl = (service, method, data) => { + // Conventionally in gRPC, the request path looks like + // "package.names.ServiceName/MethodName", + // we therefore construct such a string + const path = `/${service}/${method}` + + return new Promise((resolve, reject) => { + // makeUnaryRequest transmits the result (and error) with a callback + // transform this into a promise! + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const resultCallback: UnaryCallback = (err, res) => { + if (err) { + return reject(err) + } + resolve(res) + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function passThrough(argument: any) { + return argument + } + + // Using passThrough as the deserialize functions + client.makeUnaryRequest( + path, + (d) => Buffer.from(d), + passThrough, + data, + resultCallback, + ) + }) + } + + return { request } +} + +export const createGatewayDipsServiceClient = (url: string) => { + const rpc = createRpc(url) + return new GatewayDipsServiceClientImpl(rpc) +} diff --git a/packages/indexer-common/src/indexing-fees/index.ts b/packages/indexer-common/src/indexing-fees/index.ts new file mode 100644 index 000000000..0b71f1b8e --- /dev/null +++ b/packages/indexer-common/src/indexing-fees/index.ts @@ -0,0 +1 @@ +export * from './dips' diff --git a/packages/indexer-common/src/network-specification.ts b/packages/indexer-common/src/network-specification.ts index 7bb86d6e2..d1008b5e6 100644 --- a/packages/indexer-common/src/network-specification.ts +++ b/packages/indexer-common/src/network-specification.ts @@ -72,6 +72,10 @@ export const IndexerOptions = z .default(0), finalityTime: positiveNumber().default(3600), legacyMnemonics: z.array(z.string()).default([]), + enableDips: z.boolean().default(false), + dipperEndpoint: z.string().url().optional(), + dipsAllocationAmount: GRT().default(1), + dipsEpochsMargin: positiveNumber().default(1), }) .strict() export type IndexerOptions = z.infer diff --git a/packages/indexer-common/src/network.ts b/packages/indexer-common/src/network.ts index ee8db5efb..50788e692 100644 --- a/packages/indexer-common/src/network.ts +++ b/packages/indexer-common/src/network.ts @@ -39,6 +39,7 @@ import { NetworkMonitor, SubgraphFreshnessChecker, monitorEligibleAllocations, + IndexerManagementModels, } from '.' import { resolveChainId } from './indexer-management' import { monitorEthBalance } from './utils' @@ -46,6 +47,7 @@ import { QueryFeeModels } from './query-fees' import { TapCollector } from './allocations/tap-collector' import { GraphTallyCollector } from './allocations/graph-tally-collector' import { encodeRegistrationData } from '@graphprotocol/toolshed' +import { DipsCollector } from './indexing-fees/dips' export class Network { logger: Logger @@ -58,11 +60,13 @@ export class Network { tapCollector: TapCollector | undefined graphTallyCollector: GraphTallyCollector | undefined + dipsCollector: DipsCollector | undefined specification: spec.NetworkSpecification paused: Eventual isOperator: Eventual isHorizon: Eventual - + queryFeeModels: QueryFeeModels + managementModels: IndexerManagementModels private constructor( logger: Logger, contracts: GraphHorizonContracts & SubgraphServiceContracts, @@ -77,6 +81,9 @@ export class Network { paused: Eventual, isOperator: Eventual, isHorizon: Eventual, + queryFeeModels: QueryFeeModels, + managementModels: IndexerManagementModels, + dipsCollector: DipsCollector | undefined, ) { this.logger = logger this.contracts = contracts @@ -91,11 +98,15 @@ export class Network { this.paused = paused this.isOperator = isOperator this.isHorizon = isHorizon + this.queryFeeModels = queryFeeModels + this.managementModels = managementModels + this.dipsCollector = dipsCollector } static async create( parentLogger: Logger, specification: spec.NetworkSpecification, + managementModels: IndexerManagementModels, queryFeeModels: QueryFeeModels, graphNode: GraphNode, metrics: Metrics, @@ -305,6 +316,7 @@ export class Network { // * TAP Collector // -------------------------------------------------------------------------------- let tapCollector: TapCollector | undefined = undefined + let dipsCollector: DipsCollector | undefined = undefined if (tapContracts && tapSubgraph) { tapCollector = TapCollector.create({ logger, @@ -318,8 +330,19 @@ export class Network { networkSubgraph, legacyMnemonics: specification.indexerOptions.legacyMnemonics, }) + if (specification.indexerOptions.enableDips) { + dipsCollector = DipsCollector.create( + logger, + managementModels, + queryFeeModels, + specification, + tapCollector, + wallet, + graphNode, + ) + } } else { - logger.info(`RAV process not initiated. + logger.info(`RAV (and DIPs) process not initiated. Tap Contracts: ${!!tapContracts}. Tap Subgraph: ${!!tapSubgraph}.`) } @@ -365,6 +388,9 @@ export class Network { paused, isOperator, isHorizon, + queryFeeModels, + managementModels, + dipsCollector, ) } diff --git a/packages/indexer-common/src/operator.ts b/packages/indexer-common/src/operator.ts index b83c2ba1e..f85905aaf 100644 --- a/packages/indexer-common/src/operator.ts +++ b/packages/indexer-common/src/operator.ts @@ -16,6 +16,7 @@ import { specification as spec, Action, POIDisputeAttributes, + DipsManager, } from '@graphprotocol/indexer-common' import { Logger, formatGRT } from '@graphprotocol/common-ts' import { hexlify } from 'ethers' @@ -82,6 +83,13 @@ export class Operator { this.specification = specification } + get dipsManager(): DipsManager | null { + const network = this.specification.networkIdentifier + const allocationManager = + this.indexerManagement.actionManager?.allocationManagers[network] + return allocationManager?.dipsManager ?? null + } + // -------------------------------------------------------------------------------- // * Indexing Rules // -------------------------------------------------------------------------------- @@ -260,16 +268,26 @@ export class Operator { return result.data.actions } - async queueAction(action: ActionItem): Promise { + async queueAction(action: ActionItem, forceAction: boolean = false): Promise { let status = ActionStatus.QUEUED switch (this.specification.indexerOptions.allocationManagementMode) { case AllocationManagementMode.MANUAL: - throw Error(`Cannot queue actions when AllocationManagementMode = 'MANUAL'`) + if (forceAction) { + status = ActionStatus.APPROVED + } else { + throw Error(`Cannot queue actions when AllocationManagementMode = 'MANUAL'`) + } + break case AllocationManagementMode.AUTO: status = ActionStatus.APPROVED break case AllocationManagementMode.OVERSIGHT: - status = ActionStatus.QUEUED + if (forceAction) { + status = ActionStatus.APPROVED + } else { + status = ActionStatus.QUEUED + } + break } const actionInput = { @@ -346,6 +364,7 @@ export class Operator { deploymentAllocationDecision: AllocationDecision, mostRecentlyClosedAllocation: Allocation | undefined, isHorizon: boolean, + forceAction: boolean = false, ): Promise { const desiredAllocationAmount = deploymentAllocationDecision.ruleMatch.rule ?.allocationAmount @@ -375,16 +394,19 @@ export class Operator { } // Send AllocateAction to the queue - isLegacy value depends on the horizon upgrade - await this.queueAction({ - params: { - deploymentID: deploymentAllocationDecision.deployment.ipfsHash, - amount: formatGRT(desiredAllocationAmount), + await this.queueAction( + { + params: { + deploymentID: deploymentAllocationDecision.deployment.ipfsHash, + amount: formatGRT(desiredAllocationAmount), + }, + type: ActionType.ALLOCATE, + reason: deploymentAllocationDecision.reasonString(), + protocolNetwork: deploymentAllocationDecision.protocolNetwork, + isLegacy: !isHorizon, }, - type: ActionType.ALLOCATE, - reason: deploymentAllocationDecision.reasonString(), - protocolNetwork: deploymentAllocationDecision.protocolNetwork, - isLegacy: !isHorizon, - }) + forceAction, + ) return } @@ -393,6 +415,7 @@ export class Operator { logger: Logger, deploymentAllocationDecision: AllocationDecision, activeDeploymentAllocations: Allocation[], + forceAction: boolean = false, ): Promise { // Make sure to close all active allocations on the way out if (activeDeploymentAllocations.length > 0) { @@ -409,18 +432,21 @@ export class Operator { activeDeploymentAllocations, async (allocation) => { // Send unallocate action to the queue - isLegacy value depends on the allocation being closed - await this.queueAction({ - params: { - allocationID: allocation.id, - deploymentID: deploymentAllocationDecision.deployment.ipfsHash, - poi: undefined, - force: false, - }, - type: ActionType.UNALLOCATE, - reason: deploymentAllocationDecision.reasonString(), - protocolNetwork: deploymentAllocationDecision.protocolNetwork, - isLegacy: allocation.isLegacy, - } as ActionItem) + await this.queueAction( + { + params: { + allocationID: allocation.id, + deploymentID: deploymentAllocationDecision.deployment.ipfsHash, + poi: undefined, + force: false, + }, + type: ActionType.UNALLOCATE, + reason: deploymentAllocationDecision.reasonString(), + protocolNetwork: deploymentAllocationDecision.protocolNetwork, + isLegacy: allocation.isLegacy, + } as ActionItem, + forceAction, + ) }, { concurrency: 1 }, ) @@ -431,6 +457,7 @@ export class Operator { logger: Logger, deploymentAllocationDecision: AllocationDecision, expiredAllocations: Allocation[], + forceAction: boolean = false, ): Promise { if (deploymentAllocationDecision.ruleMatch.rule?.autoRenewal) { logger.info(`Reallocating expired allocations`, { @@ -448,17 +475,20 @@ export class Operator { await pMap( expiredAllocations, async (allocation) => { - await this.queueAction({ - params: { - allocationID: allocation.id, - deploymentID: deploymentAllocationDecision.deployment.ipfsHash, - amount: formatGRT(desiredAllocationAmount), + await this.queueAction( + { + params: { + allocationID: allocation.id, + deploymentID: deploymentAllocationDecision.deployment.ipfsHash, + amount: formatGRT(desiredAllocationAmount), + }, + type: ActionType.REALLOCATE, + reason: `${deploymentAllocationDecision.reasonString()}:allocationExpiring`, // Need to update to include 'ExpiringSoon' + protocolNetwork: deploymentAllocationDecision.protocolNetwork, + isLegacy: allocation.isLegacy, }, - type: ActionType.REALLOCATE, - reason: `${deploymentAllocationDecision.reasonString()}:allocationExpiring`, // Need to update to include 'ExpiringSoon' - protocolNetwork: deploymentAllocationDecision.protocolNetwork, - isLegacy: allocation.isLegacy, - }) + forceAction, + ) }, { stopOnError: false, diff --git a/packages/indexer-common/src/query-fees/models.ts b/packages/indexer-common/src/query-fees/models.ts index 132e9240e..6419416d0 100644 --- a/packages/indexer-common/src/query-fees/models.ts +++ b/packages/indexer-common/src/query-fees/models.ts @@ -7,19 +7,28 @@ import { BytesLike } from 'ethers' export interface ScalarTapReceiptsAttributes { id: number - allocation_id: Address - signer_address: Address + allocation_id: string + signer_address: string signature: Uint8Array timestamp_ns: bigint nonce: bigint value: bigint error_log?: string } +export interface ScalarTapReceiptsCreationAttributes { + allocation_id: string + signer_address: string + signature: Uint8Array + timestamp_ns: bigint + nonce: bigint + value: bigint +} + export class ScalarTapReceipts - extends Model + extends Model implements ScalarTapReceiptsAttributes { - public id!: number + public id!: CreationOptional public allocation_id!: Address public signer_address!: Address public signature!: Uint8Array @@ -926,10 +935,26 @@ export function defineQueryFeeModels(sequelize: Sequelize): QueryFeeModels { allocation_id: { type: DataTypes.CHAR(40), allowNull: false, + get() { + const rawValue = this.getDataValue('allocation_id') + return toAddress(rawValue) + }, + set(value: Address) { + const addressWithoutPrefix = value.toLowerCase().replace('0x', '') + this.setDataValue('allocation_id', addressWithoutPrefix) + }, }, signer_address: { type: DataTypes.CHAR(40), allowNull: false, + get() { + const rawValue = this.getDataValue('signer_address') + return toAddress(rawValue) + }, + set(value: Address) { + const addressWithoutPrefix = value.toLowerCase().replace('0x', '') + this.setDataValue('signer_address', addressWithoutPrefix) + }, }, signature: { type: DataTypes.BLOB, diff --git a/packages/indexer-common/src/rules.ts b/packages/indexer-common/src/rules.ts index e03bfd40e..0d91f8f24 100644 --- a/packages/indexer-common/src/rules.ts +++ b/packages/indexer-common/src/rules.ts @@ -4,9 +4,9 @@ import { parseGRT } from '@graphprotocol/common-ts' import { validateNetworkIdentifier } from './parsers' export const parseDecisionBasis = (s: string): IndexingDecisionBasis => { - if (!['always', 'never', 'rules', 'offchain'].includes(s)) { + if (!['always', 'never', 'rules', 'offchain', 'dips'].includes(s)) { throw new Error( - `Unknown decision basis "${s}". Supported: always, never, rules, offchain`, + `Unknown decision basis "${s}". Supported: always, never, rules, offchain, dips`, ) } else { return s as IndexingDecisionBasis diff --git a/packages/indexer-common/src/subgraphs.ts b/packages/indexer-common/src/subgraphs.ts index de2e71566..20429a324 100644 --- a/packages/indexer-common/src/subgraphs.ts +++ b/packages/indexer-common/src/subgraphs.ts @@ -151,6 +151,7 @@ export enum ActivationCriteria { NEVER = 'never', OFFCHAIN = 'offchain', INVALID_ALLOCATION_AMOUNT = 'invalid_allocation_amount', + DIPS = 'dips', } interface RuleMatch { @@ -276,6 +277,14 @@ export function isDeploymentWorthAllocatingTowards( deployment.protocolNetwork, ) + case IndexingDecisionBasis.DIPS: + return new AllocationDecision( + deployment.id, + deploymentRule, + true, + ActivationCriteria.DIPS, + deployment.protocolNetwork, + ) case IndexingDecisionBasis.ALWAYS: return new AllocationDecision( deployment.id, From 40cf61d398b410551dda2a4bf94526c06dfd1c0e Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Fri, 25 Jul 2025 17:45:55 +0000 Subject: [PATCH 2/7] feat: complete DIPs integration with ethers v6 migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Updated all DIPs-related code to use ethers v6 API - Migrated from BigNumberish to bigint for all numeric operations - Fixed provider and signer initialization patterns - Updated test suite to use new ethers v6 patterns - Removed temporary migration documentation 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- REBASE_STRATEGY.md | 232 ------------------ .../src/indexer-management/monitor.ts | 6 +- .../resolvers/allocations.ts | 4 +- .../src/indexing-fees/__tests__/dips.test.ts | 7 +- .../indexer-common/src/indexing-fees/dips.ts | 8 +- .../gateway-dips-service-client.ts | 31 ++- 6 files changed, 26 insertions(+), 262 deletions(-) delete mode 100644 REBASE_STRATEGY.md diff --git a/REBASE_STRATEGY.md b/REBASE_STRATEGY.md deleted file mode 100644 index eef3a438d..000000000 --- a/REBASE_STRATEGY.md +++ /dev/null @@ -1,232 +0,0 @@ -# DIPs-Horizon Rebase Strategy - -## Overview -This document tracks the merge conflict resolution strategy for rebasing DIPs (Distributed Indexing Payments) onto the Horizon branch. - -- **DIPs Branch**: Adds distributed indexing payments functionality -- **Horizon Branch**: Adds Graph Horizon protocol upgrade with GraphTally/RAV v2 - -## Conflict Files (8 total) - -### 1. packages/indexer-common/src/network.ts -**Status**: ❌ Unresolved - -**Horizon Changes**: -- Imports `GraphTallyCollector` and `encodeRegistrationData` -- Adds `graphTallyCollector: GraphTallyCollector | undefined` property -- Adds `isHorizon: Eventual` property -- Creates GraphTallyCollector instance for RAV v2 - -**DIPs Changes**: -- Imports `DipsCollector` -- Adds `dipsCollector: DipsCollector | undefined` property -- Adds `queryFeeModels: QueryFeeModels` property -- Adds `managementModels: IndexerManagementModels` property - -**Key Conflicts**: -1. Import section (lines 41-46) -2. Class properties (lines 58-72) -3. Constructor parameters (lines 86-92) -4. Constructor body (lines 106-112) -5. Network instantiation (lines 396-399) - -**Resolution Strategy**: -- [ ] Merge both collectors (GraphTallyCollector AND DipsCollector) -- [ ] Keep all properties from both branches -- [ ] Update constructor to accept all parameters -- [ ] Ensure both collectors can be initialized - ---- - -### 2. packages/indexer-common/src/operator.ts -**Status**: ❌ Unresolved - -**Horizon Changes**: -- `createAllocation` method takes `isHorizon: boolean` parameter -- Uses `isHorizon` to determine `isLegacy` flag on actions -- Passes `isLegacy: !isHorizon` to queueAction -- Also sets `isLegacy: allocation.isLegacy` when closing allocations - -**DIPs Changes**: -- `createAllocation` method takes `forceAction: boolean = false` parameter -- `queueAction` method signature changed to `async queueAction(action: ActionItem, forceAction: boolean = false)` -- Passes forceAction as second parameter to queueAction - -**Key Conflicts**: -1. createAllocation method signature (line 366-370) -2. queueAction calls - Horizon passes object with isLegacy, DIPs passes forceAction as 2nd param -3. closeEligibleAllocations also has forceAction parameter in DIPs -4. refreshExpiredAllocations has similar conflicts - -**Resolution Strategy**: -- [ ] Need both isHorizon AND forceAction parameters in allocation methods -- [ ] Update method signatures: `createAllocation(logger, decision, lastClosed, isHorizon, forceAction = false)` -- [ ] Merge queueAction calls to include both isLegacy (from Horizon) and forceAction (from DIPs) - ---- - -### 3. packages/indexer-common/src/query-fees/models.ts -**Status**: ❌ Unresolved - -**Horizon Changes**: -- Uses simpler Model type: `extends Model` -- id property is `public id!: number` - -**DIPs Changes**: -- Uses Model with creation attributes: `extends Model` -- id property is `public id!: CreationOptional` - -**Key Conflicts**: -- Single conflict at line 28-37 in ScalarTapReceipts class definition - -**Resolution Strategy**: -- [ ] Use DIPs version (more complete typing with CreationOptional) - ---- - -### 4. packages/indexer-common/package.json -**Status**: ❌ Unresolved - -**Horizon Changes**: -- `@graphprotocol/common-ts`: "3.0.1" (newer) -- `@graphprotocol/toolshed`: "0.6.5" -- `@semiotic-labs/tap-contracts-bindings`: "2.0.0" (newer) - -**DIPs Changes**: -- `@graphprotocol/common-ts`: "2.0.11" (older) -- `@semiotic-labs/tap-contracts-bindings`: "^1.2.1" (older) -- Adds DIPs-specific dependencies: - - `@bufbuild/protobuf`: "2.2.3" - - `@graphprotocol/dips-proto`: "0.2.2" - - `@grpc/grpc-js`: "^1.12.6" - -**Key Conflicts**: -- Dependency version mismatches - -**Resolution Strategy**: -- [ ] Use Horizon's newer versions -- [ ] Add DIPs-specific dependencies - ---- - -### 5. packages/indexer-common/src/indexer-management/allocations.ts -**Status**: ❌ Unresolved - -**Horizon Changes**: -- Empty constructor body - -**DIPs Changes**: -- Constructor initializes DipsManager if dipperEndpoint is configured -- Adds `dipsManager: DipsManager | null` property - -**Key Conflicts**: -- Constructor body (lines 131-139) - -**Resolution Strategy**: -- [ ] Keep DIPs initialization logic - ---- - -### 6. packages/indexer-common/src/indexer-management/resolvers/allocations.ts -**Status**: ❌ Unresolved - -**Horizon Changes**: -- Destructures `graphNode` from resolver context - -**DIPs Changes**: -- Destructures `actionManager` from resolver context - -**Key Conflicts**: -- reallocateAllocation resolver context destructuring (lines 1720-1724) - -**Resolution Strategy**: -- [ ] Include BOTH in destructuring: `{ logger, models, multiNetworks, graphNode, actionManager }` -- [ ] The IndexerManagementResolverContext interface already has both properties - ---- - -### 7. packages/indexer-agent/src/agent.ts -**Status**: ❌ Unresolved - -**Horizon Changes**: -- Passes `isHorizon` to createAllocation - -**DIPs Changes**: -- Passes `forceAction` to createAllocation - -**Key Conflicts**: -- createAllocation call (lines 1243-1247) - -**Resolution Strategy**: -- [ ] Pass both parameters: `createAllocation(logger, decision, lastClosed, isHorizon, forceAction)` - ---- - -### 8. yarn.lock -**Status**: ❌ Unresolved - -**Resolution Strategy**: -- [ ] Will regenerate after resolving package.json conflicts - ---- - -## General Notes -- Both branches introduce different payment/collection systems that need to coexist -- Horizon introduces protocol upgrade detection and legacy/horizon mode switching -- DIPs introduces indexing agreements and gateway payment integration - -## Important Context -- **Current branch**: dips-horizon-rebase -- **Base commit**: Squashed DIPs changes into single commit (35ceac2a) on top of 32d8f174 -- **Rebase status**: `git rebase origin/horizon` in progress with conflicts -- **To continue rebase**: After resolving conflicts, use `git add ` then `git rebase --continue` -- **To abort**: `git rebase --abort` if needed - -## Key Files/Imports Added by Each Branch -**Horizon**: -- `GraphTallyCollector` from './allocations/graph-tally-collector' -- `encodeRegistrationData` from '@graphprotocol/toolshed' -- `isHorizon` property for protocol upgrade detection -- `isLegacy` flag on actions - -**DIPs**: -- `DipsCollector` from './indexing-fees/dips' -- `DipsManager` class -- DIPs-specific dependencies in package.json -- `forceAction` parameter for manual allocation management -- New directory: `indexing-fees/` with DIPs implementation - -## Important Note: Method Call Analysis - -**Call sites found for modified methods:** -- `createAllocation`: Only 1 call in agent.ts (already in conflict) -- `closeEligibleAllocations`: 2 calls in agent.ts (already have forceAction parameter) -- `refreshExpiredAllocations`: 1 call in agent.ts (already has forceAction parameter) -- `queueAction`: 5 calls in operator.ts (all in conflicts) - -**Good news**: All method calls appear to be either: -1. Already in merge conflicts (so we'll handle them) -2. Already updated with the DIPs parameters (forceAction) - -**Action needed**: When resolving conflicts, ensure we add BOTH parameters where needed. - -## Resolution Summary - -### High Priority Decisions Needed: -1. **Method Signatures**: Most conflicts are about method parameters. We need both `isHorizon` (from Horizon) AND `forceAction` (from DIPs) -2. **Collectors**: We need both GraphTallyCollector (Horizon) and DipsCollector (DIPs) to coexist -3. **Dependencies**: Use Horizon's newer versions but add DIPs-specific dependencies - -### Recommended Approach: -1. Start with package.json - merge dependencies -2. Fix network.ts - ensure both collectors can exist -3. Fix operator.ts - update method signatures to accept both parameters -4. Fix agent.ts - pass both parameters -5. Fix remaining files with minor conflicts -6. Regenerate yarn.lock - -### Key Principle: -Both payment systems (Horizon's GraphTally and DIPs) should coexist. The system should support: -- Legacy allocations (pre-Horizon) -- Horizon allocations (with GraphTally/RAV v2) -- DIPs agreements (with distributed indexing payments) \ No newline at end of file diff --git a/packages/indexer-common/src/indexer-management/monitor.ts b/packages/indexer-common/src/indexer-management/monitor.ts index 4af48d55c..7fa51c744 100644 --- a/packages/indexer-common/src/indexer-management/monitor.ts +++ b/packages/indexer-common/src/indexer-management/monitor.ts @@ -687,9 +687,9 @@ export class NetworkMonitor { return { id: new SubgraphDeploymentID(ipfsHash), deniedAt: 1, // We assume the deployment won't be eligible for rewards if it's not found - stakedTokens: BigNumber.from(0), - signalledTokens: BigNumber.from(0), - queryFeesAmount: BigNumber.from(0), + stakedTokens: 0n, + signalledTokens: 0n, + queryFeesAmount: 0n, protocolNetwork: this.networkCAIPID, } } diff --git a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts index 77cb2d812..87faa6eeb 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts @@ -1638,7 +1638,7 @@ export default { await allocationManager.dipsManager.tryUpdateAgreementAllocation( deployment, null, - toAddress(createAllocationEventLogs.allocationID), + toAddress(allocationId), ) } @@ -1951,7 +1951,7 @@ export default { await allocationManager.dipsManager.tryUpdateAgreementAllocation( allocationData.subgraphDeployment.id.toString(), toAddress(allocation), - toAddress(createAllocationEventLogs.allocationID), + toAddress(newAllocationId), ) } diff --git a/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts b/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts index be56353f9..7189e560f 100644 --- a/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts +++ b/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts @@ -29,7 +29,6 @@ import { } from '@graphprotocol/common-ts' import { Sequelize } from 'sequelize' import { testNetworkSpecification } from '../../indexer-management/__tests__/util' -import { BigNumber } from 'ethers' import { CollectPaymentStatus } from '@graphprotocol/dips-proto/generated/gateway' // Make global Jest variables available @@ -65,9 +64,9 @@ const mockSubgraphDeployment = (id: string) => { id: new SubgraphDeploymentID(id), ipfsHash: id, deniedAt: null, - stakedTokens: BigNumber.from('1000'), - signalledTokens: BigNumber.from('1000'), - queryFeesAmount: BigNumber.from('0'), + stakedTokens: 1000n, + signalledTokens: 1000n, + queryFeesAmount: 0n, protocolNetwork: 'eip155:421614', } } diff --git a/packages/indexer-common/src/indexing-fees/dips.ts b/packages/indexer-common/src/indexing-fees/dips.ts index dc939ba64..15f76b6a5 100644 --- a/packages/indexer-common/src/indexing-fees/dips.ts +++ b/packages/indexer-common/src/indexing-fees/dips.ts @@ -34,7 +34,7 @@ import { } from '@graphprotocol/dips-proto/generated/gateway' import { IndexingAgreement } from '../indexer-management/models/indexing-agreement' import { NetworkSpecification } from '../network-specification' -import { Wallet } from 'ethers' +import { BaseWallet } from 'ethers' const DIPS_COLLECTION_INTERVAL = 60_000 @@ -364,7 +364,7 @@ export class DipsCollector { private queryFeeModels: QueryFeeModels, private specification: NetworkSpecification, private tapCollector: TapCollector, - private wallet: Wallet, + private wallet: BaseWallet, private graphNode: GraphNode, public escrowSenderGetter: GetEscrowSenderForSigner, ) { @@ -383,7 +383,7 @@ export class DipsCollector { queryFeeModels: QueryFeeModels, specification: NetworkSpecification, tapCollector: TapCollector, - wallet: Wallet, + wallet: BaseWallet, graphNode: GraphNode, escrowSenderGetter?: GetEscrowSenderForSigner, ) { @@ -467,7 +467,7 @@ export class DipsCollector { this.logger.info('Decoding TAP receipt for agreement') const tapReceipt = this.gatewayDipsServiceMessagesCodec.decodeTapReceipt( response.tapReceipt, - this.tapCollector?.tapContracts.tapVerifier.address, + this.tapCollector?.tapContracts.tapVerifier.target.toString(), ) // Check that the signer of the TAP receipt is a signer // on the corresponding escrow account for the payer (sender) of the diff --git a/packages/indexer-common/src/indexing-fees/gateway-dips-service-client.ts b/packages/indexer-common/src/indexing-fees/gateway-dips-service-client.ts index 1bfb832a5..1b8019cf8 100644 --- a/packages/indexer-common/src/indexing-fees/gateway-dips-service-client.ts +++ b/packages/indexer-common/src/indexing-fees/gateway-dips-service-client.ts @@ -1,13 +1,7 @@ import { Client, credentials } from '@grpc/grpc-js' import { UnaryCallback } from '@grpc/grpc-js/build/src/client' import { GatewayDipsServiceClientImpl } from '@graphprotocol/dips-proto/generated/gateway' -import { Wallet } from 'ethers' -import { - _TypedDataEncoder, - arrayify, - defaultAbiCoder, - recoverAddress, -} from 'ethers/lib/utils' +import { BaseWallet, TypedDataEncoder, getBytes, AbiCoder, recoverAddress } from 'ethers' import { toAddress } from '@graphprotocol/common-ts' type RpcImpl = (service: string, method: string, data: Uint8Array) => Promise @@ -46,15 +40,16 @@ export const collectPaymentsTypes = { export class GatewayDipsServiceMessagesCodec { async createSignedCancellationRequest( agreementId: string, - wallet: Wallet, + wallet: BaseWallet, ): Promise { - const signature = await wallet._signTypedData( + const signature = await wallet.signTypedData( cancelAgreementDomain, cancelAgreementTypes, { agreement_id: agreementId }, ) - return arrayify( - defaultAbiCoder.encode(['tuple(bytes16)', 'bytes'], [[agreementId], signature]), + const abiCoder = AbiCoder.defaultAbiCoder() + return getBytes( + abiCoder.encode(['tuple(bytes16)', 'bytes'], [[agreementId], signature]), ) } @@ -62,9 +57,9 @@ export class GatewayDipsServiceMessagesCodec { agreementId: string, allocationId: string, entityCount: number, - wallet: Wallet, + wallet: BaseWallet, ): Promise { - const signature = await wallet._signTypedData( + const signature = await wallet.signTypedData( collectPaymentsDomain, collectPaymentsTypes, { @@ -73,8 +68,9 @@ export class GatewayDipsServiceMessagesCodec { entity_count: entityCount, }, ) - return arrayify( - defaultAbiCoder.encode( + const abiCoder = AbiCoder.defaultAbiCoder() + return getBytes( + abiCoder.encode( ['tuple(bytes16, address, uint64)', 'bytes'], [[agreementId, toAddress(allocationId), entityCount], signature], ), @@ -82,7 +78,8 @@ export class GatewayDipsServiceMessagesCodec { } decodeTapReceipt(receipt: Uint8Array, verifyingContract: string) { - const [message, signature] = defaultAbiCoder.decode( + const abiCoder = AbiCoder.defaultAbiCoder() + const [message, signature] = abiCoder.decode( ['tuple(address,uint64,uint64,uint128)', 'bytes'], receipt, ) @@ -107,7 +104,7 @@ export class GatewayDipsServiceMessagesCodec { ], } - const digest = _TypedDataEncoder.hash(domain, types, { + const digest = TypedDataEncoder.hash(domain, types, { allocation_id: allocationId, timestamp_ns: timestampNs, nonce: nonce, From 2cbd218a45503b656e179a71fd07cd232bd71773 Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Fri, 1 Aug 2025 21:27:17 +0000 Subject: [PATCH 3/7] feat: implement receipt-based payment system for DIPs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add DipsReceipt model with snake_case fields matching database conventions - Update DipsCollector to store Receipt IDs instead of TAP receipts - Implement GetReceiptById polling in collectAllPayments method - Update to @graphprotocol/dips-proto 0.3.0 for new proto definitions - Remove TAP receipt dependencies from DIPs payment flow - Add comprehensive logging for payment status transitions This completes the indexer-agent implementation for the new DIPs Safe payment system, replacing TAP receipts with an asynchronous Receipt ID based approach. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- packages/indexer-common/package.json | 2 +- .../src/indexer-management/models/index.ts | 1 + .../models/indexing-agreement.ts | 95 +++++++++++++++ .../src/indexing-fees/__tests__/dips.test.ts | 14 +-- .../indexer-common/src/indexing-fees/dips.ts | 111 +++++++++++------- packages/indexer-common/src/network.ts | 2 - 6 files changed, 166 insertions(+), 59 deletions(-) diff --git a/packages/indexer-common/package.json b/packages/indexer-common/package.json index f99e829b6..c943bd625 100644 --- a/packages/indexer-common/package.json +++ b/packages/indexer-common/package.json @@ -25,7 +25,7 @@ "@pinax/graph-networks-registry": "0.6.7", "@bufbuild/protobuf": "2.2.3", "@graphprotocol/common-ts": "3.0.1", - "@graphprotocol/dips-proto": "0.2.2", + "@graphprotocol/dips-proto": "0.3.0", "@graphprotocol/toolshed": "1.1.1", "@grpc/grpc-js": "^1.12.6", "@semiotic-labs/tap-contracts-bindings": "2.0.0", diff --git a/packages/indexer-common/src/indexer-management/models/index.ts b/packages/indexer-common/src/indexer-management/models/index.ts index 81a59f4d3..26f52cb63 100644 --- a/packages/indexer-common/src/indexer-management/models/index.ts +++ b/packages/indexer-common/src/indexer-management/models/index.ts @@ -10,6 +10,7 @@ export * from './cost-model' export * from './indexing-rule' export * from './poi-dispute' export * from './action' +export * from './indexing-agreement' export type IndexerManagementModels = IndexingRuleModels & CostModelModels & diff --git a/packages/indexer-common/src/indexer-management/models/indexing-agreement.ts b/packages/indexer-common/src/indexer-management/models/indexing-agreement.ts index 462456a7c..1fceaee34 100644 --- a/packages/indexer-common/src/indexer-management/models/indexing-agreement.ts +++ b/packages/indexer-common/src/indexer-management/models/indexing-agreement.ts @@ -6,6 +6,7 @@ import { CreationOptional, InferCreationAttributes, InferAttributes, + ForeignKey, } from 'sequelize' // Indexing Fees AKA "DIPs" @@ -40,8 +41,26 @@ export class IndexingAgreement extends Model< declare last_payment_collected_at: Date | null } +export type DipsReceiptStatus = 'PENDING' | 'SUBMITTED' | 'FAILED' + +export class DipsReceipt extends Model< + InferAttributes, + InferCreationAttributes +> { + declare id: string // Primary key - Receipt ID from Dipper + declare agreement_id: ForeignKey + declare amount: string + declare status: DipsReceiptStatus + declare transaction_hash: string | null + declare error_message: string | null + declare created_at: CreationOptional + declare updated_at: CreationOptional + declare retry_count: CreationOptional +} + export interface IndexingFeesModels { IndexingAgreement: typeof IndexingAgreement + DipsReceipt: typeof DipsReceipt } export const defineIndexingFeesModels = (sequelize: Sequelize): IndexingFeesModels => { @@ -209,7 +228,83 @@ export const defineIndexingFeesModels = (sequelize: Sequelize): IndexingFeesMode }, ) + DipsReceipt.init( + { + id: { + type: DataTypes.STRING(255), + primaryKey: true, + allowNull: false, + }, + agreement_id: { + type: DataTypes.UUID, + allowNull: false, + references: { + model: IndexingAgreement, + key: 'id', + }, + }, + amount: { + type: DataTypes.DECIMAL(39), + allowNull: false, + }, + status: { + type: DataTypes.ENUM('PENDING', 'SUBMITTED', 'FAILED'), + allowNull: false, + defaultValue: 'PENDING', + }, + transaction_hash: { + type: DataTypes.CHAR(66), + allowNull: true, + }, + error_message: { + type: DataTypes.TEXT, + allowNull: true, + }, + created_at: { + type: DataTypes.DATE, + allowNull: false, + }, + updated_at: { + type: DataTypes.DATE, + allowNull: false, + }, + retry_count: { + type: DataTypes.INTEGER, + allowNull: false, + defaultValue: 0, + }, + }, + { + modelName: 'DipsReceipt', + sequelize, + tableName: 'dips_receipts', + timestamps: true, + createdAt: 'created_at', + updatedAt: 'updated_at', + indexes: [ + { + fields: ['agreement_id'], + }, + { + fields: ['status'], + }, + ], + }, + ) + + // Define associations + DipsReceipt.belongsTo(IndexingAgreement, { + foreignKey: 'agreement_id', + as: 'agreement', + }) + + IndexingAgreement.hasMany(DipsReceipt, { + foreignKey: 'agreement_id', + as: 'receipts', + }) + return { ['IndexingAgreement']: IndexingAgreement, + ['DipsReceipt']: DipsReceipt, } } diff --git a/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts b/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts index 7189e560f..3fe1d46b5 100644 --- a/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts +++ b/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts @@ -454,12 +454,9 @@ describe('DipsCollector', () => { const dipsCollector = new DipsCollector( logger, managementModels, - queryFeeModels, networkSpecWithDips, - network.tapCollector!, network.wallet, graphNode, - jest.fn(), ) expect(dipsCollector).toBeDefined() }) @@ -467,12 +464,9 @@ describe('DipsCollector', () => { const dipsCollector = new DipsCollector( logger, managementModels, - queryFeeModels, networkSpecWithDips, - network.tapCollector!, network.wallet, graphNode, - jest.fn(), ) expect(dipsCollector).toBeDefined() expect(startCollectionLoop).toHaveBeenCalled() @@ -490,12 +484,9 @@ describe('DipsCollector', () => { new DipsCollector( logger, managementModels, - queryFeeModels, specWithoutDipper, - network.tapCollector!, network.wallet, graphNode, - jest.fn(), ), ).toThrow('dipperEndpoint is not set') }) @@ -570,10 +561,7 @@ describe('DipsCollector', () => { value: '1000', } }) - dipsCollector.escrowSenderGetter = jest.fn().mockImplementation(() => { - logger.info('MOCK Getting escrow sender for signer') - return toAddress('0x123456df40c29949a75a6693c77834c00b8a5678') - }) + // escrowSenderGetter has been removed from DipsCollector await dipsCollector.tryCollectPayment(agreement) diff --git a/packages/indexer-common/src/indexing-fees/dips.ts b/packages/indexer-common/src/indexing-fees/dips.ts index 15f76b6a5..7b8c66c18 100644 --- a/packages/indexer-common/src/indexing-fees/dips.ts +++ b/packages/indexer-common/src/indexing-fees/dips.ts @@ -9,17 +9,14 @@ import { ActionStatus, Allocation, AllocationManager, - getEscrowSenderForSigner, + DipsReceiptStatus, GraphNode, IndexerManagementModels, IndexingDecisionBasis, IndexingRuleAttributes, Network, - QueryFeeModels, sequentialTimerMap, - SubgraphClient, SubgraphIdentifierType, - TapCollector, upsertIndexingRule, } from '@graphprotocol/indexer-common' import { Op } from 'sequelize' @@ -46,10 +43,6 @@ const normalizeAddressForDB = (address: string) => { return toAddress(address).toLowerCase().replace('0x', '') } -type GetEscrowSenderForSigner = ( - tapSubgraph: SubgraphClient, - signer: Address, -) => Promise
export class DipsManager { declare gatewayDipsServiceClient: GatewayDipsServiceClientImpl declare gatewayDipsServiceMessagesCodec: GatewayDipsServiceMessagesCodec @@ -361,12 +354,9 @@ export class DipsCollector { constructor( private logger: Logger, private managementModels: IndexerManagementModels, - private queryFeeModels: QueryFeeModels, private specification: NetworkSpecification, - private tapCollector: TapCollector, private wallet: BaseWallet, private graphNode: GraphNode, - public escrowSenderGetter: GetEscrowSenderForSigner, ) { if (!this.specification.indexerOptions.dipperEndpoint) { throw new Error('dipperEndpoint is not set') @@ -380,22 +370,16 @@ export class DipsCollector { static create( logger: Logger, managementModels: IndexerManagementModels, - queryFeeModels: QueryFeeModels, specification: NetworkSpecification, - tapCollector: TapCollector, wallet: BaseWallet, graphNode: GraphNode, - escrowSenderGetter?: GetEscrowSenderForSigner, ) { const collector = new DipsCollector( logger, managementModels, - queryFeeModels, specification, - tapCollector, wallet, graphNode, - escrowSenderGetter ?? getEscrowSenderForSigner, ) collector.startCollectionLoop() return collector @@ -421,6 +405,7 @@ export class DipsCollector { // Collect payments for all outstanding agreements async collectAllPayments() { + // Part 1: Collect new payments const outstandingAgreements = await this.managementModels.IndexingAgreement.findAll({ where: { last_payment_collected_at: null, @@ -432,6 +417,53 @@ export class DipsCollector { for (const agreement of outstandingAgreements) { await this.tryCollectPayment(agreement) } + + // Part 2: Poll pending receipts + await this.pollPendingReceipts() + } + + async pollPendingReceipts() { + // Find all pending receipts + const pendingReceipts = await this.managementModels.DipsReceipt.findAll({ + where: { + status: 'PENDING', + }, + }) + + if (pendingReceipts.length === 0) { + return + } + + this.logger.info(`Polling ${pendingReceipts.length} pending receipts`) + + for (const receipt of pendingReceipts) { + try { + const statusResponse = await this.gatewayDipsServiceClient.GetReceiptById({ + version: 1, + receiptId: receipt.id, + }) + + if (statusResponse.status !== receipt.status) { + const oldStatus = receipt.status + receipt.status = statusResponse.status as DipsReceiptStatus + receipt.transaction_hash = statusResponse.transactionHash || null + receipt.error_message = statusResponse.errorMessage || null + await receipt.save() + + this.logger.info( + `Receipt ${receipt.id} status updated from ${oldStatus} to ${statusResponse.status}`, + { + receiptId: receipt.id, + oldStatus: oldStatus, + newStatus: statusResponse.status, + transactionHash: statusResponse.transactionHash, + }, + ) + } + } catch (error) { + this.logger.error(`Error polling receipt ${receipt.id}`, { error }) + } + } } async tryCollectPayment(agreement: IndexingAgreement) { if (!agreement.last_allocation_id) { @@ -460,36 +492,29 @@ export class DipsCollector { signedCollection: collection, }) if (response.status === CollectPaymentStatus.ACCEPT) { - if (!this.tapCollector) { - throw new Error('TapCollector not initialized') - } - // Store the tap receipt in the database - this.logger.info('Decoding TAP receipt for agreement') - const tapReceipt = this.gatewayDipsServiceMessagesCodec.decodeTapReceipt( - response.tapReceipt, - this.tapCollector?.tapContracts.tapVerifier.target.toString(), - ) - // Check that the signer of the TAP receipt is a signer - // on the corresponding escrow account for the payer (sender) of the - // indexing agreement - const escrowSender = await this.escrowSenderGetter( - this.tapCollector?.tapSubgraph, - tapReceipt.signer_address, - ) - if (escrowSender !== toAddress(agreement.payer)) { - // TODO: should we cancel the agreement here? - throw new Error( - 'Signer of TAP receipt is not a signer on the indexing agreement', - ) - } - if (tapReceipt.allocation_id !== toAddress(agreement.last_allocation_id)) { - throw new Error('Allocation ID mismatch') - } - await this.queryFeeModels.scalarTapReceipts.create(tapReceipt) + const receiptId = response.receiptId + const amount = response.amount + + // Store the receipt ID in the database + this.logger.info(`Received receipt ID ${receiptId} for agreement ${agreement.id}`) + + // Create DipsReceipt record with PENDING status + await this.managementModels.DipsReceipt.create({ + id: receiptId, + agreement_id: agreement.id, + amount: amount, + status: 'PENDING', + retry_count: 0, + }) + // Mark the agreement as having had a payment collected agreement.last_payment_collected_at = new Date() agreement.updated_at = new Date() await agreement.save() + + this.logger.info( + `Payment collection initiated for agreement ${agreement.id}, receipt ID: ${receiptId}`, + ) } else { throw new Error(`Payment request not accepted: ${response.status}`) } diff --git a/packages/indexer-common/src/network.ts b/packages/indexer-common/src/network.ts index 50788e692..8b35b3378 100644 --- a/packages/indexer-common/src/network.ts +++ b/packages/indexer-common/src/network.ts @@ -334,9 +334,7 @@ export class Network { dipsCollector = DipsCollector.create( logger, managementModels, - queryFeeModels, specification, - tapCollector, wallet, graphNode, ) From 7c9328da017380652c732900f2d481f5829b4355 Mon Sep 17 00:00:00 2001 From: Miguel de Elias Date: Wed, 18 Feb 2026 18:43:33 -0300 Subject: [PATCH 4/7] fix: update DIP test mock to match receipt-based payment response The third DIP commit changed tryCollectPayment to use receipt-based responses (receiptId, amount) but the test still mocked the old tapReceipt-based response shape. Also regenerates yarn.lock with new DIP dependencies. --- .../src/indexing-fees/__tests__/dips.test.ts | 28 +--- yarn.lock | 130 +++++++++++++++++- 2 files changed, 136 insertions(+), 22 deletions(-) diff --git a/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts b/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts index 3fe1d46b5..fbd1597bc 100644 --- a/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts +++ b/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts @@ -546,22 +546,9 @@ describe('DipsCollector', () => { client.CollectPayment = jest.fn().mockResolvedValue({ version: 1, status: CollectPaymentStatus.ACCEPT, - tapReceipt: Buffer.from('1234', 'hex'), + receiptId: 'test-receipt-id-123', + amount: '1000', }) - dipsCollector.gatewayDipsServiceMessagesCodec.decodeTapReceipt = jest - .fn() - .mockImplementation(() => { - logger.info('MOCK Decoding TAP receipt') - return { - allocation_id: toAddress(testAllocationId), - signer_address: toAddress('0xabcd56df41234949a75a6693c77834c00b8abbbb'), - signature: Buffer.from('1234', 'hex'), - timestamp_ns: 1234567890, - nonce: 1, - value: '1000', - } - }) - // escrowSenderGetter has been removed from DipsCollector await dipsCollector.tryCollectPayment(agreement) @@ -571,16 +558,15 @@ describe('DipsCollector', () => { }) expect(agreement.last_payment_collected_at).not.toBeNull() - const receipt = await queryFeeModels.scalarTapReceipts.findOne({ + const receipt = await managementModels.DipsReceipt.findOne({ where: { - allocation_id: testAllocationId, + agreement_id: agreement.id, }, }) expect(receipt).not.toBeNull() - expect(receipt?.signer_address).toBe( - toAddress('0xabcd56df41234949a75a6693c77834c00b8abbbb'), - ) - expect(receipt?.value).toBe('1000') + expect(receipt?.id).toBe('test-receipt-id-123') + expect(receipt?.amount).toBe('1000') + expect(receipt?.status).toBe('PENDING') }) }) }) diff --git a/yarn.lock b/yarn.lock index fb5650084..bb5e646da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -286,6 +286,16 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@bufbuild/protobuf@2.2.3": + version "2.2.3" + resolved "https://registry.yarnpkg.com/@bufbuild/protobuf/-/protobuf-2.2.3.tgz#9cd136f6b687e63e9b517b3a54211ece942897ee" + integrity sha512-tFQoXHJdkEOSwj5tRIZSPNUuXK3RaR7T1nUrPgbYX1pUbvqqaaZAsfo+NXBPsz5rZMSKVFrgK1WL8Q/MSLvprg== + +"@bufbuild/protobuf@^2.2.3": + version "2.11.0" + resolved "https://registry.yarnpkg.com/@bufbuild/protobuf/-/protobuf-2.11.0.tgz#3ec3985c9074b23aea337957225fe15a0e845f8e" + integrity sha512-sBXGT13cpmPR5BMgHE6UEEfEaShh5Ror6rfN3yEK5si7QVrtZg8LEPQb0VVhiLRUslD2yLnXtnRzG035J/mZXQ== + "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b" @@ -573,6 +583,13 @@ prom-client "14.2.0" sequelize "6.33.0" +"@graphprotocol/dips-proto@0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@graphprotocol/dips-proto/-/dips-proto-0.3.0.tgz#97eccaabdf449479fb083865d4697e3c0725f870" + integrity sha512-zZ6mqG/OKe21PxOunnhWkz2qq5zZl+JD0n7PSFgVNq61J8lZs22Jal78utvOSsJd/E5vyNoSxELGCOyZmc4QJw== + dependencies: + "@bufbuild/protobuf" "^2.2.3" + "@graphprotocol/interfaces@^0.6.4": version "0.6.4" resolved "https://registry.yarnpkg.com/@graphprotocol/interfaces/-/interfaces-0.6.4.tgz#b7aa58219c62cae19a6746a794648d473374a308" @@ -602,6 +619,24 @@ hardhat "^2.26.0" json5 "^2.2.3" +"@grpc/grpc-js@^1.12.6": + version "1.14.3" + resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.14.3.tgz#4c9b817a900ae4020ddc28515ae4b52c78cfb8da" + integrity sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA== + dependencies: + "@grpc/proto-loader" "^0.8.0" + "@js-sdsl/ordered-map" "^4.4.2" + +"@grpc/proto-loader@^0.8.0": + version "0.8.0" + resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.8.0.tgz#b6c324dd909c458a0e4aa9bfd3d69cf78a4b9bd8" + integrity sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ== + dependencies: + lodash.camelcase "^4.3.0" + long "^5.0.0" + protobufjs "^7.5.3" + yargs "^17.7.2" + "@humanwhocodes/config-array@^0.11.11", "@humanwhocodes/config-array@^0.11.13": version "0.11.14" resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.14.tgz#d78e481a039f7566ecc9660b4ea7fe6b1fec442b" @@ -888,6 +923,11 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@js-sdsl/ordered-map@^4.4.2": + version "4.4.2" + resolved "https://registry.yarnpkg.com/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz#9299f82874bab9e4c7f9c48d865becbfe8d6907c" + integrity sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw== + "@lerna/add@6.1.0": version "6.1.0" resolved "https://registry.yarnpkg.com/@lerna/add/-/add-6.1.0.tgz#0f09495c5e1af4c4f316344af34b6d1a91b15b19" @@ -2158,6 +2198,59 @@ resolved "https://registry.yarnpkg.com/@pinax/graph-networks-registry/-/graph-networks-registry-0.6.7.tgz#ceb994f3b31e2943b9c9d9b09dd86eb00d067c0e" integrity sha512-xogeCEZ50XRMxpBwE3TZjJ8RCO8Guv39gDRrrKtlpDEDEMLm0MzD3A0SQObgj7aF7qTZNRTWzsuvQdxgzw25wQ== +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== + +"@protobufjs/inquire@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== + +"@protobufjs/utf8@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== + "@rushstack/node-core-library@5.13.0": version "5.13.0" resolved "https://registry.yarnpkg.com/@rushstack/node-core-library/-/node-core-library-5.13.0.tgz#f79d6868b74be102eee75b93c37be45fb9b47ead" @@ -2753,6 +2846,13 @@ dependencies: undici-types "~6.19.2" +"@types/node@>=13.7.0": + version "25.2.3" + resolved "https://registry.yarnpkg.com/@types/node/-/node-25.2.3.tgz#9c18245be768bdb4ce631566c7da303a5c99a7f8" + integrity sha512-m0jEgYlYz+mDJZ2+F4v8D1AyQb+QzsNqRuI7xg1VQX/KlKS0qT9r1Mo16yo5F/MtifXFgaofIFsdFMox2SxIbQ== + dependencies: + undici-types "~7.16.0" + "@types/node@^12.12.54": version "12.20.55" resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.55.tgz#c329cbd434c42164f846b909bd6f85b5537f6240" @@ -7779,6 +7879,11 @@ log-symbols@^4.1.0: chalk "^4.1.0" is-unicode-supported "^0.1.0" +long@^5.0.0: + version "5.3.2" + resolved "https://registry.yarnpkg.com/long/-/long-5.3.2.tgz#1d84463095999262d7d7b7f8bfd4a8cc55167f83" + integrity sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA== + loose-envify@^1.0.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -9357,6 +9462,24 @@ proto-list@~1.2.1: resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849" integrity sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA== +protobufjs@^7.5.3: + version "7.5.4" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.5.4.tgz#885d31fe9c4b37f25d1bb600da30b1c5b37d286a" + integrity sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/node" ">=13.7.0" + long "^5.0.0" + protocols@^2.0.0, protocols@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/protocols/-/protocols-2.0.2.tgz#822e8fcdcb3df5356538b3e91bfd890b067fd0a4" @@ -10935,6 +11058,11 @@ undici-types@~6.21.0: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.21.0.tgz#691d00af3909be93a7faa13be61b3a5b50ef12cb" integrity sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ== +undici-types@~7.16.0: + version "7.16.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.16.0.tgz#ffccdff36aea4884cbfce9a750a0580224f58a46" + integrity sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw== + undici@^5.14.0: version "5.29.0" resolved "https://registry.yarnpkg.com/undici/-/undici-5.29.0.tgz#419595449ae3f2cdcba3580a2e8903399bd1f5a3" @@ -11409,7 +11537,7 @@ yargs@^16.2.0: y18n "^5.0.5" yargs-parser "^20.2.2" -yargs@^17.3.1, yargs@^17.6.2: +yargs@^17.3.1, yargs@^17.6.2, yargs@^17.7.2: version "17.7.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== From 4867f1cf394726ea206cb655fb569142b286bad9 Mon Sep 17 00:00:00 2001 From: Miguel de Elias Date: Wed, 25 Feb 2026 17:16:08 -0300 Subject: [PATCH 5/7] chore: update toolshed to dips supported version --- packages/indexer-common/package.json | 2 +- .../src/indexer-management/allocations.ts | 78 ++++++++++--------- yarn.lock | 35 ++++++--- 3 files changed, 66 insertions(+), 49 deletions(-) diff --git a/packages/indexer-common/package.json b/packages/indexer-common/package.json index c943bd625..6e22e134f 100644 --- a/packages/indexer-common/package.json +++ b/packages/indexer-common/package.json @@ -26,7 +26,7 @@ "@bufbuild/protobuf": "2.2.3", "@graphprotocol/common-ts": "3.0.1", "@graphprotocol/dips-proto": "0.3.0", - "@graphprotocol/toolshed": "1.1.1", + "@graphprotocol/toolshed": "1.2.0-dips.0", "@grpc/grpc-js": "^1.12.6", "@semiotic-labs/tap-contracts-bindings": "2.0.0", "@thi.ng/heaps": "1.2.38", diff --git a/packages/indexer-common/src/indexer-management/allocations.ts b/packages/indexer-common/src/indexer-management/allocations.ts index 6be66c0c5..9974e7029 100644 --- a/packages/indexer-common/src/indexer-management/allocations.ts +++ b/packages/indexer-common/src/indexer-management/allocations.ts @@ -1076,12 +1076,16 @@ export class AllocationManager { force, ) - // Double-check whether the legacy allocation is still active on chain - const state = await this.network.contracts.HorizonStaking.getAllocationState( - allocation.id, - ) - if (state !== 1n) { - throw indexerError(IndexerErrorCode.IE065) + // Double-check whether the allocation is still active on chain, to + // avoid unnecessary transactions. + // TODO: Remove legacy allocation path — getAllocationState no longer exists + // in HorizonStaking. Legacy allocations now live in SubgraphService. + if (!allocation.isLegacy) { + const allocation = + await this.network.contracts.SubgraphService.getAllocation(allocationID) + if (allocation.closedAt !== 0n) { + throw indexerError(IndexerErrorCode.IE065) + } } return { @@ -1230,17 +1234,13 @@ export class AllocationManager { poiData: params.poi, }) + // TODO: Remove legacy allocation path — closeAllocation no longer exists + // in HorizonStaking. Legacy allocations now live in SubgraphService. if (params.isLegacy) { - const tx = - await this.network.contracts.HorizonStaking.closeAllocation.populateTransaction( - params.allocationID, - params.poi.poi, - ) - return { - protocolNetwork: params.protocolNetwork, - actionID: params.actionID, - ...tx, - } + throw indexerError( + IndexerErrorCode.IE065, + 'Legacy allocation close is no longer supported', + ) } // Horizon: Need to collect indexing rewards and stop service @@ -1697,17 +1697,19 @@ export class AllocationManager { // Double-check whether the allocation is still active on chain, to // avoid unnecessary transactions. + // TODO: Remove legacy allocation path — getAllocationState no longer exists + // in HorizonStaking. Legacy allocations now live in SubgraphService. if (allocation.isLegacy) { - const state = await this.network.contracts.HorizonStaking.getAllocationState( - allocation.id, - ) - if (state !== 1n) { - logger.warn(`Allocation has already been closed`) - throw indexerError( - IndexerErrorCode.IE065, - `Legacy allocation has already been closed`, - ) - } + // const state = await this.network.contracts.HorizonStaking.getAllocationState( + // allocation.id, + // ) + // if (state !== 1n) { + // logger.warn(`Allocation has already been closed`) + // throw indexerError( + // IndexerErrorCode.IE065, + // `Legacy allocation has already been closed`, + // ) + // } } else { const allocationData = await this.network.contracts.SubgraphService.getAllocation(allocationID) @@ -1759,18 +1761,20 @@ export class AllocationManager { }) throw indexerError(IndexerErrorCode.IE066, 'AllocationID already exists') } + // TODO: Remove legacy allocation path — getAllocationState no longer exists + // in HorizonStaking. Legacy allocations now live in SubgraphService. } else { - const newAllocationState = - await this.network.contracts.HorizonStaking.getAllocationState(newAllocationId) - if (newAllocationState !== 0n) { - logger.warn(`Skipping allocation as it already exists onchain (legacy)`, { - indexer: this.network.specification.indexerOptions.address, - allocation: newAllocationId, - newAllocationState, - isHorizon, - }) - throw indexerError(IndexerErrorCode.IE066, 'Legacy AllocationID already exists') - } + // const newAllocationState = + // await this.network.contracts.HorizonStaking.getAllocationState(newAllocationId) + // if (newAllocationState !== 0n) { + // logger.warn(`Skipping allocation as it already exists onchain (legacy)`, { + // indexer: this.network.specification.indexerOptions.address, + // allocation: newAllocationId, + // newAllocationState, + // isHorizon, + // }) + // throw indexerError(IndexerErrorCode.IE066, 'Legacy AllocationID already exists') + // } } logger.debug('Generating new allocation ID proof', { diff --git a/yarn.lock b/yarn.lock index bb5e646da..6dd908242 100644 --- a/yarn.lock +++ b/yarn.lock @@ -590,10 +590,17 @@ dependencies: "@bufbuild/protobuf" "^2.2.3" -"@graphprotocol/interfaces@^0.6.4": - version "0.6.4" - resolved "https://registry.yarnpkg.com/@graphprotocol/interfaces/-/interfaces-0.6.4.tgz#b7aa58219c62cae19a6746a794648d473374a308" - integrity sha512-Gpm3FR9aFgg8ChV7hkU64+97dpi9N4y0cVCh/8z/nUOG7JSzz1fkpYjP1Ql16EUZKKvHzBjor9qsV9asxQUF6Q== +"@graphprotocol/interfaces@^0.7.0-dips.0": + version "0.7.0-dips.0" + resolved "https://registry.yarnpkg.com/@graphprotocol/interfaces/-/interfaces-0.7.0-dips.0.tgz#0b325ad8e02051c51418c301b294dbb97110ca39" + integrity sha512-H82KvrO5OvFapKJQOkmYY/IPINz3ytwbg6D4G0VevIs01+SP30yE8nlJxLGDifnHC8ETNZN6B3iCz/DpiW77wQ== + +"@graphprotocol/issuance@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@graphprotocol/issuance/-/issuance-1.0.0.tgz#f3b5a723e4f42cd0d8a8ad0fae9ef5773f48954c" + integrity sha512-PX7vSndgo+SUfGnQWzkZQOd4dwVtMFo4dMjIVaHx1qS2H0yLwzgyulcD9iaD8HaVFwQ1jUlGnKBV1AKP61qgeA== + dependencies: + "@noble/hashes" "^1.8.0" "@graphprotocol/pino-sentry-simple@0.7.1": version "0.7.1" @@ -605,16 +612,17 @@ split2 "^3.1.1" through2 "^3.0.1" -"@graphprotocol/toolshed@1.1.1": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@graphprotocol/toolshed/-/toolshed-1.1.1.tgz#eb57f20a70ed415f1a601d49b4e2fb3cad1929b3" - integrity sha512-55XP4k+XqfXTlBzO2wbK54Ll1IFE4pI/KUrAoD6Y4tb5sXOMn77qUxdk09DccfNZpFs761/l0C7SGbpIoP1Rzg== +"@graphprotocol/toolshed@1.2.0-dips.0": + version "1.2.0-dips.0" + resolved "https://registry.yarnpkg.com/@graphprotocol/toolshed/-/toolshed-1.2.0-dips.0.tgz#b4855b0ceec6370d919c74d8fa56dbd3053b40f4" + integrity sha512-QAaZtc0XT2sC/e1T5DTat51FaZiH8UrzVnDS4II/YUsV0zIVjTfCwWvbBLn684FEC9bXfIDVnh7D7tB6vy4tOw== dependencies: "@graphprotocol/address-book" "^1.1.0" - "@graphprotocol/interfaces" "^0.6.4" + "@graphprotocol/interfaces" "^0.7.0-dips.0" + "@graphprotocol/issuance" "^1.0.0" "@nomicfoundation/hardhat-ethers" "^3.1.0" debug "^4.4.0" - ethers "^6.15.0" + ethers "^6.16.0" glob "^11.0.1" hardhat "^2.26.0" json5 "^2.2.3" @@ -1653,6 +1661,11 @@ resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.7.2.tgz#d53c65a21658fb02f3303e7ee3ba89d6754c64b4" integrity sha512-biZ0NUSxyjLLqo6KxEJ1b+C2NAx0wtDoFvCaXHGgUkeHzf3Xc1xKumFKREuT7f7DARNZ/slvYUwFG6B0f2b6hQ== +"@noble/hashes@^1.8.0": + version "1.8.0" + resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.8.0.tgz#cee43d801fcef9644b11b8194857695acd5f815a" + integrity sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A== + "@noble/secp256k1@1.7.1": version "1.7.1" resolved "https://registry.yarnpkg.com/@noble/secp256k1/-/secp256k1-1.7.1.tgz#b251c70f824ce3ca7f8dc3df08d58f005cc0507c" @@ -5353,7 +5366,7 @@ ethereum-cryptography@^2.2.1: "@scure/bip32" "1.4.0" "@scure/bip39" "1.3.0" -ethers@6.13.7, ethers@^6.15.0: +ethers@6.13.7, ethers@^6.16.0: version "6.13.7" resolved "https://registry.yarnpkg.com/ethers/-/ethers-6.13.7.tgz#7457fcb32413b441a3ee6e9f4cd63bf782de6226" integrity sha512-qbaJ0uIrjh+huP1Lad2f2QtzW5dcqSVjIzVH6yWB4dKoMuj2WqYz5aMeeQTCNpAKgTJBM5J9vcc2cYJ23UAimQ== From e8eb72f7c34fde9f3f663480d8dd1e5d1ac28985 Mon Sep 17 00:00:00 2001 From: Miguel de Elias Date: Thu, 26 Feb 2026 10:56:03 -0300 Subject: [PATCH 6/7] fix: formatting error --- packages/indexer-common/src/indexer-management/allocations.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/indexer-common/src/indexer-management/allocations.ts b/packages/indexer-common/src/indexer-management/allocations.ts index 9974e7029..c4ead2449 100644 --- a/packages/indexer-common/src/indexer-management/allocations.ts +++ b/packages/indexer-common/src/indexer-management/allocations.ts @@ -1761,8 +1761,8 @@ export class AllocationManager { }) throw indexerError(IndexerErrorCode.IE066, 'AllocationID already exists') } - // TODO: Remove legacy allocation path — getAllocationState no longer exists - // in HorizonStaking. Legacy allocations now live in SubgraphService. + // TODO: Remove legacy allocation path — getAllocationState no longer exists + // in HorizonStaking. Legacy allocations now live in SubgraphService. } else { // const newAllocationState = // await this.network.contracts.HorizonStaking.getAllocationState(newAllocationId) From 87552cb26a98bbd9b0265eea434e7f69c9d5e55e Mon Sep 17 00:00:00 2001 From: Maikol <86025070+Maikol@users.noreply.github.com> Date: Fri, 10 Apr 2026 10:12:22 -0300 Subject: [PATCH 7/7] feat: add pending RCA proposal consumer and migration (#1174) --- packages/indexer-agent/src/agent.ts | 5 + packages/indexer-agent/src/commands/start.ts | 5 + .../23-add-pending-rca-proposals.ts | 74 +++ .../src/indexer-management/actions.ts | 3 + .../src/indexer-management/allocations.ts | 10 +- .../src/indexer-management/client.ts | 12 +- .../src/indexer-management/models/index.ts | 1 + .../models/pending-rca-proposal.ts | 55 ++ .../__tests__/accept-proposals.test.ts | 593 ++++++++++++++++++ .../src/indexing-fees/__tests__/dips.test.ts | 13 +- .../__tests__/pending-rca-consumer.test.ts | 256 ++++++++ .../indexer-common/src/indexing-fees/dips.ts | 418 +++++++++++- .../indexer-common/src/indexing-fees/index.ts | 2 + .../src/indexing-fees/pending-rca-consumer.ts | 83 +++ .../indexer-common/src/indexing-fees/types.ts | 28 + .../src/network-specification.ts | 2 +- 16 files changed, 1544 insertions(+), 16 deletions(-) create mode 100644 packages/indexer-agent/src/db/migrations/23-add-pending-rca-proposals.ts create mode 100644 packages/indexer-common/src/indexer-management/models/pending-rca-proposal.ts create mode 100644 packages/indexer-common/src/indexing-fees/__tests__/accept-proposals.test.ts create mode 100644 packages/indexer-common/src/indexing-fees/__tests__/pending-rca-consumer.test.ts create mode 100644 packages/indexer-common/src/indexing-fees/pending-rca-consumer.ts create mode 100644 packages/indexer-common/src/indexing-fees/types.ts diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index 6ff9320db..afe7c401a 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -716,6 +716,11 @@ export class Agent { if (!operator.dipsManager) { throw new Error('DipsManager is not available') } + + await operator.dipsManager.acceptPendingProposals( + activeAllocations, + ) + this.logger.debug( `Matching agreement allocations for network ${network.specification.networkIdentifier}`, ) diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts index 655582d48..fd68c5a8e 100644 --- a/packages/indexer-agent/src/commands/start.ts +++ b/packages/indexer-agent/src/commands/start.ts @@ -15,6 +15,7 @@ import { createIndexerManagementClient, createIndexerManagementServer, defineIndexerManagementModels, + definePendingRcaProposalModel, defineQueryFeeModels, GraphNode, indexerError, @@ -670,6 +671,9 @@ export async function run( await sequelize.sync() logger.info(`Successfully synced database models`) + // Define after sync so Sequelize won't try to create/alter this indexer-rs-owned table + const pendingRcaModel = definePendingRcaProposalModel(sequelize) + // -------------------------------------------------------------------------------- // * Networks // -------------------------------------------------------------------------------- @@ -710,6 +714,7 @@ export async function run( }, }, multiNetworks, + pendingRcaModel, }) // -------------------------------------------------------------------------------- diff --git a/packages/indexer-agent/src/db/migrations/23-add-pending-rca-proposals.ts b/packages/indexer-agent/src/db/migrations/23-add-pending-rca-proposals.ts new file mode 100644 index 000000000..66bf70ede --- /dev/null +++ b/packages/indexer-agent/src/db/migrations/23-add-pending-rca-proposals.ts @@ -0,0 +1,74 @@ +import { Logger } from '@graphprotocol/common-ts' + +import { QueryInterface, DataTypes } from 'sequelize' + +interface MigrationContext { + queryInterface: QueryInterface + logger: Logger +} + +interface Context { + context: MigrationContext +} + +export async function up({ context }: Context): Promise { + const { queryInterface, logger } = context + + const tables = await queryInterface.showAllTables() + logger.debug('Checking if pending_rca_proposals table exists', { tables }) + + if (tables.includes('pending_rca_proposals')) { + logger.debug( + 'pending_rca_proposals already exists, migration not necessary', + ) + return + } + + logger.info('Create pending_rca_proposals') + await queryInterface.createTable('pending_rca_proposals', { + id: { + type: DataTypes.UUID, + primaryKey: true, + }, + signed_payload: { + type: DataTypes.BLOB, + allowNull: false, + }, + version: { + type: DataTypes.SMALLINT, + allowNull: false, + defaultValue: 2, + }, + status: { + type: DataTypes.STRING(20), + allowNull: false, + defaultValue: 'pending', + }, + created_at: { + type: DataTypes.DATE, + allowNull: false, + }, + updated_at: { + type: DataTypes.DATE, + allowNull: false, + }, + }) + + await queryInterface.addIndex( + 'pending_rca_proposals', + ['status', 'created_at'], + { + name: 'idx_pending_rca_status', + }, + ) + await queryInterface.addIndex('pending_rca_proposals', { + fields: [{ name: 'created_at', order: 'DESC' }], + name: 'idx_pending_rca_created', + }) +} + +export async function down({ context }: Context): Promise { + const { queryInterface, logger } = context + logger.info('Drop pending_rca_proposals') + await queryInterface.dropTable('pending_rca_proposals') +} diff --git a/packages/indexer-common/src/indexer-management/actions.ts b/packages/indexer-common/src/indexer-management/actions.ts index 1f6900de8..b00f26312 100644 --- a/packages/indexer-common/src/indexer-management/actions.ts +++ b/packages/indexer-common/src/indexer-management/actions.ts @@ -24,6 +24,7 @@ import { import { Order, Transaction } from 'sequelize' import { Eventual, join, Logger } from '@graphprotocol/common-ts' import groupBy from 'lodash.groupby' +import { PendingRcaProposal } from './models/pending-rca-proposal' export class ActionManager { declare multiNetworks: MultiNetworks @@ -38,6 +39,7 @@ export class ActionManager { logger: Logger, models: IndexerManagementModels, graphNode: GraphNode, + pendingRcaModel?: typeof PendingRcaProposal, ): Promise { const actionManager = new ActionManager() actionManager.multiNetworks = multiNetworks @@ -52,6 +54,7 @@ export class ActionManager { models, graphNode, network, + pendingRcaModel, ) }) diff --git a/packages/indexer-common/src/indexer-management/allocations.ts b/packages/indexer-common/src/indexer-management/allocations.ts index c4ead2449..6fc02c116 100644 --- a/packages/indexer-common/src/indexer-management/allocations.ts +++ b/packages/indexer-common/src/indexer-management/allocations.ts @@ -47,6 +47,7 @@ import { encodeStopServiceData, PaymentTypes, } from '@graphprotocol/toolshed' +import { PendingRcaProposal } from './models/pending-rca-proposal' import { encodeCollectIndexingRewardsData, encodePOIMetadata, @@ -166,9 +167,16 @@ export class AllocationManager { private models: IndexerManagementModels, private graphNode: GraphNode, private network: Network, + private pendingRcaModel?: typeof PendingRcaProposal, ) { if (this.network.specification.indexerOptions.dipperEndpoint) { - this.dipsManager = new DipsManager(this.logger, this.models, this.network, this) + this.dipsManager = new DipsManager( + this.logger, + this.models, + this.network, + this, + this.pendingRcaModel, + ) } } diff --git a/packages/indexer-common/src/indexer-management/client.ts b/packages/indexer-common/src/indexer-management/client.ts index 041f7d65f..c21a37c14 100644 --- a/packages/indexer-common/src/indexer-management/client.ts +++ b/packages/indexer-common/src/indexer-management/client.ts @@ -20,6 +20,7 @@ import { Network, RulesManager, } from '@graphprotocol/indexer-common' +import { PendingRcaProposal } from './models/pending-rca-proposal' export interface IndexerManagementResolverContext { models: IndexerManagementModels @@ -574,6 +575,7 @@ export interface IndexerManagementClientOptions { multiNetworks: MultiNetworks | undefined defaults: IndexerManagementDefaults actionManager?: ActionManager | undefined + pendingRcaModel?: typeof PendingRcaProposal } export class IndexerManagementClient extends Client { @@ -595,7 +597,7 @@ export class IndexerManagementClient extends Client { export const createIndexerManagementClient = async ( options: IndexerManagementClientOptions, ): Promise => { - const { models, graphNode, logger, defaults, multiNetworks } = options + const { models, graphNode, logger, defaults, multiNetworks, pendingRcaModel } = options const schema = buildSchema(print(SCHEMA_SDL)) const resolvers = { ...indexingRuleResolvers, @@ -608,7 +610,13 @@ export const createIndexerManagementClient = async ( } const actionManager = multiNetworks - ? await ActionManager.create(multiNetworks, logger, models, graphNode) + ? await ActionManager.create( + multiNetworks, + logger, + models, + graphNode, + pendingRcaModel, + ) : undefined const rulesManager = multiNetworks diff --git a/packages/indexer-common/src/indexer-management/models/index.ts b/packages/indexer-common/src/indexer-management/models/index.ts index 26f52cb63..4fc550528 100644 --- a/packages/indexer-common/src/indexer-management/models/index.ts +++ b/packages/indexer-common/src/indexer-management/models/index.ts @@ -11,6 +11,7 @@ export * from './indexing-rule' export * from './poi-dispute' export * from './action' export * from './indexing-agreement' +export * from './pending-rca-proposal' export type IndexerManagementModels = IndexingRuleModels & CostModelModels & diff --git a/packages/indexer-common/src/indexer-management/models/pending-rca-proposal.ts b/packages/indexer-common/src/indexer-management/models/pending-rca-proposal.ts new file mode 100644 index 000000000..b3baa5d7a --- /dev/null +++ b/packages/indexer-common/src/indexer-management/models/pending-rca-proposal.ts @@ -0,0 +1,55 @@ +import { DataTypes, Sequelize, Model, InferAttributes } from 'sequelize' + +export class PendingRcaProposal extends Model> { + declare id: string + declare signed_payload: Buffer + declare version: number + declare status: string + declare created_at: Date + declare updated_at: Date +} + +export const definePendingRcaProposalModel = ( + sequelize: Sequelize, +): typeof PendingRcaProposal => { + PendingRcaProposal.init( + { + id: { + type: DataTypes.UUID, + primaryKey: true, + }, + signed_payload: { + type: DataTypes.BLOB, + allowNull: false, + }, + version: { + type: DataTypes.SMALLINT, + allowNull: false, + defaultValue: 2, + }, + status: { + type: DataTypes.STRING(20), + allowNull: false, + defaultValue: 'pending', + }, + created_at: { + type: DataTypes.DATE, + allowNull: false, + }, + updated_at: { + type: DataTypes.DATE, + allowNull: false, + }, + }, + { + modelName: 'PendingRcaProposal', + sequelize, + tableName: 'pending_rca_proposals', + timestamps: true, + createdAt: 'created_at', + updatedAt: 'updated_at', + }, + ) + + return PendingRcaProposal +} diff --git a/packages/indexer-common/src/indexing-fees/__tests__/accept-proposals.test.ts b/packages/indexer-common/src/indexing-fees/__tests__/accept-proposals.test.ts new file mode 100644 index 000000000..e09a7c6b3 --- /dev/null +++ b/packages/indexer-common/src/indexing-fees/__tests__/accept-proposals.test.ts @@ -0,0 +1,593 @@ +import { createLogger, Logger, SubgraphDeploymentID } from '@graphprotocol/common-ts' +import { DipsManager } from '../dips' +import { PendingRcaConsumer } from '../pending-rca-consumer' +import { DecodedRcaProposal } from '../types' +import { + Allocation, + AllocationStatus, + IndexerManagementModels, + Network, +} from '@graphprotocol/indexer-common' + +let logger: Logger + +beforeAll(() => { + logger = createLogger({ + name: 'AcceptProposals Test', + async: false, + level: 'error', + }) +}) + +const TEST_DEPLOYMENT_BYTES32 = + '0x0100000000000000000000000000000000000000000000000000000000000000' + +function createMockProposal( + overrides: Partial = {}, +): DecodedRcaProposal { + const deployment = new SubgraphDeploymentID( + overrides.subgraphDeploymentId?.bytes32 ?? TEST_DEPLOYMENT_BYTES32, + ) + return { + id: 'proposal-1', + status: 'pending', + createdAt: new Date(), + signedRca: { + rca: { + deadline: BigInt(Math.floor(Date.now() / 1000) + 3600), + endsAt: BigInt(Math.floor(Date.now() / 1000) + 86400), + payer: '0x1111111111111111111111111111111111111111', + dataService: '0x2222222222222222222222222222222222222222', + serviceProvider: '0x3333333333333333333333333333333333333333', + maxInitialTokens: 10000n, + maxOngoingTokensPerSecond: 100n, + minSecondsPerCollection: 3600n, + maxSecondsPerCollection: 86400n, + nonce: 42n, + metadata: '0x', + }, + signature: '0xaabbccdd', + }, + signedPayload: new Uint8Array(), + payer: '0x1111111111111111111111111111111111111111', + serviceProvider: '0x3333333333333333333333333333333333333333', + dataService: '0x2222222222222222222222222222222222222222', + deadline: BigInt(Math.floor(Date.now() / 1000) + 3600), + endsAt: BigInt(Math.floor(Date.now() / 1000) + 86400), + maxInitialTokens: 10000n, + maxOngoingTokensPerSecond: 100n, + minSecondsPerCollection: 3600n, + maxSecondsPerCollection: 86400n, + nonce: 42n, + subgraphDeploymentId: deployment, + tokensPerSecond: 1000n, + tokensPerEntityPerSecond: 50n, + ...overrides, + } +} + +function createMockAllocation( + deploymentBytes32: string = TEST_DEPLOYMENT_BYTES32, +): Allocation { + return { + id: '0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + status: AllocationStatus.ACTIVE, + isLegacy: false, + subgraphDeployment: { + id: new SubgraphDeploymentID(deploymentBytes32), + }, + indexer: '0x5555555555555555555555555555555555555555', + allocatedTokens: 1000000000000000000n, + createdAt: 0, + createdAtEpoch: 100, + createdAtBlockHash: '0x', + closedAt: 0, + closedAtEpoch: 0, + closedAtEpochStartBlockHash: undefined, + previousEpochStartBlockHash: undefined, + closedAtBlockHash: '0x', + poi: undefined, + queryFeeRebates: 0n, + queryFeesCollected: 0n, + } as Allocation +} + +function createMockConsumer(proposals: DecodedRcaProposal[] = []) { + return { + getPendingProposals: jest.fn().mockResolvedValue(proposals), + getPendingProposalsForDeployment: jest.fn().mockResolvedValue([]), + markAccepted: jest.fn().mockResolvedValue(undefined), + markRejected: jest.fn().mockResolvedValue(undefined), + } as unknown as PendingRcaConsumer +} + +function createMockModels() { + return { + IndexingRule: { + findOne: jest.fn().mockResolvedValue(null), + findAll: jest.fn().mockResolvedValue([]), + destroy: jest.fn().mockResolvedValue(1), + }, + } as unknown as IndexerManagementModels +} + +function createMockNetwork() { + return { + contracts: { + SubgraphService: { + acceptIndexingAgreement: Object.assign(jest.fn(), { + estimateGas: jest.fn().mockResolvedValue(100000n), + populateTransaction: jest.fn().mockResolvedValue({ data: '0xaccept' }), + }), + startService: { + populateTransaction: jest.fn().mockResolvedValue({ data: '0xstart' }), + }, + multicall: Object.assign(jest.fn(), { + estimateGas: jest.fn().mockResolvedValue(200000n), + }), + getAllocation: jest.fn().mockResolvedValue({ createdAt: 0n }), + getLegacyAllocation: jest + .fn() + .mockResolvedValue({ indexer: '0x0000000000000000000000000000000000000000' }), + target: '0x4444444444444444444444444444444444444444', + indexers: jest.fn().mockResolvedValue({ url: 'http://test' }), + }, + EpochManager: { + currentEpoch: jest.fn().mockResolvedValue(100n), + }, + RewardsManager: { + isDenied: jest.fn().mockResolvedValue(false), + }, + }, + transactionManager: { + executeTransaction: jest.fn(), + wallet: { + mnemonic: { + phrase: 'test test test test test test test test test test test junk', + }, + }, + }, + networkMonitor: { + currentEpoch: jest.fn().mockResolvedValue(100n), + }, + specification: { + indexerOptions: { + address: '0x5555555555555555555555555555555555555555', + enableDips: true, + dipsAllocationAmount: 0n, + defaultAllocationAmount: 10000000000000000000n, // 10 GRT + }, + networkIdentifier: 'eip155:1337', + }, + isHorizon: { value: jest.fn().mockResolvedValue(true) }, + } as unknown as Network +} + +function createDipsManager( + network: Network, + models: IndexerManagementModels, + consumer: PendingRcaConsumer, +): DipsManager { + const dm = new DipsManager(logger, models, network, null) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ;(dm as any).pendingRcaConsumer = consumer + return dm +} + +describe('DipsManager.acceptPendingProposals', () => { + test('rejects proposals with expired deadlines', async () => { + const expiredProposal = createMockProposal({ + deadline: BigInt(Math.floor(Date.now() / 1000) - 100), // expired + }) + const consumer = createMockConsumer([expiredProposal]) + const models = createMockModels() + const network = createMockNetwork() + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([]) + + expect(consumer.markRejected).toHaveBeenCalledWith( + expiredProposal.id, + 'deadline_expired', + ) + expect(consumer.markAccepted).not.toHaveBeenCalled() + }) + + test('cleans up DIPS rule when rejecting last proposal for a deployment', async () => { + const proposal = createMockProposal({ + deadline: BigInt(Math.floor(Date.now() / 1000) - 100), + }) + const consumer = createMockConsumer([proposal]) + // After rejection, no other proposals for this deployment + ;(consumer.getPendingProposalsForDeployment as jest.Mock).mockResolvedValue([]) + const mockRule = { id: 42 } + const models = createMockModels() + ;(models.IndexingRule.findOne as jest.Mock).mockResolvedValue(mockRule) + + const network = createMockNetwork() + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([]) + + expect(models.IndexingRule.destroy).toHaveBeenCalledWith({ where: { id: 42 } }) + }) + + test('does not clean up DIPS rule when other proposals exist for deployment', async () => { + const proposal = createMockProposal({ + deadline: BigInt(Math.floor(Date.now() / 1000) - 100), + }) + const otherProposal = createMockProposal({ id: 'proposal-2' }) + const consumer = createMockConsumer([proposal]) + // Another proposal exists for same deployment + ;(consumer.getPendingProposalsForDeployment as jest.Mock).mockResolvedValue([ + otherProposal, + ]) + const models = createMockModels() + const network = createMockNetwork() + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([]) + + expect(models.IndexingRule.destroy).not.toHaveBeenCalled() + }) + + test('returns early when no pending proposals', async () => { + const consumer = createMockConsumer([]) + const models = createMockModels() + const network = createMockNetwork() + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([]) + + expect(consumer.markAccepted).not.toHaveBeenCalled() + expect(consumer.markRejected).not.toHaveBeenCalled() + }) + + test('returns early when pendingRcaConsumer is null', async () => { + const models = createMockModels() + const network = createMockNetwork() + const dm = new DipsManager(logger, models, network, null) + + // Should not throw + await dm.acceptPendingProposals([]) + }) + + describe('with existing allocation', () => { + test('accepts proposal on-chain and marks accepted', async () => { + const proposal = createMockProposal() + const allocation = createMockAllocation() + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const network = createMockNetwork() + const mockReceipt = { hash: '0xtxhash', status: 1 } + ;(network.transactionManager.executeTransaction as jest.Mock).mockResolvedValue( + mockReceipt, + ) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([allocation]) + + expect(network.transactionManager.executeTransaction).toHaveBeenCalled() + expect(consumer.markAccepted).toHaveBeenCalledWith(proposal.id) + }) + + test('skips acceptance when network is paused', async () => { + const proposal = createMockProposal() + const allocation = createMockAllocation() + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const network = createMockNetwork() + ;(network.transactionManager.executeTransaction as jest.Mock).mockResolvedValue( + 'paused', + ) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([allocation]) + + expect(consumer.markAccepted).not.toHaveBeenCalled() + expect(consumer.markRejected).not.toHaveBeenCalled() + }) + + test('skips acceptance when unauthorized', async () => { + const proposal = createMockProposal() + const allocation = createMockAllocation() + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const network = createMockNetwork() + ;(network.transactionManager.executeTransaction as jest.Mock).mockResolvedValue( + 'unauthorized', + ) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([allocation]) + + expect(consumer.markAccepted).not.toHaveBeenCalled() + expect(consumer.markRejected).not.toHaveBeenCalled() + }) + + test('uses multicall path when allocation is for different deployment', async () => { + const proposal = createMockProposal() + const differentDeployment = + '0x0200000000000000000000000000000000000000000000000000000000000000' + const allocation = createMockAllocation(differentDeployment) + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const network = createMockNetwork() + const mockReceipt = { hash: '0xmulticall', status: 1 } + ;(network.transactionManager.executeTransaction as jest.Mock).mockResolvedValue( + mockReceipt, + ) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([allocation]) + + // Should go to acceptWithNewAllocation (multicall) path, not existing allocation + expect( + network.contracts.SubgraphService.startService.populateTransaction, + ).toHaveBeenCalled() + expect(consumer.markAccepted).toHaveBeenCalledWith(proposal.id) + }) + }) + + describe('with new allocation (multicall)', () => { + test('creates allocation and accepts in single multicall', async () => { + const proposal = createMockProposal() + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const network = createMockNetwork() + const mockReceipt = { hash: '0xmulticallhash', status: 1 } + ;(network.transactionManager.executeTransaction as jest.Mock).mockResolvedValue( + mockReceipt, + ) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([]) // no active allocations + + expect(network.transactionManager.executeTransaction).toHaveBeenCalled() + expect(consumer.markAccepted).toHaveBeenCalledWith(proposal.id) + }) + + test('skips when allocation already exists on-chain', async () => { + const proposal = createMockProposal() + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const network = createMockNetwork() + // Allocation exists on-chain + ;( + network.contracts.SubgraphService.getAllocation as unknown as jest.Mock + ).mockResolvedValue({ + createdAt: 100n, + }) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([]) + + expect(consumer.markAccepted).not.toHaveBeenCalled() + expect(network.transactionManager.executeTransaction).not.toHaveBeenCalled() + }) + + test('uses dipsAllocationAmount for token amount', async () => { + const proposal = createMockProposal() + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const network = createMockNetwork() + const mockReceipt = { hash: '0x', status: 1 } + ;(network.transactionManager.executeTransaction as jest.Mock).mockResolvedValue( + mockReceipt, + ) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([]) + + // Verify startService was called with the configured allocation amount + expect( + network.contracts.SubgraphService.startService.populateTransaction, + ).toHaveBeenCalled() + }) + + test('handles paused network during multicall', async () => { + const proposal = createMockProposal() + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const network = createMockNetwork() + ;(network.transactionManager.executeTransaction as jest.Mock).mockResolvedValue( + 'paused', + ) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([]) + + expect(consumer.markAccepted).not.toHaveBeenCalled() + expect(consumer.markRejected).not.toHaveBeenCalled() + }) + }) + + describe('error handling', () => { + test('rejects proposal on deterministic CALL_EXCEPTION error', async () => { + const proposal = createMockProposal() + const allocation = createMockAllocation() + const consumer = createMockConsumer([proposal]) + // After rejection, no remaining proposals for cleanup + ;(consumer.getPendingProposalsForDeployment as jest.Mock).mockResolvedValue([]) + const models = createMockModels() + const network = createMockNetwork() + ;(network.transactionManager.executeTransaction as jest.Mock).mockRejectedValue({ + code: 'CALL_EXCEPTION', + data: '0x', + }) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([allocation]) + + expect(consumer.markRejected).toHaveBeenCalledWith(proposal.id, expect.any(String)) + }) + + test('leaves proposal pending on transient network error', async () => { + const proposal = createMockProposal() + const allocation = createMockAllocation() + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const network = createMockNetwork() + ;(network.transactionManager.executeTransaction as jest.Mock).mockRejectedValue( + new Error('ECONNRESET'), + ) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([allocation]) + + expect(consumer.markRejected).not.toHaveBeenCalled() + expect(consumer.markAccepted).not.toHaveBeenCalled() + }) + + test('continues processing other proposals after error', async () => { + const failProposal = createMockProposal({ id: 'fail-1' }) + const okDeployment = + '0x0200000000000000000000000000000000000000000000000000000000000000' + const okProposal = createMockProposal({ + id: 'ok-1', + subgraphDeploymentId: new SubgraphDeploymentID(okDeployment), + }) + const consumer = createMockConsumer([failProposal, okProposal]) + const models = createMockModels() + const network = createMockNetwork() + const failAllocation = { + id: '0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + status: AllocationStatus.ACTIVE, + isLegacy: false, + subgraphDeployment: { + id: new SubgraphDeploymentID(TEST_DEPLOYMENT_BYTES32), + }, + } as Allocation + const okAllocation = { + id: '0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + status: AllocationStatus.ACTIVE, + isLegacy: false, + subgraphDeployment: { + id: new SubgraphDeploymentID(okDeployment), + }, + } as Allocation + + const mockReceipt = { hash: '0x', status: 1 } + ;(network.transactionManager.executeTransaction as jest.Mock) + .mockRejectedValueOnce(new Error('first fails')) + .mockResolvedValueOnce(mockReceipt) // second succeeds + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([failAllocation, okAllocation]) + + // Second proposal should still be processed and accepted + expect(consumer.markAccepted).toHaveBeenCalledWith('ok-1') + }) + }) + + describe('allocation amount selection', () => { + test('uses defaultAllocationAmount for rewarded (not denied) subgraph', async () => { + const proposal = createMockProposal() + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const network = createMockNetwork() + ;( + network.contracts.RewardsManager.isDenied as unknown as jest.Mock + ).mockResolvedValue(false) + const mockReceipt = { hash: '0x', status: 1 } + ;(network.transactionManager.executeTransaction as jest.Mock).mockResolvedValue( + mockReceipt, + ) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([]) + + // Should check isDenied + expect(network.contracts.RewardsManager.isDenied).toHaveBeenCalledWith( + proposal.subgraphDeploymentId.bytes32, + ) + // startService should be called (new allocation path) + expect( + network.contracts.SubgraphService.startService.populateTransaction, + ).toHaveBeenCalled() + }) + + test('uses dipsAllocationAmount (0) for denied subgraph', async () => { + const proposal = createMockProposal() + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const network = createMockNetwork() + ;( + network.contracts.RewardsManager.isDenied as unknown as jest.Mock + ).mockResolvedValue(true) + const mockReceipt = { hash: '0x', status: 1 } + ;(network.transactionManager.executeTransaction as jest.Mock).mockResolvedValue( + mockReceipt, + ) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([]) + + expect(network.contracts.RewardsManager.isDenied).toHaveBeenCalledWith( + proposal.subgraphDeploymentId.bytes32, + ) + expect(consumer.markAccepted).toHaveBeenCalledWith(proposal.id) + }) + + test('uses per-deployment rule amount for rewarded subgraph with existing rule', async () => { + const proposal = createMockProposal() + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const ruleAmount = 5000000000000000000n // 5 GRT + ;(models.IndexingRule.findOne as jest.Mock).mockResolvedValue({ + allocationAmount: ruleAmount.toString(), + }) + const network = createMockNetwork() + ;( + network.contracts.RewardsManager.isDenied as unknown as jest.Mock + ).mockResolvedValue(false) + const mockReceipt = { hash: '0x', status: 1 } + ;(network.transactionManager.executeTransaction as jest.Mock).mockResolvedValue( + mockReceipt, + ) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([]) + + expect(consumer.markAccepted).toHaveBeenCalledWith(proposal.id) + }) + + test('uses custom dipsAllocationAmount for denied subgraph with override', async () => { + const proposal = createMockProposal() + const consumer = createMockConsumer([proposal]) + const models = createMockModels() + const network = createMockNetwork() + // Override dipsAllocationAmount to non-zero + ;( + network.specification.indexerOptions as { dipsAllocationAmount: bigint } + ).dipsAllocationAmount = 1000000000000000000n // 1 GRT + ;( + network.contracts.RewardsManager.isDenied as unknown as jest.Mock + ).mockResolvedValue(true) + const mockReceipt = { hash: '0x', status: 1 } + ;(network.transactionManager.executeTransaction as jest.Mock).mockResolvedValue( + mockReceipt, + ) + + const dm = createDipsManager(network, models, consumer) + + await dm.acceptPendingProposals([]) + + expect(consumer.markAccepted).toHaveBeenCalledWith(proposal.id) + }) + }) +}) diff --git a/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts b/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts index fbd1597bc..fcb8c807d 100644 --- a/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts +++ b/packages/indexer-common/src/indexing-fees/__tests__/dips.test.ts @@ -179,7 +179,7 @@ describe('DipsManager', () => { expect(dipsManager).toBeDefined() }) - test('throws error when dipperEndpoint is not configured', async () => { + test('creates DipsManager without gRPC client when dipperEndpoint is not configured', async () => { const specWithoutDipper = { ...testNetworkSpecification, indexerOptions: { @@ -197,9 +197,14 @@ describe('DipsManager', () => { graphNode, metrics, ) - expect( - () => new DipsManager(logger, managementModels, networkWithoutDipper, null), - ).toThrow('dipperEndpoint is not set') + const dipsManager = new DipsManager( + logger, + managementModels, + networkWithoutDipper, + null, + ) + expect(dipsManager).toBeDefined() + expect(dipsManager.gatewayDipsServiceClient).toBeUndefined() }) }) diff --git a/packages/indexer-common/src/indexing-fees/__tests__/pending-rca-consumer.test.ts b/packages/indexer-common/src/indexing-fees/__tests__/pending-rca-consumer.test.ts new file mode 100644 index 000000000..961609826 --- /dev/null +++ b/packages/indexer-common/src/indexing-fees/__tests__/pending-rca-consumer.test.ts @@ -0,0 +1,256 @@ +import { ethers } from 'ethers' +import { createLogger, Logger, SubgraphDeploymentID } from '@graphprotocol/common-ts' +import { PendingRcaConsumer } from '../pending-rca-consumer' +import { PendingRcaProposal } from '../../indexer-management/models/pending-rca-proposal' + +// ABI tuple types matching toolshed's recurring-collector.js +const RCA_TUPLE = + 'tuple(uint64 deadline, uint64 endsAt, address payer, address dataService, address serviceProvider, uint256 maxInitialTokens, uint256 maxOngoingTokensPerSecond, uint32 minSecondsPerCollection, uint32 maxSecondsPerCollection, uint256 nonce, bytes metadata)' +const SIGNED_RCA_TUPLE = `tuple(${RCA_TUPLE} rca, bytes signature)` +const ACCEPT_METADATA_TUPLE = + 'tuple(bytes32 subgraphDeploymentId, uint8 version, bytes terms)' +const TERMS_V1_TUPLE = 'tuple(uint256 tokensPerSecond, uint256 tokensPerEntityPerSecond)' + +const coder = ethers.AbiCoder.defaultAbiCoder() + +// Test data +const TEST_PAYER = '0x1234567890abcdef1234567890abcdef12345678' +const TEST_DATA_SERVICE = '0xabcdefabcdefabcdefabcdefabcdefabcdefabcd' +const TEST_SERVICE_PROVIDER = '0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef' +const TEST_DEPLOYMENT_BYTES32 = + '0x0100000000000000000000000000000000000000000000000000000000000000' +const TEST_SIGNATURE = '0xaabbccdd' + +function encodeTestPayload(overrides?: { + deadline?: bigint + endsAt?: bigint + tokensPerSecond?: bigint + tokensPerEntityPerSecond?: bigint + minSecondsPerCollection?: number + maxSecondsPerCollection?: number +}): Buffer { + const tokensPerSecond = overrides?.tokensPerSecond ?? 1000n + const tokensPerEntityPerSecond = overrides?.tokensPerEntityPerSecond ?? 50n + + const termsEncoded = coder.encode( + [TERMS_V1_TUPLE], + [{ tokensPerSecond, tokensPerEntityPerSecond }], + ) + + const metadataEncoded = coder.encode( + [ACCEPT_METADATA_TUPLE], + [ + { + subgraphDeploymentId: TEST_DEPLOYMENT_BYTES32, + version: 1n, + terms: termsEncoded, + }, + ], + ) + + const signedRcaEncoded = coder.encode( + [SIGNED_RCA_TUPLE], + [ + { + rca: { + deadline: overrides?.deadline ?? 1700000000n, + endsAt: overrides?.endsAt ?? 1800000000n, + payer: TEST_PAYER, + dataService: TEST_DATA_SERVICE, + serviceProvider: TEST_SERVICE_PROVIDER, + maxInitialTokens: 10000n, + maxOngoingTokensPerSecond: 100n, + minSecondsPerCollection: overrides?.minSecondsPerCollection ?? 3600, + maxSecondsPerCollection: overrides?.maxSecondsPerCollection ?? 86400, + nonce: 42n, + metadata: metadataEncoded, + }, + signature: TEST_SIGNATURE, + }, + ], + ) + + return Buffer.from(ethers.getBytes(signedRcaEncoded)) +} + +let logger: Logger + +beforeAll(() => { + logger = createLogger({ + name: 'PendingRcaConsumer Test', + async: false, + level: 'error', + }) +}) + +function createMockModel(rows: Partial[] = []) { + return { + findAll: jest.fn().mockResolvedValue(rows), + update: jest.fn().mockResolvedValue([1]), + } as unknown as typeof PendingRcaProposal +} + +describe('PendingRcaConsumer', () => { + describe('getPendingProposals', () => { + test('decodes valid pending proposals', async () => { + const payload = encodeTestPayload() + const model = createMockModel([ + { + id: 'test-uuid-1', + signed_payload: payload, + version: 2, + status: 'pending', + created_at: new Date('2024-01-01'), + updated_at: new Date('2024-01-01'), + }, + ]) + + const consumer = new PendingRcaConsumer(logger, model) + const proposals = await consumer.getPendingProposals() + + expect(proposals).toHaveLength(1) + const p = proposals[0] + + expect(p.id).toBe('test-uuid-1') + expect(p.status).toBe('pending') + expect(p.payer.toLowerCase()).toBe(TEST_PAYER.toLowerCase()) + expect(p.serviceProvider.toLowerCase()).toBe(TEST_SERVICE_PROVIDER.toLowerCase()) + expect(p.dataService.toLowerCase()).toBe(TEST_DATA_SERVICE.toLowerCase()) + expect(p.deadline).toBe(1700000000n) + expect(p.endsAt).toBe(1800000000n) + expect(p.maxInitialTokens).toBe(10000n) + expect(p.maxOngoingTokensPerSecond).toBe(100n) + expect(p.minSecondsPerCollection).toBe(3600n) + expect(p.maxSecondsPerCollection).toBe(86400n) + expect(p.nonce).toBe(42n) + expect(p.tokensPerSecond).toBe(1000n) + expect(p.tokensPerEntityPerSecond).toBe(50n) + + expect(p.subgraphDeploymentId).toBeInstanceOf(SubgraphDeploymentID) + expect(p.subgraphDeploymentId.bytes32).toBe(TEST_DEPLOYMENT_BYTES32) + + expect(p.signedRca).toBeDefined() + expect(p.signedRca.rca.payer.toLowerCase()).toBe(TEST_PAYER.toLowerCase()) + expect(p.signedRca.signature).toBe(TEST_SIGNATURE) + + expect(p.signedPayload).toBeInstanceOf(Uint8Array) + }) + + test('queries only pending rows', async () => { + const model = createMockModel([]) + const consumer = new PendingRcaConsumer(logger, model) + + await consumer.getPendingProposals() + + expect(model.findAll).toHaveBeenCalledWith({ + where: { status: 'pending' }, + }) + }) + + test('skips rows with corrupt payloads and logs warning', async () => { + const warnSpy = jest.fn() + const testLogger = { + ...logger, + warn: warnSpy, + child: () => testLogger, + } as unknown as Logger + + const model = createMockModel([ + { + id: 'good-uuid', + signed_payload: encodeTestPayload(), + version: 2, + status: 'pending', + created_at: new Date(), + updated_at: new Date(), + }, + { + id: 'bad-uuid', + signed_payload: Buffer.from('deadbeef', 'hex'), + version: 2, + status: 'pending', + created_at: new Date(), + updated_at: new Date(), + }, + ]) + + const consumer = new PendingRcaConsumer(testLogger, model) + const proposals = await consumer.getPendingProposals() + + expect(proposals).toHaveLength(1) + expect(proposals[0].id).toBe('good-uuid') + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('bad-uuid'), + expect.any(Object), + ) + }) + + test('decodes multiple proposals', async () => { + const model = createMockModel([ + { + id: 'uuid-1', + signed_payload: encodeTestPayload({ tokensPerSecond: 100n }), + version: 2, + status: 'pending', + created_at: new Date(), + updated_at: new Date(), + }, + { + id: 'uuid-2', + signed_payload: encodeTestPayload({ tokensPerSecond: 200n }), + version: 2, + status: 'pending', + created_at: new Date(), + updated_at: new Date(), + }, + ]) + + const consumer = new PendingRcaConsumer(logger, model) + const proposals = await consumer.getPendingProposals() + + expect(proposals).toHaveLength(2) + expect(proposals[0].tokensPerSecond).toBe(100n) + expect(proposals[1].tokensPerSecond).toBe(200n) + }) + }) + + describe('markAccepted', () => { + test('updates status to accepted', async () => { + const model = createMockModel() + const consumer = new PendingRcaConsumer(logger, model) + + await consumer.markAccepted('test-uuid') + + expect(model.update).toHaveBeenCalledWith( + { status: 'accepted' }, + { where: { id: 'test-uuid' } }, + ) + }) + }) + + describe('markRejected', () => { + test('updates status to rejected', async () => { + const model = createMockModel() + const consumer = new PendingRcaConsumer(logger, model) + + await consumer.markRejected('test-uuid', 'deployment blocklisted') + + expect(model.update).toHaveBeenCalledWith( + { status: 'rejected' }, + { where: { id: 'test-uuid' } }, + ) + }) + + test('updates status without reason', async () => { + const model = createMockModel() + const consumer = new PendingRcaConsumer(logger, model) + + await consumer.markRejected('test-uuid') + + expect(model.update).toHaveBeenCalledWith( + { status: 'rejected' }, + { where: { id: 'test-uuid' } }, + ) + }) + }) +}) diff --git a/packages/indexer-common/src/indexing-fees/dips.ts b/packages/indexer-common/src/indexing-fees/dips.ts index 7b8c66c18..f7ec8bce6 100644 --- a/packages/indexer-common/src/indexing-fees/dips.ts +++ b/packages/indexer-common/src/indexing-fees/dips.ts @@ -30,8 +30,14 @@ import { GatewayDipsServiceClientImpl, } from '@graphprotocol/dips-proto/generated/gateway' import { IndexingAgreement } from '../indexer-management/models/indexing-agreement' +import { PendingRcaProposal } from '../indexer-management/models/pending-rca-proposal' +import { PendingRcaConsumer } from './pending-rca-consumer' +import { DecodedRcaProposal } from './types' +import { tryParseCustomError } from '../utils' +import { uniqueAllocationID, horizonAllocationIdProof } from '../allocations/keys' +import { encodeStartServiceData } from '@graphprotocol/toolshed' import { NetworkSpecification } from '../network-specification' -import { BaseWallet } from 'ethers' +import { BaseWallet, Signer } from 'ethers' const DIPS_COLLECTION_INTERVAL = 60_000 @@ -46,19 +52,28 @@ const normalizeAddressForDB = (address: string) => { export class DipsManager { declare gatewayDipsServiceClient: GatewayDipsServiceClientImpl declare gatewayDipsServiceMessagesCodec: GatewayDipsServiceMessagesCodec + declare pendingRcaConsumer: PendingRcaConsumer | null constructor( private logger: Logger, private models: IndexerManagementModels, private network: Network, private parent: AllocationManager | null, + pendingRcaModel?: typeof PendingRcaProposal, ) { - if (!this.network.specification.indexerOptions.dipperEndpoint) { - throw new Error('dipperEndpoint is not set') + // gRPC client — still needed for tryCancelAgreement() and DipsCollector + if (this.network.specification.indexerOptions.dipperEndpoint) { + this.gatewayDipsServiceClient = createGatewayDipsServiceClient( + this.network.specification.indexerOptions.dipperEndpoint, + ) + this.gatewayDipsServiceMessagesCodec = new GatewayDipsServiceMessagesCodec() + } + + // Pending RCA consumer — new data source for ensureAgreementRules() + if (pendingRcaModel) { + this.pendingRcaConsumer = new PendingRcaConsumer(this.logger, pendingRcaModel) + } else { + this.pendingRcaConsumer = null } - this.gatewayDipsServiceClient = createGatewayDipsServiceClient( - this.network.specification.indexerOptions.dipperEndpoint, - ) - this.gatewayDipsServiceMessagesCodec = new GatewayDipsServiceMessagesCodec() } // Cancel an agreement associated to an allocation if it exists async tryCancelAgreement(allocationId: string) { @@ -121,6 +136,393 @@ export class DipsManager { ) return } + + // Use PendingRcaConsumer if available, otherwise fall back to old IndexingAgreement model + if (this.pendingRcaConsumer) { + await this.ensureAgreementRulesFromRca() + } else { + await this.ensureAgreementRulesFromLegacy() + } + } + + private async getDipsAllocationAmount( + subgraphDeploymentId: SubgraphDeploymentID, + ): Promise<{ amount: bigint; isDenied: boolean }> { + const isDenied = await this.network.contracts.RewardsManager.isDenied( + subgraphDeploymentId.bytes32, + ) + + if (isDenied) { + return { + amount: BigInt(this.network.specification.indexerOptions.dipsAllocationAmount), + isDenied, + } + } + + // Rewarded subgraph: use rule's allocationAmount or defaultAllocationAmount + const rule = await this.models.IndexingRule.findOne({ + where: { + identifier: subgraphDeploymentId.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + }, + }) + + if (rule?.allocationAmount) { + return { amount: BigInt(rule.allocationAmount), isDenied } + } + + return { + amount: BigInt(this.network.specification.indexerOptions.defaultAllocationAmount), + isDenied, + } + } + + private async ensureAgreementRulesFromRca() { + const proposals = await this.pendingRcaConsumer!.getPendingProposals() + this.logger.debug( + `Ensuring indexing rules for ${proposals.length} pending RCA proposal${ + proposals.length === 1 ? '' : 's' + }`, + ) + + for (const proposal of proposals) { + const subgraphDeploymentID = proposal.subgraphDeploymentId + this.logger.info( + `Checking if indexing rule exists for proposal ${ + proposal.id + }, deployment ${subgraphDeploymentID.toString()}`, + ) + + const ruleExists = await this.parent!.matchingRuleExists( + this.logger, + subgraphDeploymentID, + ) + + const allDeploymentRules = await this.models.IndexingRule.findAll({ + where: { + identifierType: SubgraphIdentifierType.DEPLOYMENT, + }, + }) + const blocklistedRule = allDeploymentRules.find( + (rule) => + new SubgraphDeploymentID(rule.identifier).bytes32 === + subgraphDeploymentID.bytes32 && + rule.decisionBasis === IndexingDecisionBasis.NEVER, + ) + + if (blocklistedRule) { + this.logger.info( + `Blocklisted deployment ${subgraphDeploymentID.toString()}, rejecting proposal`, + ) + await this.pendingRcaConsumer!.markRejected(proposal.id, 'deployment blocklisted') + } else if (!ruleExists) { + this.logger.info( + `Creating indexing rule for proposal ${ + proposal.id + }, deployment ${subgraphDeploymentID.toString()}`, + ) + const { amount } = await this.getDipsAllocationAmount(subgraphDeploymentID) + const indexingRule = { + identifier: subgraphDeploymentID.ipfsHash, + allocationAmount: formatGRT(amount), + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.DIPS, + protocolNetwork: this.network.specification.networkIdentifier, + autoRenewal: true, + allocationLifetime: Math.max( + Number(proposal.minSecondsPerCollection), + Number(proposal.maxSecondsPerCollection), + ), + requireSupported: false, + } as Partial + + await upsertIndexingRule(this.logger, this.models, indexingRule) + } + } + } + + async acceptPendingProposals(activeAllocations: Allocation[]): Promise { + if (!this.pendingRcaConsumer) { + return + } + const consumer = this.pendingRcaConsumer + + const proposals = await consumer.getPendingProposals() + if (proposals.length === 0) { + return + } + + this.logger.info('Processing pending RCA proposals for on-chain acceptance', { + count: proposals.length, + }) + + for (const proposal of proposals) { + try { + await this.processProposal(consumer, proposal, activeAllocations) + } catch (error) { + this.logger.error('Unexpected error processing proposal', { + proposalId: proposal.id, + error, + }) + } + } + } + + private async processProposal( + consumer: PendingRcaConsumer, + proposal: DecodedRcaProposal, + activeAllocations: Allocation[], + ): Promise { + const now = BigInt(Math.floor(Date.now() / 1000)) + + if (proposal.deadline <= now) { + this.logger.info('Rejecting proposal: deadline expired', { + proposalId: proposal.id, + deadline: proposal.deadline.toString(), + now: now.toString(), + }) + await consumer.markRejected(proposal.id, 'deadline_expired') + await this.cleanupDipsRule(consumer, proposal) + return + } + + const allocation = activeAllocations.find( + (a) => a.subgraphDeployment.id.bytes32 === proposal.subgraphDeploymentId.bytes32, + ) + + if (allocation) { + await this.acceptWithExistingAllocation(consumer, proposal, allocation) + } else { + await this.acceptWithNewAllocation(consumer, proposal, activeAllocations) + } + } + + private async acceptWithExistingAllocation( + consumer: PendingRcaConsumer, + proposal: DecodedRcaProposal, + allocation: Allocation, + ): Promise { + this.logger.info('Accepting proposal with existing allocation', { + proposalId: proposal.id, + allocationId: allocation.id, + deployment: proposal.subgraphDeploymentId.ipfsHash, + }) + + try { + const receipt = await this.network.transactionManager.executeTransaction( + async () => + this.network.contracts.SubgraphService.acceptIndexingAgreement.estimateGas( + allocation.id, + proposal.signedRca, + ), + async (gasLimit) => + this.network.contracts.SubgraphService.acceptIndexingAgreement( + allocation.id, + proposal.signedRca, + { gasLimit }, + ), + this.logger.child({ + function: 'SubgraphService.acceptIndexingAgreement', + proposalId: proposal.id, + }), + ) + + if (receipt === 'paused' || receipt === 'unauthorized') { + this.logger.warn( + 'Skipping proposal acceptance: network is paused or unauthorized', + { proposalId: proposal.id, status: receipt }, + ) + return + } + + await consumer.markAccepted(proposal.id) + this.logger.info('Proposal accepted on-chain', { + proposalId: proposal.id, + allocationId: allocation.id, + txHash: receipt.hash, + }) + } catch (error) { + await this.handleAcceptError(consumer, proposal, error) + } + } + + private async acceptWithNewAllocation( + consumer: PendingRcaConsumer, + proposal: DecodedRcaProposal, + activeAllocations: Allocation[], + ): Promise { + this.logger.info('Accepting proposal with new allocation (multicall)', { + proposalId: proposal.id, + deployment: proposal.subgraphDeploymentId.ipfsHash, + }) + + try { + const currentEpoch = await this.network.contracts.EpochManager.currentEpoch() + + // Include both active and on-chain (closed) allocation IDs to avoid collisions + const excludeIds = activeAllocations.map((a) => a.id) + let allocationSigner: Signer | undefined + let allocationId: Address | undefined + + for (let attempt = 0; attempt < 10; attempt++) { + const result = uniqueAllocationID( + this.network.transactionManager.wallet.mnemonic!.phrase, + Number(currentEpoch), + proposal.subgraphDeploymentId, + excludeIds, + ) + + // Verify allocation doesn't already exist on-chain (e.g. closed allocations) + const onchainAllocation = + await this.network.contracts.SubgraphService.getAllocation(result.allocationId) + if (onchainAllocation.createdAt === 0n) { + allocationSigner = result.allocationSigner + allocationId = result.allocationId + break + } + + this.logger.debug( + 'Generated allocation ID already exists on-chain, trying next', + { + proposalId: proposal.id, + allocationId: result.allocationId, + attempt, + }, + ) + excludeIds.push(result.allocationId) + } + + if (!allocationSigner || !allocationId) { + this.logger.warn('Could not generate unique allocation ID after 10 attempts', { + proposalId: proposal.id, + }) + return + } + + // Generate allocation proof + const chainId = Number(this.network.specification.networkIdentifier.split(':')[1]) + const proof = await horizonAllocationIdProof( + allocationSigner, + chainId, + this.network.specification.indexerOptions.address, + allocationId, + this.network.contracts.SubgraphService.target.toString(), + ) + + // Build startService calldata + const { amount, isDenied } = await this.getDipsAllocationAmount( + proposal.subgraphDeploymentId, + ) + this.logger.info('Determined allocation amount for DIPS agreement', { + proposalId: proposal.id, + deployment: proposal.subgraphDeploymentId.ipfsHash, + amount: amount.toString(), + isDenied, + }) + const encodedStartData = encodeStartServiceData( + proposal.subgraphDeploymentId.bytes32, + amount, + allocationId, + proof, + ) + const startServiceTx = + await this.network.contracts.SubgraphService.startService.populateTransaction( + this.network.specification.indexerOptions.address, + encodedStartData, + ) + + // Build acceptIndexingAgreement calldata + const acceptTx = + await this.network.contracts.SubgraphService.acceptIndexingAgreement.populateTransaction( + allocationId, + proposal.signedRca, + ) + + // Atomic multicall + const calldata = [startServiceTx.data!, acceptTx.data!] + const receipt = await this.network.transactionManager.executeTransaction( + async () => + this.network.contracts.SubgraphService.multicall.estimateGas(calldata), + async (gasLimit) => + this.network.contracts.SubgraphService.multicall(calldata, { gasLimit }), + this.logger.child({ + function: 'SubgraphService.multicall(startService+acceptIndexingAgreement)', + proposalId: proposal.id, + }), + ) + + if (receipt === 'paused' || receipt === 'unauthorized') { + this.logger.warn( + 'Skipping proposal acceptance: network is paused or unauthorized', + { proposalId: proposal.id, status: receipt }, + ) + return + } + + await consumer.markAccepted(proposal.id) + this.logger.info('Proposal accepted on-chain with new allocation', { + proposalId: proposal.id, + allocationId, + txHash: receipt.hash, + }) + } catch (error) { + await this.handleAcceptError(consumer, proposal, error) + } + } + + private async handleAcceptError( + consumer: PendingRcaConsumer, + proposal: DecodedRcaProposal, + error: unknown, + ): Promise { + if (this.isDeterministicError(error)) { + const parsedError = tryParseCustomError(error) + this.logger.warn('Rejecting proposal: deterministic contract error', { + proposalId: proposal.id, + error: parsedError, + }) + await consumer.markRejected(proposal.id, String(parsedError)) + await this.cleanupDipsRule(consumer, proposal) + } else { + this.logger.warn('Transient error accepting proposal, will retry', { + proposalId: proposal.id, + error, + }) + } + } + + private isDeterministicError(error: unknown): boolean { + const typedError = error as { code?: string } + return typedError?.code === 'CALL_EXCEPTION' + } + + private async cleanupDipsRule( + consumer: PendingRcaConsumer, + proposal: DecodedRcaProposal, + ): Promise { + const otherProposalsForDeployment = await consumer.getPendingProposalsForDeployment( + proposal.subgraphDeploymentId.bytes32, + ) + + if (otherProposalsForDeployment.length === 0) { + const rule = await this.models.IndexingRule.findOne({ + where: { + identifier: proposal.subgraphDeploymentId.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.DIPS, + }, + }) + if (rule) { + await this.models.IndexingRule.destroy({ where: { id: rule.id } }) + this.logger.info('Removed DIPS indexing rule after rejection', { + proposalId: proposal.id, + deployment: proposal.subgraphDeploymentId.ipfsHash, + }) + } + } + } + + private async ensureAgreementRulesFromLegacy() { // Get all the indexing agreements that are not cancelled const indexingAgreements = await this.models.IndexingAgreement.findAll({ where: { @@ -144,7 +546,7 @@ export class DipsManager { }, deployment ${subgraphDeploymentID.toString()}`, ) // If there is not yet an indexingRule that deems this deployment worth allocating to, make one - const ruleExists = await this.parent.matchingRuleExists( + const ruleExists = await this.parent!.matchingRuleExists( this.logger, subgraphDeploymentID, ) diff --git a/packages/indexer-common/src/indexing-fees/index.ts b/packages/indexer-common/src/indexing-fees/index.ts index 0b71f1b8e..0f8dba604 100644 --- a/packages/indexer-common/src/indexing-fees/index.ts +++ b/packages/indexer-common/src/indexing-fees/index.ts @@ -1 +1,3 @@ export * from './dips' +export * from './types' +export * from './pending-rca-consumer' diff --git a/packages/indexer-common/src/indexing-fees/pending-rca-consumer.ts b/packages/indexer-common/src/indexing-fees/pending-rca-consumer.ts new file mode 100644 index 000000000..d14d3a984 --- /dev/null +++ b/packages/indexer-common/src/indexing-fees/pending-rca-consumer.ts @@ -0,0 +1,83 @@ +import { Logger, SubgraphDeploymentID } from '@graphprotocol/common-ts' +import { + decodeSignedRCA, + decodeAcceptIndexingAgreementMetadata, + decodeIndexingAgreementTermsV1, +} from '@graphprotocol/toolshed' +import { PendingRcaProposal } from '../indexer-management/models/pending-rca-proposal' +import { DecodedRcaProposal } from './types' + +export class PendingRcaConsumer { + constructor( + private logger: Logger, + private model: typeof PendingRcaProposal, + ) {} + + async getPendingProposals(): Promise { + const rows = await this.model.findAll({ + where: { status: 'pending' }, + }) + + const decoded: DecodedRcaProposal[] = [] + for (const row of rows) { + try { + decoded.push(this.decodeRow(row)) + } catch (error) { + this.logger.warn(`Failed to decode pending RCA proposal ${row.id}, skipping`, { + error, + }) + } + } + return decoded + } + + async getPendingProposalsForDeployment( + deploymentBytes32: string, + ): Promise { + const all = await this.getPendingProposals() + return all.filter((p) => p.subgraphDeploymentId.bytes32 === deploymentBytes32) + } + + async markAccepted(id: string): Promise { + await this.model.update({ status: 'accepted' }, { where: { id } }) + } + + async markRejected(id: string, reason?: string): Promise { + await this.model.update({ status: 'rejected' }, { where: { id } }) + if (reason) { + this.logger.info(`Rejected proposal ${id}: ${reason}`) + } + } + + private decodeRow(row: PendingRcaProposal): DecodedRcaProposal { + const signedPayload = new Uint8Array(row.signed_payload) + const signedRca = decodeSignedRCA(signedPayload) + const { rca } = signedRca + + const metadata = decodeAcceptIndexingAgreementMetadata(rca.metadata) + const terms = decodeIndexingAgreementTermsV1(metadata.terms) + + return { + id: row.id, + status: row.status, + createdAt: row.created_at, + + signedRca, + signedPayload, + payer: rca.payer, + serviceProvider: rca.serviceProvider, + dataService: rca.dataService, + deadline: rca.deadline, + endsAt: rca.endsAt, + maxInitialTokens: rca.maxInitialTokens, + maxOngoingTokensPerSecond: rca.maxOngoingTokensPerSecond, + minSecondsPerCollection: rca.minSecondsPerCollection, + maxSecondsPerCollection: rca.maxSecondsPerCollection, + nonce: rca.nonce, + + subgraphDeploymentId: new SubgraphDeploymentID(metadata.subgraphDeploymentId), + tokensPerSecond: terms.tokensPerSecond, + tokensPerEntityPerSecond: terms.tokensPerEntityPerSecond, + } + } +} diff --git a/packages/indexer-common/src/indexing-fees/types.ts b/packages/indexer-common/src/indexing-fees/types.ts new file mode 100644 index 000000000..01a9dfc8f --- /dev/null +++ b/packages/indexer-common/src/indexing-fees/types.ts @@ -0,0 +1,28 @@ +import { SubgraphDeploymentID } from '@graphprotocol/common-ts' +import { SignedRCA } from '@graphprotocol/toolshed' + +export interface DecodedRcaProposal { + // From DB row + id: string + status: string + createdAt: Date + + // Decoded from signed_payload (via toolshed) + signedRca: SignedRCA + signedPayload: Uint8Array + payer: string + serviceProvider: string + dataService: string + deadline: bigint + endsAt: bigint + maxInitialTokens: bigint + maxOngoingTokensPerSecond: bigint + minSecondsPerCollection: bigint + maxSecondsPerCollection: bigint + nonce: bigint + + // Decoded from metadata (via toolshed) + subgraphDeploymentId: SubgraphDeploymentID + tokensPerSecond: bigint + tokensPerEntityPerSecond: bigint +} diff --git a/packages/indexer-common/src/network-specification.ts b/packages/indexer-common/src/network-specification.ts index d1008b5e6..965c70ec3 100644 --- a/packages/indexer-common/src/network-specification.ts +++ b/packages/indexer-common/src/network-specification.ts @@ -74,7 +74,7 @@ export const IndexerOptions = z legacyMnemonics: z.array(z.string()).default([]), enableDips: z.boolean().default(false), dipperEndpoint: z.string().url().optional(), - dipsAllocationAmount: GRT().default(1), + dipsAllocationAmount: GRT().default(0), dipsEpochsMargin: positiveNumber().default(1), }) .strict()