Allow Grain Integrations to update their own configs (#3271)

A grain integration might want to store some details within an instance,
such as a smart contract address. This update allows arbitrary updates
to a scoped object within the grain config file.

Test Plan:

Unit tests are provided at the grain integration interface. Integration
testing will be demonstrated in the test instance using the merkle
integration.

packages/grainIntegration-csv/index.js

@@ -19,6 +19,7 @@ const csvIntegration /*: any */ = (payoutDistributions, _unused_config) => {
       fileName: `Payouts-${timestamp}.csv`,
       content: csvString,
     },
+    configUpdate: {},
   };
 };
 

packages/sourcecred/src/api/bundledGrainIntegrations.js

@@ -1,13 +1,16 @@
 // @flow
 
-import type {GrainIntegrationFunction} from "../core/ledger/grainIntegration";
+import type {
+  GrainIntegration,
+  GrainIntegrationFunction,
+} from "../core/ledger/grainIntegration";
 import * as C from "../util/combo";
 
 import {csvIntegration} from "@sourcecred/grain-integration-csv";
 
-export type GrainIntegration = {|
-  name: string,
-  function: GrainIntegrationFunction,
+export type RawGrainIntegration = {|
+  type: string,
+  config?: Object,
 |};
 
 type AllowedDeclarations = {[pluginKey: string]: GrainIntegrationFunction};
@@ -27,10 +30,19 @@ export function bundledGrainIntegrations(
   return integration;
 }
 
-export const parser: C.Parser<GrainIntegration> = C.fmap(
-  C.exactly(["csv"]),
-  (integrationKey) => ({
-    name: integrationKey,
-    function: bundledGrainIntegrations(integrationKey),
-  })
+export const rawParser: C.Parser<RawGrainIntegration> = C.object(
+  {
+    type: C.exactly(["csv"]),
+  },
+  {config: C.raw}
 );
+
+function upgrade(c: RawGrainIntegration): GrainIntegration {
+  const {type, config} = c;
+  return {
+    name: type,
+    function: bundledGrainIntegrations(type),
+    config,
+  };
+}
+export const parser: C.Parser<GrainIntegration> = C.fmap(rawParser, upgrade);

packages/sourcecred/src/api/grainConfig.js

@@ -9,10 +9,8 @@ import {
   allocationConfigParser,
 } from "../core/ledger/policies";
 import {type Name, parser as nameParser} from "../core/identity/name";
-import {
-  type GrainIntegration,
-  parser as bundledGrainIntegrationParser,
-} from "./bundledGrainIntegrations";
+import type {GrainIntegration} from "../core/ledger/grainIntegration";
+import {parser as bundledGrainIntegrationParser} from "./bundledGrainIntegrations";
 
 export type RawGrainConfig = {|
   +allocationPolicies: $ReadOnlyArray<AllocationConfig>,

packages/sourcecred/src/api/instance/instance.js

@@ -56,4 +56,10 @@ export interface Instance extends ReadOnlyInstance {
   writeGrainIntegrationOutput(
     result: $Shape<GrainIntegrationMultiResult>
   ): Promise<void>;
+
+  /** Write grain config updates back into the instance */
+  updateGrainIntegrationConfig(
+    result: $Shape<GrainIntegrationMultiResult>,
+    config: GrainInput
+  ): Promise<void>;
 }

packages/sourcecred/src/api/instance/localInstance.js

@@ -4,9 +4,11 @@ import {Instance} from "./instance";
 import {ReadInstance} from "./readInstance";
 import type {CredrankOutput} from "../main/credrank";
 import type {GraphInput, GraphOutput} from "../main/graph";
+import type {GrainInput} from "../main/grain";
 import type {AnalysisOutput} from "../main/analysis";
 import type {Neo4jOutput} from "../main/analysisUtils/neo4j";
 import {join as pathJoin} from "path";
+import {raw as rawParser} from "../../util/combo";
 import stringify from "json-stable-stringify";
 import {
   type WeightedGraph,
@@ -38,6 +40,8 @@ const PERSONAL_ATTRIBUTIONS_PATH: $ReadOnlyArray<string> = [
   "config",
   "personalAttributions.json",
 ];
+
+const GRAIN_PATH: $ReadOnlyArray<string> = ["config", "grain.json"];
 const INSTANCE_CONFIG_PATH: $ReadOnlyArray<string> = ["sourcecred.json"];
 const CREDGRAPH_PATH: $ReadOnlyArray<string> = ["output", "credGraph"];
 const CREDGRAINVIEW_PATH: $ReadOnlyArray<string> = ["output", "credGrainView"];
@@ -149,6 +153,40 @@ export class LocalInstance extends ReadInstance implements Instance {
     this._writableStorage.set(grainIntegrationPath, encode(content));
   }
 
+  /**
+   *
+   */
+  async updateGrainIntegrationConfig(
+    result: $Shape<GrainIntegrationMultiResult>,
+    config: GrainInput
+  ): Promise<void> {
+    const configName = config.grainConfig.integration?.name;
+    if (!configName) return;
+    const configUpdate = result.configUpdate;
+    if (Object.keys(configUpdate).length > 0) {
+      const grainConfigPath = pathJoin(...GRAIN_PATH);
+      const currentConfig = await loadJson<any>(
+        this._storage,
+        grainConfigPath,
+        rawParser
+      );
+      const newConfig = {
+        ...currentConfig,
+        integration: {
+          ...currentConfig.integration,
+          config: {
+            ...currentConfig.integration.config,
+            ...configUpdate,
+          },
+        },
+      };
+      await this._writableStorage.set(
+        grainConfigPath,
+        stringify(newConfig, {space: 2})
+      );
+    }
+  }
+
   //////////////////////////////
   //  Private Functions
   //////////////////////////////

packages/sourcecred/src/api/main/grain.js

@@ -2,11 +2,11 @@
 
 import {CredGraph} from "../../core/credrank/credGraph";
 import {Ledger} from "../../core/ledger/ledger";
+import {applyDistributions} from "../../core/ledger/applyDistributions";
 import type {CurrencyDetails} from "../currencyConfig";
-import {type GrainConfig} from "../grainConfig";
-import {type Currency as IntegrationCurrency} from "../../core/ledger/currency";
+import type {GrainConfig} from "../grainConfig";
+import type {Currency as IntegrationCurrency} from "../../core/ledger/currency";
 import type {Distribution} from "../../core/ledger/distribution";
-import {applyDistributions} from "../../core/ledger/applyDistributions";
 import type {TimestampMs} from "../../util/timestamp";
 import {
   executeGrainIntegration,
@@ -30,6 +30,7 @@ export type GrainOutput = {|
 // since only the most recent ledger is relevant
 export type GrainIntegrationMultiResult = {|
   output?: GrainIntegrationOutput,
+  configUpdate: Object,
   distributionCredTimestamp: TimestampMs,
 |};
 
@@ -62,11 +63,11 @@ export async function grain(input: GrainInput): Promise<GrainOutput> {
  * Marshall grainInput from a Grain Configuration file for use with
  * executeGrainIntegration function
  */
-export function executeGrainIntegrationsFromGrainInput(
+export async function executeGrainIntegrationsFromGrainInput(
   grainInput: GrainInput,
   ledger: Ledger,
   distributions: $ReadOnlyArray<Distribution>
-): GrainIntegrationResults {
+): Promise<GrainIntegrationResults> {
   const integrationCurrency = grainInput.currencyDetails.integrationCurrency;
   const grainIntegration = grainInput.grainConfig.integration;
   const results = [];
@@ -76,16 +77,21 @@ export function executeGrainIntegrationsFromGrainInput(
   // supporting the case where the distributions parameter is an empty array
   let ledgerResult = ledger;
   if (integrationCurrency && grainIntegration) {
-    distributions.forEach((distribution) => {
-      const result = executeGrainIntegration(
+    for (const distribution of distributions) {
+      const result = await executeGrainIntegration(
         ledgerResult,
-        grainIntegration.function,
+        grainIntegration,
         distribution
       );
-      const {output, distributionCredTimestamp, ledger: nextLedger} = result;
+      const {
+        output,
+        distributionCredTimestamp,
+        ledger: nextLedger,
+        configUpdate,
+      } = result;
       ledgerResult = nextLedger;
-      results.push({output, distributionCredTimestamp});
-    });
+      results.push({output, distributionCredTimestamp, configUpdate});
+    }
   }
   return {ledger: ledgerResult, results};
 }

packages/sourcecred/src/cli/grain.js

@@ -47,14 +47,15 @@ const grainCommand: Command = async (args, std) => {
     grainInput
   );
 
-  const {results, ledger} = executeGrainIntegrationsFromGrainInput(
+  const {results, ledger} = await executeGrainIntegrationsFromGrainInput(
     grainInput,
     ledgerBeforeIntegrations,
     distributions
   );
 
   for (const result of results) {
-    instance.writeGrainIntegrationOutput(result);
+    await instance.writeGrainIntegrationOutput(result);
+    await instance.updateGrainIntegrationConfig(result, grainInput);
   }
 
   let totalDistributed = G.ZERO;

packages/sourcecred/src/cli/update/v0_10_0.js

@@ -13,10 +13,8 @@ import {
   allocationConfigParser,
 } from "../../core/ledger/policies";
 import {type Name, parser as nameParser} from "../../core/identity/name";
-import {
-  type GrainIntegration,
-  parser as bundledGrainIntegrationParser,
-} from "../../api/bundledGrainIntegrations";
+import type {GrainIntegration} from "../../core/ledger/grainIntegration";
+import {parser as bundledGrainIntegrationParser} from "../../api/bundledGrainIntegrations";
 import {toDiscount} from "../../core/ledger/policies/recent";
 import stringify from "json-stable-stringify";
 import {DiskStorage} from "../../core/storage/disk";

packages/sourcecred/src/core/ledger/grainIntegration.js

@@ -28,6 +28,9 @@ export type PayoutResult = {|
   // If Grain balances are tracked in the ledger, these will be recorded as
   // transfers in the ledger to the "sink" Identity.
   transferredGrain: TransferredGrain,
+  // integration-specific config changes that need to be persisted are
+  // returned in the configUpdate object.
+  configUpdate: Object,
   // output files and content
   outputFile?: GrainIntegrationOutput,
 |};
@@ -46,6 +49,10 @@ export type IntegrationConfig = {|
   // distributed funds via a configured integration
   processDistributions: boolean,
   currency: Currency,
+  // This optional object contains the self-configuration set by the grain
+  // integration on previous runs. It allows the grain integration to
+  // read its own configuration from disk.
+  integration: ?Object,
 |};
 
 /**
@@ -60,12 +67,19 @@ export type IntegrationConfig = {|
 export type GrainIntegrationFunction = (
   PayoutDistributions,
   IntegrationConfig
-) => PayoutResult;
+) => Promise<PayoutResult>;
+
+export type GrainIntegration = {|
+  name: string,
+  function: GrainIntegrationFunction,
+  config?: Object,
+|};
 
 export type GrainIntegrationResult = {|
   ledger: Ledger,
-  output?: GrainIntegrationOutput,
   distributionCredTimestamp: TimestampMs,
+  configUpdate: Object,
+  output?: GrainIntegrationOutput,
 |};
 
 ///////////////////
@@ -75,12 +89,12 @@ export type GrainIntegrationResult = {|
 // TODO @topocount: Refactor interface using instance/config properties
 // after grainConfig is updated to include The payout currency details and
 // sink identity
-export function executeGrainIntegration(
+export async function executeGrainIntegration(
   ledger: Ledger,
-  integration: GrainIntegrationFunction,
+  integration: GrainIntegration,
   distribution: Distribution,
   sink?: IdentityId
-): GrainIntegrationResult {
+): Promise<GrainIntegrationResult> {
   const currency = ledger.externalCurrency();
   const {
     enabled: accountingEnabled,
@@ -96,8 +110,9 @@ export function executeGrainIntegration(
   // the fixed-point amount for some reason.
   let result;
   try {
-    result = integration(payoutDistributions, {
+    result = await integration.function(payoutDistributions, {
       accounting: ledger.accounting(),
+      integration: integration.config,
       processDistributions,
       currency,
     });
@@ -122,6 +137,7 @@ export function executeGrainIntegration(
   return {
     ledger,
     output: result.outputFile,
+    configUpdate: result.configUpdate,
     distributionCredTimestamp: distribution.credTimestamp,
   };
 }