i/tendermint (#51)

* feat: updated kyvejs to 1.3.0 changes * feat: added new versioning system * test: finished unit tests for version validation * feat: log warnings if storage provider balance is low * test: completed unit tests for validation checks * feat: skip uploader role if storage funds are not sufficient * refactor: improved logging of warning messages * fix: fixed valname generation with new versioning sytem * refactor: improved logging * feat: added base implementation of tendermint runtime * chore: added schema validation for tendermint blocks * chore: added schema validation for tendermint block results * chore: set beta version * chore: migrated to lerna v7 * fix: gst * fix: removed additionalProperties except on root level * fix: remove ack and log in block results * fix: set ack to empty string * feat: implemented s3 storage provider * chore: upgraded to lerna v7 * chore: added dedicated standardize error message * fix: yarn setup * chore: sort attributes in events * fix: properly set event attribute order * chore: made storage priv optional depending on storage provider * chore: removed .DS_Store and updated gitignore * chore: updated README
KYVENetwork · Jul 28, 2023 · 2bae6a4 · 2bae6a4
1 parent 7a4b05d
commit 2bae6a4
Show file tree

Hide file tree

Showing 48 changed files with 3,774 additions and 1,901 deletions.
diff --git a/common/protocol/package.json b/common/protocol/package.json
@@ -19,6 +19,7 @@
     "doc": "typedoc"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.370.0",
     "@bundlr-network/client": "^0.8.9",
     "@cosmjs/proto-signing": "^0.27.1",
     "@cosmjs/stargate": "^0.27.1",
@@ -40,7 +41,8 @@
     "seedrandom": "^3.0.5",
     "semver": "^7.5.3",
     "tslog": "^3.2.2",
-    "unique-names-generator": "^4.6.0"
+    "unique-names-generator": "^4.6.0",
+    "uuid": "^9.0.0"
   },
   "devDependencies": {
     "@types/clone": "^2.1.1",
@@ -51,6 +53,7 @@
     "@types/object-hash": "^2.2.1",
     "@types/seedrandom": "^3.0.2",
     "@types/semver": "^7.3.9",
+    "@types/uuid": "^9.0.2",
     "jest": "^28.1.3",
     "prettier": "^2.4.1",
     "rimraf": "^3.0.2",

diff --git a/common/protocol/src/commander/parser.ts b/common/protocol/src/commander/parser.ts
@@ -28,16 +28,6 @@ export const parseValaccount = (value: string): string => {
   return process.env[value] || "";
 };
 
-export const parseStoragePriv = (value: string): string => {
-  if (!process.env[value]) {
-    throw new commander.InvalidArgumentError(
-      `Environment variable "${value}" has no value`
-    );
-  }
-
-  return process.env[value] || "";
-};
-
 export const parseKeyfile = (value: string): string => {
   if (!existsSync(value)) {
     throw new commander.InvalidArgumentError(

diff --git a/common/protocol/src/index.ts b/common/protocol/src/index.ts
@@ -10,7 +10,6 @@ import {
   parseEndpoints,
   parseValaccount,
   parsePoolId,
-  parseStoragePriv,
 } from "./commander";
 import {
   archiveDebugBundle,
@@ -49,7 +48,7 @@ import {
   isStorageBalanceLow,
 } from "./methods";
 import { ICacheProvider, IMetrics, IRuntime } from "./types";
-import { standardizeJSON } from "./utils";
+import { standardizeError } from "./utils";
 import { SupportedChains } from "@kyvejs/sdk/dist/constants";
 import { storageProviderFactory } from "./reactors/storageProviders";
 import { compressionFactory } from "./reactors/compression";
@@ -210,11 +209,6 @@ export class Validator {
         "The environment variable pointing to the valaccount mnemonic",
         parseValaccount
       )
-      .requiredOption(
-        "--storage-priv <string>",
-        "The environment variable pointing to the private key of the storage provider",
-        parseStoragePriv
-      )
       .requiredOption("--chain-id <string>", "The chain ID of the network")
       .requiredOption(
         "--rpc <string>",
@@ -226,6 +220,10 @@ export class Validator {
         "Comma separated list of rest endpoints. If the first fails the next endpoint will be used as fallback.",
         parseEndpoints
       )
+      .option(
+        "--storage-priv <string>",
+        "The environment variable pointing to the private key of the storage provider. Only required when using storage providers Arweave or Bundlr."
+      )
       .option(
         "--coin-denom <string>",
         "The denom of the coin, this value will be loaded by default based on the chain id"
@@ -287,7 +285,7 @@ export class Validator {
     // assign program options to node instance
     this.poolId = options.pool;
     this.valaccount = options.valaccount;
-    this.storagePriv = options.storagePriv;
+    this.storagePriv = process.env[options.storagePriv] || "";
     this.chainId = options.chainId;
     this.rpc = options.rpc;
     this.rest = options.rest;
@@ -335,7 +333,7 @@ export class Validator {
       this.runCache();
     } catch (err) {
       this.logger.fatal(`Unexpected runtime error. Exiting ...`);
-      this.logger.fatal(standardizeJSON(err));
+      this.logger.fatal(standardizeError(err));
 
       process.exit(1);
     }

diff --git a/common/protocol/src/methods/checks/isDataAvailable.ts b/common/protocol/src/methods/checks/isDataAvailable.ts
@@ -1,4 +1,4 @@
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * isDataAvailable checks if the current next data item
@@ -46,7 +46,7 @@ export async function isDataAvailable(this: Validator): Promise<boolean> {
     return true;
   } catch (err) {
     this.logger.fatal(`Data not available. Exiting ...`);
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     return false;
   }

diff --git a/common/protocol/src/methods/checks/isNodeValidator.ts b/common/protocol/src/methods/checks/isNodeValidator.ts
@@ -1,4 +1,4 @@
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * isNodeValidator checks if the staker of the node is in the
@@ -31,7 +31,7 @@ export function isNodeValidator(this: Validator): boolean {
     this.logger.fatal(
       `Error while validating if node is a validator. Exiting ...`
     );
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     return false;
   }

diff --git a/common/protocol/src/methods/checks/isStorageBalanceZero.ts b/common/protocol/src/methods/checks/isStorageBalanceZero.ts
@@ -1,5 +1,5 @@
 import BigNumber from "bignumber.js";
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * isStorageBalanceZero checks if the account of the storage provider
@@ -23,6 +23,14 @@ export async function isStorageBalanceZero(this: Validator): Promise<boolean> {
     const address = await storageProvider.getAddress();
     const balance = await storageProvider.getBalance();
 
+    // if storage provider has no balance we don't need to validate it
+    if (!balance) {
+      this.logger.info(
+        `StorageProvider:${storageProvider.name} has no balance. Continuing...\n`
+      );
+      return false;
+    }
+
     this.logger.debug(`Account "${address}" has "${balance}" balance`);
 
     if (new BigNumber(balance).isZero()) {
@@ -45,7 +53,7 @@ export async function isStorageBalanceZero(this: Validator): Promise<boolean> {
     this.logger.fatal(
       `Error while checking storage provider balance. Exiting ...`
     );
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     return true;
   }

diff --git a/common/protocol/src/methods/checks/isValidRuntime.ts b/common/protocol/src/methods/checks/isValidRuntime.ts
@@ -1,4 +1,4 @@
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * isValidRuntime checks if the runtime of the pool matches with the runtime of
@@ -28,7 +28,7 @@ export function isValidRuntime(this: Validator): boolean {
     return true;
   } catch (err) {
     this.logger.fatal(`Error while validating runtime. Exiting ...`);
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     return false;
   }

diff --git a/common/protocol/src/methods/main/runCache.ts b/common/protocol/src/methods/main/runCache.ts
@@ -1,5 +1,5 @@
 import { DataItem, Validator } from "../..";
-import { callWithBackoffStrategy, sleep, standardizeJSON } from "../../utils";
+import { callWithBackoffStrategy, sleep, standardizeError } from "../../utils";
 import clone from "clone";
 
 /**
@@ -111,7 +111,7 @@ export async function runCache(this: Validator): Promise<void> {
           this.logger.error(
             `Unexpected error deleting data item ${i.toString()} from local cache. Continuing ...`
           );
-          this.logger.error(standardizeJSON(err));
+          this.logger.error(standardizeError(err));
           continue;
         }
       }
@@ -172,7 +172,7 @@ export async function runCache(this: Validator): Promise<void> {
                   ctx.nextTimeoutInMs / 1000
                 ).toFixed(2)}s ...`
               );
-              this.logger.debug(standardizeJSON(err));
+              this.logger.debug(standardizeError(err));
 
               this.m.runtime_get_data_item_failed.inc();
             }
@@ -207,7 +207,7 @@ export async function runCache(this: Validator): Promise<void> {
       this.logger.error(
         `Unexpected error collecting data items to local cache. Continuing ...`
       );
-      this.logger.error(standardizeJSON(err));
+      this.logger.error(standardizeError(err));
 
       try {
         // drop cache if an unexpected error occurs during caching
@@ -219,7 +219,7 @@ export async function runCache(this: Validator): Promise<void> {
         this.logger.error(
           `Unexpected error dropping local cache. Continuing ...`
         );
-        this.logger.error(standardizeJSON(dropError));
+        this.logger.error(standardizeError(dropError));
       }
     }
   }

diff --git a/common/protocol/src/methods/queries/canPropose.ts b/common/protocol/src/methods/queries/canPropose.ts
@@ -1,5 +1,5 @@
 import { Validator } from "../..";
-import { callWithBackoffStrategy, sleep, standardizeJSON } from "../../utils";
+import { callWithBackoffStrategy, sleep, standardizeError } from "../../utils";
 
 const INFINITY_LOOP = true;
 
@@ -84,7 +84,7 @@ export async function canPropose(
             }
           } catch (err) {
             this.logger.error(`REST call to "${this.rest[l]}" failed`);
-            this.logger.error(standardizeJSON(err));
+            this.logger.error(standardizeError(err));
           }
         }
 
@@ -97,7 +97,7 @@ export async function canPropose(
             ctx.nextTimeoutInMs / 1000
           ).toFixed(2)}s ...`
         );
-        this.logger.debug(standardizeJSON(err));
+        this.logger.debug(standardizeError(err));
         this.m.query_can_propose_failed.inc();
       }
     );
@@ -116,7 +116,7 @@ export async function canPropose(
     }
   } catch (err) {
     this.logger.error(`Failed to call canPropose`);
-    this.logger.error(standardizeJSON(err));
+    this.logger.error(standardizeError(err));
 
     return false;
   }

diff --git a/common/protocol/src/methods/queries/canVote.ts b/common/protocol/src/methods/queries/canVote.ts
@@ -1,5 +1,5 @@
 import { Validator } from "../..";
-import { callWithBackoffStrategy, standardizeJSON } from "../../utils";
+import { callWithBackoffStrategy, standardizeError } from "../../utils";
 
 /**
  * canVote checks if the node is able to vote on the current
@@ -63,7 +63,7 @@ export async function canVote(
             });
           } catch (err) {
             this.logger.error(`REST call to "${this.rest[l]}" failed`);
-            this.logger.error(standardizeJSON(err));
+            this.logger.error(standardizeError(err));
           }
         }
 
@@ -76,7 +76,7 @@ export async function canVote(
             ctx.nextTimeoutInMs / 1000
           ).toFixed(2)}s ...`
         );
-        this.logger.debug(standardizeJSON(err));
+        this.logger.debug(standardizeError(err));
         this.m.query_can_vote_failed.inc();
       }
     );
@@ -93,7 +93,7 @@ export async function canVote(
     }
   } catch (err) {
     this.logger.error(`Failed to call canVote`);
-    this.logger.error(standardizeJSON(err));
+    this.logger.error(standardizeError(err));
 
     return false;
   }

diff --git a/common/protocol/src/methods/queries/getBalancesForMetrics.ts b/common/protocol/src/methods/queries/getBalancesForMetrics.ts
@@ -1,6 +1,6 @@
 import BigNumber from "bignumber.js";
 
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * getBalancesForMetrics tries to retrieve the $KYVE balance of the staker account, the $KYVE
@@ -36,7 +36,7 @@ export async function getBalancesForMetrics(this: Validator): Promise<void> {
       break;
     } catch (err) {
       this.logger.error(`Failed to get $KYVE balance of staker`);
-      this.logger.error(standardizeJSON(err));
+      this.logger.error(standardizeError(err));
     }
   }
 
@@ -64,7 +64,7 @@ export async function getBalancesForMetrics(this: Validator): Promise<void> {
       break;
     } catch (err) {
       this.logger.error(`Failed to get $KYVE balance of valaccount`);
-      this.logger.error(standardizeJSON(err));
+      this.logger.error(standardizeError(err));
     }
   }
 
@@ -95,6 +95,6 @@ export async function getBalancesForMetrics(this: Validator): Promise<void> {
     this.m.balance_storage_provider.set(storageProviderBalance);
   } catch (err) {
     this.logger.error(`Failed to get balance of storage provider`);
-    this.logger.error(standardizeJSON(err));
+    this.logger.error(standardizeError(err));
   }
 }
diff --git a/common/protocol/src/methods/queries/syncPoolState.ts b/common/protocol/src/methods/queries/syncPoolState.ts
@@ -1,5 +1,5 @@
 import { Validator } from "../..";
-import { callWithBackoffStrategy, standardizeJSON } from "../../utils";
+import { callWithBackoffStrategy, standardizeError } from "../../utils";
 
 /**
  * syncPoolState fetches the state of the pool the node is running on.
@@ -55,7 +55,7 @@ export async function syncPoolState(
               this.logger.fatal(
                 `Failed to sync runtime config. Either the config could not be parsed or was invalid.`
               );
-              this.logger.fatal(standardizeJSON(err));
+              this.logger.fatal(standardizeError(err));
 
               if (exitOnConfigError) process.exit(1);
             }
@@ -64,7 +64,7 @@ export async function syncPoolState(
           return;
         } catch (err) {
           this.logger.error(`REST call to "${this.rest[l]}" failed`);
-          this.logger.error(standardizeJSON(err));
+          this.logger.error(standardizeError(err));
         }
       }
 
@@ -77,7 +77,7 @@ export async function syncPoolState(
           ctx.nextTimeoutInMs / 1000
         ).toFixed(2)}s ...`
       );
-      this.logger.debug(standardizeJSON(err));
+      this.logger.debug(standardizeError(err));
 
       this.m.query_pool_failed.inc();
     }

diff --git a/common/protocol/src/methods/setups/setupCacheProvider.ts b/common/protocol/src/methods/setups/setupCacheProvider.ts
@@ -1,6 +1,6 @@
 import path from "path";
 
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 import fse from "fs-extra";
 
 /**
@@ -31,7 +31,7 @@ export async function setupCacheProvider(this: Validator): Promise<void> {
     this.logger.info(`Using cache provider: ${this.cacheProvider.name}`);
   } catch (err) {
     this.logger.fatal(`Failed to setup cache provider. Exiting ...`);
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     process.exit(1);
   }

diff --git a/common/protocol/src/methods/setups/setupLogger.ts b/common/protocol/src/methods/setups/setupLogger.ts
@@ -2,7 +2,7 @@ import { appendFileSync, existsSync, mkdirSync } from "fs";
 import path from "path";
 import { ILogObject, Logger } from "tslog";
 
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * setupLogger creates the logger instance and defines the home and file
@@ -83,7 +83,7 @@ export function setupLogger(this: Validator): void {
     this.logger.info("Starting node ...\n");
   } catch (err) {
     this.logger.fatal(`Failed to init logger. Exiting ...`);
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     process.exit(1);
   }