Feat/s3 storage provider

common/protocol/package.json

@@ -19,6 +19,7 @@
     "doc": "typedoc"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.370.0",
     "@bundlr-network/client": "^0.8.9",
     "@cosmjs/proto-signing": "^0.27.1",
     "@cosmjs/stargate": "^0.27.1",
@@ -40,7 +41,8 @@
     "seedrandom": "^3.0.5",
     "semver": "^7.5.3",
     "tslog": "^3.2.2",
-    "unique-names-generator": "^4.6.0"
+    "unique-names-generator": "^4.6.0",
+    "uuid": "^9.0.0"
   },
   "devDependencies": {
     "@types/clone": "^2.1.1",
@@ -51,6 +53,7 @@
     "@types/object-hash": "^2.2.1",
     "@types/seedrandom": "^3.0.2",
     "@types/semver": "^7.3.9",
+    "@types/uuid": "^9.0.2",
     "jest": "^28.1.3",
     "prettier": "^2.4.1",
     "rimraf": "^3.0.2",

common/protocol/src/commander/parser.ts

@@ -29,12 +29,6 @@ export const parseValaccount = (value: string): string => {
 };
 
 export const parseStoragePriv = (value: string): string => {
-  if (!process.env[value]) {
-    throw new commander.InvalidArgumentError(
-      `Environment variable "${value}" has no value`
-    );
-  }
-
   return process.env[value] || "";
 };
 

common/protocol/src/index.ts

@@ -49,7 +49,7 @@ import {
   isStorageBalanceLow,
 } from "./methods";
 import { ICacheProvider, IMetrics, IRuntime } from "./types";
-import { standardizeJSON } from "./utils";
+import { standardizeError } from "./utils";
 import { SupportedChains } from "@kyvejs/sdk/dist/constants";
 import { storageProviderFactory } from "./reactors/storageProviders";
 import { compressionFactory } from "./reactors/compression";
@@ -335,7 +335,7 @@ export class Validator {
       this.runCache();
     } catch (err) {
       this.logger.fatal(`Unexpected runtime error. Exiting ...`);
-      this.logger.fatal(standardizeJSON(err));
+      this.logger.fatal(standardizeError(err));
 
       process.exit(1);
     }

common/protocol/src/methods/checks/isDataAvailable.ts

@@ -1,4 +1,4 @@
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * isDataAvailable checks if the current next data item
@@ -46,7 +46,7 @@ export async function isDataAvailable(this: Validator): Promise<boolean> {
     return true;
   } catch (err) {
     this.logger.fatal(`Data not available. Exiting ...`);
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     return false;
   }

common/protocol/src/methods/checks/isNodeValidator.ts

@@ -1,4 +1,4 @@
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * isNodeValidator checks if the staker of the node is in the
@@ -31,7 +31,7 @@ export function isNodeValidator(this: Validator): boolean {
     this.logger.fatal(
       `Error while validating if node is a validator. Exiting ...`
     );
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     return false;
   }

common/protocol/src/methods/checks/isStorageBalanceZero.ts

@@ -1,5 +1,5 @@
 import BigNumber from "bignumber.js";
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * isStorageBalanceZero checks if the account of the storage provider
@@ -23,6 +23,14 @@ export async function isStorageBalanceZero(this: Validator): Promise<boolean> {
     const address = await storageProvider.getAddress();
     const balance = await storageProvider.getBalance();
 
+    // if storage provider has no balance we don't need to validate it
+    if (!balance) {
+      this.logger.info(
+        `StorageProvider:${storageProvider.name} has no balance. Continuing...\n`
+      );
+      return false;
+    }
+
     this.logger.debug(`Account "${address}" has "${balance}" balance`);
 
     if (new BigNumber(balance).isZero()) {
@@ -45,7 +53,7 @@ export async function isStorageBalanceZero(this: Validator): Promise<boolean> {
     this.logger.fatal(
       `Error while checking storage provider balance. Exiting ...`
     );
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     return true;
   }

common/protocol/src/methods/checks/isValidRuntime.ts

@@ -1,4 +1,4 @@
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * isValidRuntime checks if the runtime of the pool matches with the runtime of
@@ -28,7 +28,7 @@ export function isValidRuntime(this: Validator): boolean {
     return true;
   } catch (err) {
     this.logger.fatal(`Error while validating runtime. Exiting ...`);
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     return false;
   }

common/protocol/src/methods/main/runCache.ts

@@ -1,5 +1,5 @@
 import { DataItem, Validator } from "../..";
-import { callWithBackoffStrategy, sleep, standardizeJSON } from "../../utils";
+import { callWithBackoffStrategy, sleep, standardizeError } from "../../utils";
 import clone from "clone";
 
 /**
@@ -111,7 +111,7 @@ export async function runCache(this: Validator): Promise<void> {
           this.logger.error(
             `Unexpected error deleting data item ${i.toString()} from local cache. Continuing ...`
           );
-          this.logger.error(standardizeJSON(err));
+          this.logger.error(standardizeError(err));
           continue;
         }
       }
@@ -172,7 +172,7 @@ export async function runCache(this: Validator): Promise<void> {
                   ctx.nextTimeoutInMs / 1000
                 ).toFixed(2)}s ...`
               );
-              this.logger.debug(standardizeJSON(err));
+              this.logger.debug(standardizeError(err));
 
               this.m.runtime_get_data_item_failed.inc();
             }
@@ -207,7 +207,7 @@ export async function runCache(this: Validator): Promise<void> {
       this.logger.error(
         `Unexpected error collecting data items to local cache. Continuing ...`
       );
-      this.logger.error(standardizeJSON(err));
+      this.logger.error(standardizeError(err));
 
       try {
         // drop cache if an unexpected error occurs during caching
@@ -219,7 +219,7 @@ export async function runCache(this: Validator): Promise<void> {
         this.logger.error(
           `Unexpected error dropping local cache. Continuing ...`
         );
-        this.logger.error(standardizeJSON(dropError));
+        this.logger.error(standardizeError(dropError));
       }
     }
   }

common/protocol/src/methods/queries/canPropose.ts

@@ -1,5 +1,5 @@
 import { Validator } from "../..";
-import { callWithBackoffStrategy, sleep, standardizeJSON } from "../../utils";
+import { callWithBackoffStrategy, sleep, standardizeError } from "../../utils";
 
 const INFINITY_LOOP = true;
 
@@ -84,7 +84,7 @@ export async function canPropose(
             }
           } catch (err) {
             this.logger.error(`REST call to "${this.rest[l]}" failed`);
-            this.logger.error(standardizeJSON(err));
+            this.logger.error(standardizeError(err));
           }
         }
 
@@ -97,7 +97,7 @@ export async function canPropose(
             ctx.nextTimeoutInMs / 1000
           ).toFixed(2)}s ...`
         );
-        this.logger.debug(standardizeJSON(err));
+        this.logger.debug(standardizeError(err));
         this.m.query_can_propose_failed.inc();
       }
     );
@@ -116,7 +116,7 @@ export async function canPropose(
     }
   } catch (err) {
     this.logger.error(`Failed to call canPropose`);
-    this.logger.error(standardizeJSON(err));
+    this.logger.error(standardizeError(err));
 
     return false;
   }

common/protocol/src/methods/queries/canVote.ts

@@ -1,5 +1,5 @@
 import { Validator } from "../..";
-import { callWithBackoffStrategy, standardizeJSON } from "../../utils";
+import { callWithBackoffStrategy, standardizeError } from "../../utils";
 
 /**
  * canVote checks if the node is able to vote on the current
@@ -63,7 +63,7 @@ export async function canVote(
             });
           } catch (err) {
             this.logger.error(`REST call to "${this.rest[l]}" failed`);
-            this.logger.error(standardizeJSON(err));
+            this.logger.error(standardizeError(err));
           }
         }
 
@@ -76,7 +76,7 @@ export async function canVote(
             ctx.nextTimeoutInMs / 1000
           ).toFixed(2)}s ...`
         );
-        this.logger.debug(standardizeJSON(err));
+        this.logger.debug(standardizeError(err));
         this.m.query_can_vote_failed.inc();
       }
     );
@@ -93,7 +93,7 @@ export async function canVote(
     }
   } catch (err) {
     this.logger.error(`Failed to call canVote`);
-    this.logger.error(standardizeJSON(err));
+    this.logger.error(standardizeError(err));
 
     return false;
   }

common/protocol/src/methods/queries/getBalancesForMetrics.ts

@@ -1,6 +1,6 @@
 import BigNumber from "bignumber.js";
 
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * getBalancesForMetrics tries to retrieve the $KYVE balance of the staker account, the $KYVE
@@ -36,7 +36,7 @@ export async function getBalancesForMetrics(this: Validator): Promise<void> {
       break;
     } catch (err) {
       this.logger.error(`Failed to get $KYVE balance of staker`);
-      this.logger.error(standardizeJSON(err));
+      this.logger.error(standardizeError(err));
     }
   }
 
@@ -64,7 +64,7 @@ export async function getBalancesForMetrics(this: Validator): Promise<void> {
       break;
     } catch (err) {
       this.logger.error(`Failed to get $KYVE balance of valaccount`);
-      this.logger.error(standardizeJSON(err));
+      this.logger.error(standardizeError(err));
     }
   }
 
@@ -95,6 +95,6 @@ export async function getBalancesForMetrics(this: Validator): Promise<void> {
     this.m.balance_storage_provider.set(storageProviderBalance);
   } catch (err) {
     this.logger.error(`Failed to get balance of storage provider`);
-    this.logger.error(standardizeJSON(err));
+    this.logger.error(standardizeError(err));
   }
 }

common/protocol/src/methods/queries/syncPoolState.ts

@@ -1,5 +1,5 @@
 import { Validator } from "../..";
-import { callWithBackoffStrategy, standardizeJSON } from "../../utils";
+import { callWithBackoffStrategy, standardizeError } from "../../utils";
 
 /**
  * syncPoolState fetches the state of the pool the node is running on.
@@ -55,7 +55,7 @@ export async function syncPoolState(
               this.logger.fatal(
                 `Failed to sync runtime config. Either the config could not be parsed or was invalid.`
               );
-              this.logger.fatal(standardizeJSON(err));
+              this.logger.fatal(standardizeError(err));
 
               if (exitOnConfigError) process.exit(1);
             }
@@ -64,7 +64,7 @@ export async function syncPoolState(
           return;
         } catch (err) {
           this.logger.error(`REST call to "${this.rest[l]}" failed`);
-          this.logger.error(standardizeJSON(err));
+          this.logger.error(standardizeError(err));
         }
       }
 
@@ -77,7 +77,7 @@ export async function syncPoolState(
           ctx.nextTimeoutInMs / 1000
         ).toFixed(2)}s ...`
       );
-      this.logger.debug(standardizeJSON(err));
+      this.logger.debug(standardizeError(err));
 
       this.m.query_pool_failed.inc();
     }

common/protocol/src/methods/setups/setupCacheProvider.ts

@@ -1,6 +1,6 @@
 import path from "path";
 
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 import fse from "fs-extra";
 
 /**
@@ -31,7 +31,7 @@ export async function setupCacheProvider(this: Validator): Promise<void> {
     this.logger.info(`Using cache provider: ${this.cacheProvider.name}`);
   } catch (err) {
     this.logger.fatal(`Failed to setup cache provider. Exiting ...`);
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     process.exit(1);
   }

common/protocol/src/methods/setups/setupLogger.ts

@@ -2,7 +2,7 @@ import { appendFileSync, existsSync, mkdirSync } from "fs";
 import path from "path";
 import { ILogObject, Logger } from "tslog";
 
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * setupLogger creates the logger instance and defines the home and file
@@ -83,7 +83,7 @@ export function setupLogger(this: Validator): void {
     this.logger.info("Starting node ...\n");
   } catch (err) {
     this.logger.fatal(`Failed to init logger. Exiting ...`);
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     process.exit(1);
   }

common/protocol/src/methods/setups/setupMetrics.ts

@@ -2,7 +2,7 @@ import http from "http";
 import prom_client, { register } from "prom-client";
 import url from "url";
 
-import { IMetrics, Validator, standardizeJSON } from "../..";
+import { IMetrics, Validator, standardizeError } from "../..";
 
 /**
  * setupMetrics initiates all metrics and starts if enabled a local prometheus
@@ -413,7 +413,7 @@ export function setupMetrics(this: Validator): void {
     }
   } catch (err) {
     this.logger.fatal(`Failed to setup metrics. Exiting ...`);
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     process.exit(1);
   }

common/protocol/src/methods/setups/setupSDK.ts

@@ -1,5 +1,5 @@
 import KyveSDK from "@kyvejs/sdk";
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * setupSDK creates the main KYVE SDK and the client which is used for transactions
@@ -41,7 +41,7 @@ export async function setupSDK(this: Validator): Promise<void> {
     this.lcd = this.sdk.map((sdk) => sdk.createLCDClient());
   } catch (err) {
     this.logger.fatal(`Failed to init KYVE SDK. Exiting ...`);
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     process.exit(1);
   }

common/protocol/src/methods/setups/setupValidator.ts

@@ -7,7 +7,7 @@ import {
 } from "unique-names-generator";
 import { major, minor } from "semver";
 
-import { Validator, standardizeJSON } from "../..";
+import { Validator, standardizeError } from "../..";
 
 /**
  * setupValidator ensures the node starts as a valid validator
@@ -62,7 +62,7 @@ export async function setupValidator(this: Validator): Promise<void> {
     this.m.cache_current_items.set(0);
   } catch (err) {
     this.logger.fatal(`Failed to setup validator. Exiting ...`);
-    this.logger.fatal(standardizeJSON(err));
+    this.logger.fatal(standardizeError(err));
 
     process.exit(1);
   }