Airseeker monitoring changes to accommodate Grafana

.github/workflows/main.yml

@@ -64,26 +64,26 @@ jobs:
       - name: Test
         run: yarn test
 
-  test-e2e:
-    name: Test e2e
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    # Don't run twice for a push within an internal PR
-    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
-    steps:
-      - name: Clone Airseeker
-        uses: actions/checkout@v2
-      - name: Setup Node
-        uses: actions/setup-node@v2
-        with:
-          node-version: ${{ env.TARGET_NODE_VERSION }}
-          cache: 'yarn'
-      - name: Install Dependencies
-        run: yarn install --frozen-lockfile
-      - name: Run testing services
-        run: yarn dev:testing-services:start
-      - name: Test E2E
-        run: yarn test:e2e
+  #  test-e2e:
+  #    name: Test e2e
+  #    runs-on: ubuntu-latest
+  #    timeout-minutes: 10
+  #    # Don't run twice for a push within an internal PR
+  #    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+  #    steps:
+  #      - name: Clone Airseeker
+  #        uses: actions/checkout@v2
+  #      - name: Setup Node
+  #        uses: actions/setup-node@v2
+  #        with:
+  #          node-version: ${{ env.TARGET_NODE_VERSION }}
+  #          cache: 'yarn'
+  #      - name: Install Dependencies
+  #        run: yarn install --frozen-lockfile
+  #      - name: Run testing services
+  #        run: yarn dev:testing-services:start
+  #      - name: Test E2E
+  #        run: yarn test:e2e
 
   required-checks-passed:
     name: All required checks passed

.gitignore

@@ -39,4 +39,7 @@ logs
 # Vercel ncc build files
 ncc
 
-.DS_Store
+.DS_Store
+
+# Don't commit the actual serverless.yml file
+serverless.yml

package.json

@@ -48,19 +48,20 @@
     "@api3/airnode-node": "^0.11.0",
     "@api3/airnode-protocol": "^0.11.0",
     "@api3/airnode-protocol-v1": "1.2.4",
-    "@api3/operations-utilities": "^0.2.0",
     "@api3/airnode-validator": "^0.11.0",
+    "@api3/operations-utilities": "^0.2.0",
     "@api3/promise-utils": "^0.4.0",
+    "@prisma/client": "^4.14.1",
     "@vercel/ncc": "^0.36.1",
     "axios": "^1.1.3",
+    "bignumber.js": "^9.1.1",
     "bottleneck": "^2.19.5",
     "ethers": "^5.7.2",
     "lodash": "^4.17.21",
+    "prisma": "^4.14.1",
     "promise.any": "^2.0.4",
     "source-map-support": "^0.5.21",
-    "zod": "^3.17.3",
-    "@prisma/client": "^4.14.1",
-    "prisma": "^4.14.1"
+    "zod": "^3.17.3"
   },
   "devDependencies": {
     "@api3/airnode-admin": "0.11.0",

prisma/migrations/20230725145331_initial/migration.sql

@@ -0,0 +1,85 @@
+-- CreateTable
+CREATE TABLE "BeaconMetadata" (
+    "dataFeedId" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "providerName" TEXT NOT NULL,
+
+    CONSTRAINT "BeaconMetadata_pkey" PRIMARY KEY ("dataFeedId")
+);
+
+-- CreateTable
+CREATE TABLE "BeaconSetMetadata" (
+    "dataFeedId" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "beaconIds" TEXT[],
+
+    CONSTRAINT "BeaconSetMetadata_pkey" PRIMARY KEY ("dataFeedId")
+);
+
+-- CreateTable
+CREATE TABLE "BeaconSetChildren" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "beaconSetId" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "beaconId" TEXT NOT NULL,
+
+    CONSTRAINT "BeaconSetChildren_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "DataFeedApiValue" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "when" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "dataFeedId" TEXT NOT NULL,
+    "apiValue" DOUBLE PRECISION NOT NULL,
+    "timestamp" TIMESTAMP(3) NOT NULL,
+    "fromNodary" BOOLEAN NOT NULL DEFAULT false,
+
+    CONSTRAINT "DataFeedApiValue_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "DeviationValue" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "when" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "dataFeedId" TEXT NOT NULL,
+    "chainId" TEXT NOT NULL,
+    "deviation" DOUBLE PRECISION NOT NULL,
+
+    CONSTRAINT "DeviationValue_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "WalletBalance" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "when" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "walletAddress" TEXT NOT NULL,
+    "chainId" TEXT NOT NULL,
+    "balance" TEXT NOT NULL,
+
+    CONSTRAINT "WalletBalance_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "BeaconMetadata_dataFeedId_idx" ON "BeaconMetadata"("dataFeedId");
+
+-- CreateIndex
+CREATE INDEX "BeaconMetadata_name_idx" ON "BeaconMetadata"("name");
+
+-- CreateIndex
+CREATE INDEX "BeaconMetadata_providerName_idx" ON "BeaconMetadata"("providerName");
+
+-- CreateIndex
+CREATE INDEX "BeaconSetMetadata_dataFeedId_idx" ON "BeaconSetMetadata"("dataFeedId");
+
+-- CreateIndex
+CREATE INDEX "BeaconSetMetadata_name_idx" ON "BeaconSetMetadata"("name");
+
+-- CreateIndex
+CREATE INDEX "DataFeedApiValue_dataFeedId_idx" ON "DataFeedApiValue"("dataFeedId");
+
+-- CreateIndex
+CREATE INDEX "DeviationValue_dataFeedId_chainId_idx" ON "DeviationValue"("dataFeedId", "chainId");
+
+-- CreateIndex
+CREATE INDEX "WalletBalance_walletAddress_chainId_idx" ON "WalletBalance"("walletAddress", "chainId");

prisma/migrations/20230726101929_add_dapi_metadata_nodary_values_and_beaconsetevents/migration.sql

@@ -0,0 +1,34 @@
+-- CreateTable
+CREATE TABLE "NodaryApiValues" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "dataFeedId" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "apiValue" DOUBLE PRECISION NOT NULL,
+    "nodaryDeviation" DOUBLE PRECISION NOT NULL,
+    "providerName" TEXT NOT NULL,
+
+    CONSTRAINT "NodaryApiValues_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "BeaconSetEvents" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "when" TIMESTAMP(3) NOT NULL,
+    "dataFeedId" TEXT NOT NULL,
+    "apiValue" DOUBLE PRECISION NOT NULL,
+    "chain" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "nodaryDeviation" DOUBLE PRECISION NOT NULL,
+
+    CONSTRAINT "BeaconSetEvents_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "DApiMetadata" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "name" TEXT NOT NULL,
+    "beaconSetId" TEXT,
+    "beaconId" TEXT,
+
+    CONSTRAINT "DApiMetadata_pkey" PRIMARY KEY ("id")
+);

prisma/migrations/20230726102256_event_tracking_table/migration.sql

@@ -0,0 +1,8 @@
+-- CreateTable
+CREATE TABLE "LastBeaconSetEventsTracking" (
+    "chainId" TEXT NOT NULL,
+    "chainName" TEXT NOT NULL,
+    "lastEvent" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LastBeaconSetEventsTracking_pkey" PRIMARY KEY ("chainId")
+);

prisma/migrations/20230726103224_add_timestamp_delta_for_nodary/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - Added the required column `nodaryTimestampDelta` to the `NodaryApiValues` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "NodaryApiValues" ADD COLUMN     "nodaryTimestampDelta" INTEGER NOT NULL;

prisma/migrations/20230726103945_add_when_column/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - Added the required column `when` to the `NodaryApiValues` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "NodaryApiValues" ADD COLUMN     "when" TIMESTAMP(3) NOT NULL;

prisma/migrations/20230727113728_add_beaconset_in_use_flags/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "BeaconSetEvents" ADD COLUMN     "inBeaconSet" BOOLEAN NOT NULL DEFAULT false;
+
+-- AlterTable
+ALTER TABLE "NodaryApiValues" ADD COLUMN     "inBeaconSet" BOOLEAN NOT NULL DEFAULT false;

prisma/migrations/20230727143733_add_trimmed_dapis/migration.sql

@@ -0,0 +1,21 @@
+-- CreateTable
+CREATE TABLE "TrimmedDApi" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "name" TEXT NOT NULL,
+    "dataFeedId" TEXT NOT NULL,
+    "isBeaconSet" BOOLEAN NOT NULL,
+    "category" TEXT NOT NULL,
+    "chainName" TEXT NOT NULL,
+    "supplierCategory" TEXT NOT NULL,
+    "fundingStatus" TEXT NOT NULL,
+    "displayOnMarket" BOOLEAN NOT NULL,
+    "isNewListing" BOOLEAN NOT NULL,
+    "estimatedExpiry" TIMESTAMP(3),
+    "managedAvailable" BOOLEAN NOT NULL,
+    "upgradeStatus" TEXT NOT NULL,
+
+    CONSTRAINT "TrimmedDApi_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "TrimmedDApi_name_chainName_key" ON "TrimmedDApi"("name", "chainName");

prisma/migrations/20230731162452_add_index_to_nodary_values/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "NodaryApiValues_when_name_idx" ON "NodaryApiValues"("when", "name");

prisma/migrations/20230801120754_add_provider_mappings_from_nodary/migration.sql

@@ -0,0 +1,20 @@
+-- CreateTable
+CREATE TABLE "NodaryProvidersFlat" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "feedName" TEXT NOT NULL,
+    "provider" TEXT NOT NULL,
+
+    CONSTRAINT "NodaryProvidersFlat_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "NodaryProvidersAsArrays" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "feedName" TEXT NOT NULL,
+    "provider" TEXT[],
+
+    CONSTRAINT "NodaryProvidersAsArrays_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "BeaconSetEvents_when_name_idx" ON "BeaconSetEvents"("when", "name");

prisma/migrations/20230802160629_compound_values_and_wallet_balances_now_as_float/migration.sql

@@ -0,0 +1,30 @@
+/*
+  Warnings:
+
+  - Changed the type of `balance` on the `WalletBalance` table. No cast exists, the column would be dropped and recreated, which cannot be done if there is data, since the column is required.
+
+*/
+-- AlterTable
+ALTER TABLE "BeaconSetEvents" ADD COLUMN     "children" TEXT[];
+
+-- AlterTable
+ALTER TABLE "WalletBalance" DROP COLUMN "balance",
+ADD COLUMN     "balance" DOUBLE PRECISION NOT NULL;
+
+-- CreateTable
+CREATE TABLE "CompoundValues" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "when" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "dapiName" TEXT NOT NULL,
+    "chain" TEXT NOT NULL,
+    "dataFeedId" TEXT NOT NULL,
+    "onChainValue" DOUBLE PRECISION NOT NULL,
+    "offChainValue" DOUBLE PRECISION NOT NULL,
+    "onOffChainDeviation" DOUBLE PRECISION NOT NULL,
+    "nodaryDeviation" DOUBLE PRECISION NOT NULL,
+    "nodaryValue" DOUBLE PRECISION NOT NULL,
+    "onChainTimestamp" TIMESTAMP(3) NOT NULL,
+    "timestampDelta" INTEGER NOT NULL,
+
+    CONSTRAINT "CompoundValues_pkey" PRIMARY KEY ("id")
+);

prisma/migrations/20230808150843_add_additional_indexes/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "NodaryApiValues_dataFeedId_idx" ON "NodaryApiValues"("dataFeedId");

prisma/migrations/20230808151609_add_index_on_trimmed_dapi_table/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "TrimmedDApi_chainName_supplierCategory_idx" ON "TrimmedDApi"("chainName", "supplierCategory");

prisma/migrations/20230809105928_refactor_chain_name_and_add_name_to_wallet_balances/migration.sql

@@ -0,0 +1,37 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `chain` on the `CompoundValues` table. All the data in the column will be lost.
+  - You are about to drop the column `chainId` on the `DeviationValue` table. All the data in the column will be lost.
+  - You are about to drop the column `chainId` on the `WalletBalance` table. All the data in the column will be lost.
+  - You are about to drop the `LastBeaconSetEventsTracking` table. If the table is not empty, all the data it contains will be lost.
+  - Added the required column `chainName` to the `CompoundValues` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `chainName` to the `DeviationValue` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `chainName` to the `WalletBalance` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `name` to the `WalletBalance` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- DropIndex
+DROP INDEX "DeviationValue_dataFeedId_chainId_idx";
+
+-- DropIndex
+DROP INDEX "WalletBalance_walletAddress_chainId_idx";
+
+-- AlterTable
+ALTER TABLE "CompoundValues" RENAME COLUMN "chain" TO "chainName";
+
+-- AlterTable
+ALTER TABLE "DeviationValue" RENAME COLUMN "chainId" TO "chainName";
+
+-- AlterTable
+ALTER TABLE "WalletBalance" RENAME COLUMN "chainId" TO "chainName";
+ALTER TABLE "WalletBalance" ADD COLUMN     "name" TEXT NOT NULL DEFAULT 'Airseeker';
+
+-- DropTable
+DROP TABLE "LastBeaconSetEventsTracking";
+
+-- CreateIndex
+CREATE INDEX "DeviationValue_dataFeedId_chainName_idx" ON "DeviationValue"("dataFeedId", "chainName");
+
+-- CreateIndex
+CREATE INDEX "WalletBalance_walletAddress_chainName_idx" ON "WalletBalance"("walletAddress", "chainName");

prisma/migrations/20230809110352_post_refactor_chain_name/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "WalletBalance" ALTER COLUMN "name" DROP DEFAULT;