Merge pull request #98 from moreal/feature/bridge/alert-pagerduty

feat(bridge): alert error through pagerduty

bridge/package.json

@@ -66,6 +66,7 @@
     "web3-core-promievent": "^1.5.3"
   },
   "dependencies": {
+    "@pagerduty/pdjs": "^2.2.3",
     "@planetarium/aws-kms-provider": "^0.3.4",
     "@sentry/node": "^6.4.1",
     "@sentry/tracing": "^6.4.1",

bridge/src/index.ts

@@ -23,6 +23,8 @@ import { Sqlite3ExchangeHistoryStore } from "./sqlite3-exchange-history-store";
 import consoleStamp from 'console-stamp';
 import { AddressBanPolicy } from "./policies/address-ban";
 import { GasPriceLimitPolicy, GasPricePolicies, GasPriceTipPolicy, IGasPricePolicy } from "./policies/gas-price";
+import { Integration } from "./integrations";
+import { PagerDutyIntegration } from "./integrations/pagerduty";
 
 consoleStamp(console);
 
@@ -61,6 +63,8 @@ process.on("uncaughtException", console.error);
     const MAX_GAS_PRICE_STRING: string = Configuration.get("MAX_GAS_PRICE", true, "string");
     const MAX_GAS_PRICE = new Decimal(MAX_GAS_PRICE_STRING);
 
+    const PAGERDUTY_ROUTING_KEY: string = Configuration.get("PAGERDUTY_ROUTING_KEY", true, "string");;
+
     const CONFIRMATIONS = 10;
 
     const monitorStateStore: IMonitorStateStore = await Sqlite3MonitorStateStore.open(MONITOR_STATE_STORE_PATH);
@@ -69,6 +73,7 @@ process.on("uncaughtException", console.error);
 
     const GRAPHQL_REQUEST_RETRY = 5;
     const headlessGraphQLCLient = new HeadlessGraphQLClient(GRAPHQL_API_ENDPOINT, GRAPHQL_REQUEST_RETRY);
+    const integration: Integration = new PagerDutyIntegration(PAGERDUTY_ROUTING_KEY);
     const kmsProvider = new KmsProvider(KMS_PROVIDER_URL, {
       region: KMS_PROVIDER_REGION,
       keyIds: [KMS_PROVIDER_KEY_ID],
@@ -133,15 +138,15 @@ process.on("uncaughtException", console.error);
         "0xa86E321048C397C0f7f23C65B1EE902AFE24644e",
     ]);
 
-    const ethereumBurnEventObserver = new EthereumBurnEventObserver(ncgKmsTransfer, slackWebClient, monitorStateStore, EXPLORER_ROOT_URL, ETHERSCAN_ROOT_URL);
+    const ethereumBurnEventObserver = new EthereumBurnEventObserver(ncgKmsTransfer, slackWebClient, monitorStateStore, EXPLORER_ROOT_URL, ETHERSCAN_ROOT_URL, integration);
     const ethereumBurnEventMonitor = new EthereumBurnEventMonitor(web3, wNCGToken, await monitorStateStore.load("ethereum"), CONFIRMATIONS);
     ethereumBurnEventMonitor.attach(ethereumBurnEventObserver);
 
     const ncgExchangeFeeRatio = new Decimal(0.01);  // 1%
     const ncgTransferredEventObserver = new NCGTransferredEventObserver(ncgKmsTransfer, minter, slackWebClient, monitorStateStore, exchangeHistoryStore, EXPLORER_ROOT_URL, ETHERSCAN_ROOT_URL, ncgExchangeFeeRatio, {
         maximum: MAXIMUM_NCG,
         minimum: MINIMUM_NCG,
-    }, addressBanPolicy);
+    }, addressBanPolicy, integration);
     const nineChroniclesMonitor = new NineChroniclesTransferredEventMonitor(await monitorStateStore.load("nineChronicles"), headlessGraphQLCLient, kmsAddress);
     nineChroniclesMonitor.attach(ncgTransferredEventObserver);
 

bridge/src/integrations/index.ts

@@ -0,0 +1,3 @@
+export interface Integration {
+    error(summary: string, error: Record<string, any>): void;
+}

bridge/src/integrations/pagerduty.ts

@@ -0,0 +1,26 @@
+import os from "os";
+import { event } from "@pagerduty/pdjs";
+import { Integration } from ".";
+
+export class PagerDutyIntegration implements Integration {
+    private readonly _routingKey: string;
+
+    constructor(routingKey: string) {
+        this._routingKey = routingKey;
+    }
+
+    error(summary: string, error: Record<string, any>) {
+        event({
+            data: {
+                routing_key: this._routingKey,
+                event_action: "trigger",
+                payload: {
+                    severity: "warning",
+                    source: os.hostname(),
+                    summary: summary,
+                    custom_details: error,
+                },
+            }
+        });
+    }
+}

bridge/src/observers/burn-event-observer.ts

@@ -9,20 +9,23 @@ import { IMonitorStateStore } from "../interfaces/monitor-state-store";
 import { UnwrappedEvent } from "../messages/unwrapped-event";
 import Decimal from "decimal.js";
 import { UnwrappingFailureEvent } from "../messages/unwrapping-failure-event";
+import { Integration } from "../integrations";
 
 export class EthereumBurnEventObserver implements IObserver<{ blockHash: BlockHash, events: (EventData & TransactionLocation)[] }> {
     private readonly _ncgTransfer: INCGTransfer;
     private readonly _slackWebClient: SlackWebClient;
     private readonly _monitorStateStore: IMonitorStateStore;
     private readonly _explorerUrl: string;
     private readonly _etherscanUrl: string;
+    private readonly _integration: Integration;
 
-    constructor(ncgTransfer: INCGTransfer, slackWebClient: SlackWebClient, monitorStateStore: IMonitorStateStore, explorerUrl: string, etherscanUrl: string) {
+    constructor(ncgTransfer: INCGTransfer, slackWebClient: SlackWebClient, monitorStateStore: IMonitorStateStore, explorerUrl: string, etherscanUrl: string, integration: Integration) {
         this._ncgTransfer = ncgTransfer;
         this._slackWebClient = slackWebClient;
         this._monitorStateStore = monitorStateStore;
         this._explorerUrl = explorerUrl;
         this._etherscanUrl = etherscanUrl;
+        this._integration = integration;
     }
 
     async notify(data: { blockHash: BlockHash; events: (EventData & TransactionLocation)[]; }): Promise<void> {
@@ -51,6 +54,13 @@ export class EthereumBurnEventObserver implements IObserver<{ blockHash: BlockHa
                     channel: "#nine-chronicles-bridge-bot",
                     ...new UnwrappingFailureEvent(this._etherscanUrl, sender, recipient, amountString, transactionHash, String(e)).render()
                 });
+                await this._integration.error("Unexpected error during unwrapping NCG", {
+                    errorMessage: String(e),
+                    sender,
+                    recipient,
+                    transactionHash,
+                    amountString,
+                });
             }
         }
     }

bridge/src/observers/nine-chronicles.ts

@@ -12,6 +12,7 @@ import Decimal from "decimal.js"
 import { WrappingFailureEvent } from "../messages/wrapping-failure-event";
 import { IExchangeHistoryStore } from "../interfaces/exchange-history-store";
 import { IAddressBanPolicy } from "../policies/address-ban";
+import { Integration } from "../integrations";
 
 // See also https://ethereum.github.io/yellowpaper/paper.pdf 4.2 The Transaction section.
 const ZERO_ADDRESS = "0x0000000000000000000000000000000000000000";
@@ -40,7 +41,9 @@ export class NCGTransferredEventObserver implements IObserver<{ blockHash: Block
     private readonly _limitationPolicy: LimitationPolicy;
     private readonly _addressBanPolicy: IAddressBanPolicy;
 
-    constructor(ncgTransfer: INCGTransfer, wrappedNcgTransfer: IWrappedNCGMinter, slackWebClient: SlackWebClient, monitorStateStore: IMonitorStateStore, exchangeHistoryStore: IExchangeHistoryStore, explorerUrl: string, etherscanUrl: string, exchangeFeeRatio: Decimal, limitationPolicy: LimitationPolicy, addressBanPolicy: IAddressBanPolicy) {
+    private readonly _integration: Integration;
+
+    constructor(ncgTransfer: INCGTransfer, wrappedNcgTransfer: IWrappedNCGMinter, slackWebClient: SlackWebClient, monitorStateStore: IMonitorStateStore, exchangeHistoryStore: IExchangeHistoryStore, explorerUrl: string, etherscanUrl: string, exchangeFeeRatio: Decimal, limitationPolicy: LimitationPolicy, addressBanPolicy: IAddressBanPolicy, integration: Integration) {
         this._ncgTransfer = ncgTransfer;
         this._wrappedNcgTransfer = wrappedNcgTransfer;
         this._slackWebClient = slackWebClient;
@@ -51,6 +54,7 @@ export class NCGTransferredEventObserver implements IObserver<{ blockHash: Block
         this._exchangeFeeRatio = exchangeFeeRatio;
         this._limitationPolicy = limitationPolicy;
         this._addressBanPolicy = addressBanPolicy;
+        this._integration = integration;
     }
 
     async notify(data: { blockHash: BlockHash, events: (NCGTransferredEvent & TransactionLocation)[] }): Promise<void> {
@@ -155,10 +159,18 @@ export class NCGTransferredEventObserver implements IObserver<{ blockHash: Block
                 });
             } catch (e) {
                 console.log("EERRRR", e)
+                // TODO: it should be replaced with `Integration` Slack implementation.
                 await this._slackWebClient.chat.postMessage({
                     channel: "#nine-chronicles-bridge-bot",
                     ...new WrappingFailureEvent(this._explorerUrl, sender, String(recipient), amountString, txId, String(e)).render()
                 });
+                await this._integration.error("Unexpected error during wrapping NCG", {
+                    errorMessage: String(e),
+                    sender,
+                    recipient,
+                    txId,
+                    amountString,
+                });
             }
         }
 

bridge/test/observers/__snapshots__/burn-event-observer.spec.ts.snap

@@ -1,5 +1,20 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`EthereumBurnEventObserver notify pagerduty 9c transfer error message - snapshot 1`] = `
+Array [
+  Array [
+    "Unexpected error during unwrapping NCG",
+    Object {
+      "amountString": "1.00",
+      "errorMessage": "Error: mockNcgTransfer.transfer error",
+      "recipient": "0x6d29f9923C86294363e59BAaA46FcBc37Ee5aE2e",
+      "sender": "0x2734048eC2892d111b4fbAB224400847544FC872",
+      "transactionHash": "TX-ID",
+    },
+  ],
+]
+`;
+
 exports[`EthereumBurnEventObserver notify slack 9c transfer error message - snapshot 1`] = `
 Array [
   Array [

bridge/test/observers/__snapshots__/nine-chronicles.spec.ts.snap

@@ -1,5 +1,20 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`NCGTransferredEventObserver notify pagerduty ethereum transfer error message - snapshot 1`] = `
+Array [
+  Array [
+    "Unexpected error during wrapping NCG",
+    Object {
+      "amountString": "100.23",
+      "errorMessage": "Error: mockWrappedNcgMinter.mint error",
+      "recipient": "0x4029bC50b4747A037d38CF2197bCD335e22Ca301",
+      "sender": "0x2734048eC2892d111b4fbAB224400847544FC872",
+      "txId": "TX-ID",
+    },
+  ],
+]
+`;
+
 exports[`NCGTransferredEventObserver notify slack ethereum transfer error message - snapshot 1`] = `
 Array [
   Array [

bridge/test/observers/burn-event-observer.spec.ts

@@ -5,6 +5,7 @@ import { WebClient as SlackWebClient } from "@slack/web-api";
 import { TxId } from "../../src/types/txid";
 import { EthereumBurnEventObserver } from "../../src/observers/burn-event-observer";
 import { TransactionLocation } from '../../src/types/transaction-location';
+import { Integration } from '../../src/integrations';
 
 jest.mock("@slack/web-api", () => {
     return {
@@ -38,7 +39,11 @@ describe(EthereumBurnEventObserver.name, () => {
         store: jest.fn(),
     };
 
-    const observer = new EthereumBurnEventObserver(mockNcgTransfer, mockSlackWebClient, mockMonitorStateStore, "https://explorer.libplanet.io/9c-internal", "https://ropsten.etherscan.io");
+    const mockIntegration: jest.Mocked<Integration> = {
+        error: jest.fn(),
+    };
+
+    const observer = new EthereumBurnEventObserver(mockNcgTransfer, mockSlackWebClient, mockMonitorStateStore, "https://explorer.libplanet.io/9c-internal", "https://ropsten.etherscan.io", mockIntegration);
 
     describe(EthereumBurnEventObserver.prototype.notify.name, () => {
         it("should record the block hash even if there is no events", () => {
@@ -166,5 +171,39 @@ describe(EthereumBurnEventObserver.name, () => {
 
             expect(mockSlackWebClient.chat.postMessage.mock.calls).toMatchSnapshot();
         });
+
+        it("pagerduty 9c transfer error message - snapshot", async () => {
+            mockNcgTransfer.transfer.mockImplementationOnce((address, amount, memo) => {
+                throw new Error("mockNcgTransfer.transfer error");
+            });
+
+            await observer.notify({
+                blockHash: "BLOCK-HASH",
+                events: [
+                    {
+                        blockHash: "BLOCK-HASH",
+                        address: "0x4029bC50b4747A037d38CF2197bCD335e22Ca301",
+                        logIndex: 0,
+                        blockNumber: 0,
+                        event: "Burn",
+                        raw: {
+                            data: "",
+                            topics: [],
+                        },
+                        signature: "",
+                        transactionIndex: 0,
+                        transactionHash: "TX-ID",
+                        txId: "TX-ID",
+                        returnValues: {
+                            _sender: "0x2734048eC2892d111b4fbAB224400847544FC872",
+                            _to: "0x6d29f9923C86294363e59BAaA46FcBc37Ee5aE2e",
+                            amount: 1000000000000000000
+                        }
+                    }
+                ],
+            });
+
+            expect(mockIntegration.error.mock.calls).toMatchSnapshot();
+        });
     })
 })

bridge/test/observers/nine-chronicles.spec.ts

@@ -8,6 +8,7 @@ import { WebClient as SlackWebClient } from "@slack/web-api";
 import { TxId } from "../../src/types/txid";
 import { IExchangeHistoryStore } from "../../src/interfaces/exchange-history-store";
 import { IAddressBanPolicy } from "../../src/policies/address-ban";
+import { Integration } from "../../src/integrations";
 
 jest.mock("@slack/web-api", () => {
     return {
@@ -62,7 +63,11 @@ describe(NCGTransferredEventObserver.name, () => {
         isBannedAddress: jest.fn().mockImplementation(address => address === BANNED_ADDRESS),
     };
 
-    const observer = new NCGTransferredEventObserver(mockNcgTransfer, mockWrappedNcgMinter, mockSlackWebClient, mockMonitorStateStore, mockExchangeHistoryStore, "https://explorer.libplanet.io/9c-internal", "https://ropsten.etherscan.io", exchangeFeeRatio, limitationPolicy, addressBanPolicy);
+    const mockIntegration: jest.Mocked<Integration> = {
+        error: jest.fn(),
+    };
+
+    const observer = new NCGTransferredEventObserver(mockNcgTransfer, mockWrappedNcgMinter, mockSlackWebClient, mockMonitorStateStore, mockExchangeHistoryStore, "https://explorer.libplanet.io/9c-internal", "https://ropsten.etherscan.io", exchangeFeeRatio, limitationPolicy, addressBanPolicy, mockIntegration);
 
     describe(NCGTransferredEventObserver.prototype.notify.name, () => {
         it("should record the block hash even if there is no events", () => {
@@ -447,5 +452,27 @@ describe(NCGTransferredEventObserver.name, () => {
 
             expect(mockSlackWebClient.chat.postMessage.mock.calls).toMatchSnapshot();
         });
+
+        it("pagerduty ethereum transfer error message - snapshot", async () => {
+            mockWrappedNcgMinter.mint.mockImplementationOnce((address, amount) => {
+                throw new Error("mockWrappedNcgMinter.mint error");
+            });
+
+            await observer.notify({
+                blockHash: "BLOCK-HASH",
+                events: [
+                    {
+                        amount: "100.23",
+                        memo: "0x4029bC50b4747A037d38CF2197bCD335e22Ca301",
+                        blockHash: "BLOCK-HASH",
+                        txId: "TX-ID",
+                        recipient: "0x6d29f9923C86294363e59BAaA46FcBc37Ee5aE2e",
+                        sender: "0x2734048eC2892d111b4fbAB224400847544FC872",
+                    },
+                ],
+            });
+
+            expect(mockIntegration.error.mock.calls).toMatchSnapshot();
+        });
     })
 })

bridge/yarn.lock

@@ -1961,6 +1961,14 @@
     strict-event-emitter-types "^2.0.0"
     ws "^7.0.0"
 
+"@pagerduty/pdjs@^2.2.3":
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/@pagerduty/pdjs/-/pdjs-2.2.3.tgz#6efe281b356cb825b71e119db1842594aea0397b"
+  integrity sha512-d3cwxN7DYwUNERfGYjqo+7uthWJ6ONdbU0O5x9q12/TkuXFbGVdxiT40jKcfHg4MfoAXNUfzN7h3IfhrY9FFrA==
+  dependencies:
+    browser-or-node "^1.3.0"
+    cross-fetch "^3.0.6"
+
 "@planetarium/aws-kms-provider@^0.3.4":
   version "0.3.4"
   resolved "https://registry.yarnpkg.com/@planetarium/aws-kms-provider/-/aws-kms-provider-0.3.4.tgz#dbf084daa3aa459969539437dbaf0f7d8344f6f3"
@@ -3003,6 +3011,11 @@ brorand@^1.0.1, brorand@^1.1.0:
   resolved "https://registry.yarnpkg.com/brorand/-/brorand-1.1.0.tgz#12c25efe40a45e3c323eb8675a0a0ce57b22371f"
   integrity sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=
 
+browser-or-node@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/browser-or-node/-/browser-or-node-1.3.0.tgz#f2a4e8568f60263050a6714b2cc236bb976647a7"
+  integrity sha512-0F2z/VSnLbmEeBcUrSuDH5l0HxTXdQQzLjkmBR4cYfvg1zJrKSlmIZFqyFR8oX0NrwPhy3c3HQ6i3OxMbew4Tg==
+
 browser-process-hrtime@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626"
@@ -3529,6 +3542,13 @@ cross-fetch@^2.1.0, cross-fetch@^2.1.1:
     node-fetch "2.1.2"
     whatwg-fetch "2.0.4"
 
+cross-fetch@^3.0.6:
+  version "3.1.4"
+  resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.4.tgz#9723f3a3a247bf8b89039f3a380a9244e8fa2f39"
+  integrity sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ==
+  dependencies:
+    node-fetch "2.6.1"
+
 cross-spawn@^6.0.5:
   version "6.0.5"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
@@ -6690,7 +6710,7 @@ node-fetch@2.1.2:
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.1.2.tgz#ab884e8e7e57e38a944753cec706f788d1768bb5"
   integrity sha1-q4hOjn5X44qUR1POxwb3iNF2i7U=
 
-node-fetch@^2.6.0, node-fetch@^2.6.1:
+node-fetch@2.6.1, node-fetch@^2.6.0, node-fetch@^2.6.1:
   version "2.6.1"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
   integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==