From 21f0a0ea3c1763249c119480e296245cb0713ffb Mon Sep 17 00:00:00 2001 From: Moreal Date: Fri, 24 Sep 2021 16:05:35 +0900 Subject: [PATCH] feat(bridge): alert error through pagerduty --- bridge/package.json | 1 + bridge/src/index.ts | 9 +++- bridge/src/integrations/index.ts | 3 ++ bridge/src/integrations/pagerduty.ts | 26 ++++++++++++ bridge/src/observers/burn-event-observer.ts | 12 +++++- bridge/src/observers/nine-chronicles.ts | 14 ++++++- .../burn-event-observer.spec.ts.snap | 15 +++++++ .../nine-chronicles.spec.ts.snap | 15 +++++++ .../observers/burn-event-observer.spec.ts | 41 ++++++++++++++++++- bridge/test/observers/nine-chronicles.spec.ts | 29 ++++++++++++- bridge/yarn.lock | 22 +++++++++- 11 files changed, 180 insertions(+), 7 deletions(-) create mode 100644 bridge/src/integrations/index.ts create mode 100644 bridge/src/integrations/pagerduty.ts diff --git a/bridge/package.json b/bridge/package.json index c08b594..26ec543 100644 --- a/bridge/package.json +++ b/bridge/package.json @@ -66,6 +66,7 @@ "web3-core-promievent": "^1.5.3" }, "dependencies": { + "@pagerduty/pdjs": "^2.2.3", "@planetarium/aws-kms-provider": "^0.3.4", "@sentry/node": "^6.4.1", "@sentry/tracing": "^6.4.1", diff --git a/bridge/src/index.ts b/bridge/src/index.ts index 4caaff9..fab16a8 100644 --- a/bridge/src/index.ts +++ b/bridge/src/index.ts @@ -23,6 +23,8 @@ import { Sqlite3ExchangeHistoryStore } from "./sqlite3-exchange-history-store"; import consoleStamp from 'console-stamp'; import { AddressBanPolicy } from "./policies/address-ban"; import { GasPriceLimitPolicy, GasPricePolicies, GasPriceTipPolicy, IGasPricePolicy } from "./policies/gas-price"; +import { Integration } from "./integrations"; +import { PagerDutyIntegration } from "./integrations/pagerduty"; consoleStamp(console); (async () => { @@ -54,6 +56,8 @@ consoleStamp(console); const MAX_GAS_PRICE_STRING: string = Configuration.get("MAX_GAS_PRICE", true, "string"); const MAX_GAS_PRICE = new Decimal(MAX_GAS_PRICE_STRING); + const PAGERDUTY_ROUTING_KEY: string = Configuration.get("PAGERDUTY_ROUTING_KEY", true, "string");; + const CONFIRMATIONS = 10; const monitorStateStore: IMonitorStateStore = await Sqlite3MonitorStateStore.open(MONITOR_STATE_STORE_PATH); @@ -62,6 +66,7 @@ consoleStamp(console); const GRAPHQL_REQUEST_RETRY = 5; const headlessGraphQLCLient = new HeadlessGraphQLClient(GRAPHQL_API_ENDPOINT, GRAPHQL_REQUEST_RETRY); + const integration: Integration = new PagerDutyIntegration(PAGERDUTY_ROUTING_KEY); const kmsProvider = new KmsProvider(KMS_PROVIDER_URL, { region: KMS_PROVIDER_REGION, keyIds: [KMS_PROVIDER_KEY_ID], @@ -126,7 +131,7 @@ consoleStamp(console); "0xa86E321048C397C0f7f23C65B1EE902AFE24644e", ]); - const ethereumBurnEventObserver = new EthereumBurnEventObserver(ncgKmsTransfer, slackWebClient, monitorStateStore, EXPLORER_ROOT_URL, ETHERSCAN_ROOT_URL); + const ethereumBurnEventObserver = new EthereumBurnEventObserver(ncgKmsTransfer, slackWebClient, monitorStateStore, EXPLORER_ROOT_URL, ETHERSCAN_ROOT_URL, integration); const ethereumBurnEventMonitor = new EthereumBurnEventMonitor(web3, wNCGToken, await monitorStateStore.load("ethereum"), CONFIRMATIONS); ethereumBurnEventMonitor.attach(ethereumBurnEventObserver); @@ -134,7 +139,7 @@ consoleStamp(console); const ncgTransferredEventObserver = new NCGTransferredEventObserver(ncgKmsTransfer, minter, slackWebClient, monitorStateStore, exchangeHistoryStore, EXPLORER_ROOT_URL, ETHERSCAN_ROOT_URL, ncgExchangeFeeRatio, { maximum: MAXIMUM_NCG, minimum: MINIMUM_NCG, - }, addressBanPolicy); + }, addressBanPolicy, integration); const nineChroniclesMonitor = new NineChroniclesTransferredEventMonitor(await monitorStateStore.load("nineChronicles"), headlessGraphQLCLient, kmsAddress); nineChroniclesMonitor.attach(ncgTransferredEventObserver); diff --git a/bridge/src/integrations/index.ts b/bridge/src/integrations/index.ts new file mode 100644 index 0000000..7fdcae2 --- /dev/null +++ b/bridge/src/integrations/index.ts @@ -0,0 +1,3 @@ +export interface Integration { + error(summary: string, error: Record): void; +} diff --git a/bridge/src/integrations/pagerduty.ts b/bridge/src/integrations/pagerduty.ts new file mode 100644 index 0000000..af6006e --- /dev/null +++ b/bridge/src/integrations/pagerduty.ts @@ -0,0 +1,26 @@ +import os from "os"; +import { event } from "@pagerduty/pdjs"; +import { Integration } from "."; + +export class PagerDutyIntegration implements Integration { + private readonly _routingKey: string; + + constructor(routingKey: string) { + this._routingKey = routingKey; + } + + error(summary: string, error: Record) { + event({ + data: { + routing_key: this._routingKey, + event_action: "trigger", + payload: { + severity: "warning", + source: os.hostname(), + summary: summary, + custom_details: error, + }, + } + }); + } +} diff --git a/bridge/src/observers/burn-event-observer.ts b/bridge/src/observers/burn-event-observer.ts index 4721354..d16813e 100644 --- a/bridge/src/observers/burn-event-observer.ts +++ b/bridge/src/observers/burn-event-observer.ts @@ -9,6 +9,7 @@ import { IMonitorStateStore } from "../interfaces/monitor-state-store"; import { UnwrappedEvent } from "../messages/unwrapped-event"; import Decimal from "decimal.js"; import { UnwrappingFailureEvent } from "../messages/unwrapping-failure-event"; +import { Integration } from "../integrations"; export class EthereumBurnEventObserver implements IObserver<{ blockHash: BlockHash, events: (EventData & TransactionLocation)[] }> { private readonly _ncgTransfer: INCGTransfer; @@ -16,13 +17,15 @@ export class EthereumBurnEventObserver implements IObserver<{ blockHash: BlockHa private readonly _monitorStateStore: IMonitorStateStore; private readonly _explorerUrl: string; private readonly _etherscanUrl: string; + private readonly _integration: Integration; - constructor(ncgTransfer: INCGTransfer, slackWebClient: SlackWebClient, monitorStateStore: IMonitorStateStore, explorerUrl: string, etherscanUrl: string) { + constructor(ncgTransfer: INCGTransfer, slackWebClient: SlackWebClient, monitorStateStore: IMonitorStateStore, explorerUrl: string, etherscanUrl: string, integration: Integration) { this._ncgTransfer = ncgTransfer; this._slackWebClient = slackWebClient; this._monitorStateStore = monitorStateStore; this._explorerUrl = explorerUrl; this._etherscanUrl = etherscanUrl; + this._integration = integration; } async notify(data: { blockHash: BlockHash; events: (EventData & TransactionLocation)[]; }): Promise { @@ -51,6 +54,13 @@ export class EthereumBurnEventObserver implements IObserver<{ blockHash: BlockHa channel: "#nine-chronicles-bridge-bot", ...new UnwrappingFailureEvent(this._etherscanUrl, sender, recipient, amountString, transactionHash, String(e)).render() }); + await this._integration.error("Unexpected error during unwrapping NCG", { + errorMessage: String(e), + sender, + recipient, + transactionHash, + amountString, + }); } } } diff --git a/bridge/src/observers/nine-chronicles.ts b/bridge/src/observers/nine-chronicles.ts index b3fd9db..6606888 100644 --- a/bridge/src/observers/nine-chronicles.ts +++ b/bridge/src/observers/nine-chronicles.ts @@ -12,6 +12,7 @@ import Decimal from "decimal.js" import { WrappingFailureEvent } from "../messages/wrapping-failure-event"; import { IExchangeHistoryStore } from "../interfaces/exchange-history-store"; import { IAddressBanPolicy } from "../policies/address-ban"; +import { Integration } from "../integrations"; // See also https://ethereum.github.io/yellowpaper/paper.pdf 4.2 The Transaction section. const ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"; @@ -40,7 +41,9 @@ export class NCGTransferredEventObserver implements IObserver<{ blockHash: Block private readonly _limitationPolicy: LimitationPolicy; private readonly _addressBanPolicy: IAddressBanPolicy; - constructor(ncgTransfer: INCGTransfer, wrappedNcgTransfer: IWrappedNCGMinter, slackWebClient: SlackWebClient, monitorStateStore: IMonitorStateStore, exchangeHistoryStore: IExchangeHistoryStore, explorerUrl: string, etherscanUrl: string, exchangeFeeRatio: Decimal, limitationPolicy: LimitationPolicy, addressBanPolicy: IAddressBanPolicy) { + private readonly _integration: Integration; + + constructor(ncgTransfer: INCGTransfer, wrappedNcgTransfer: IWrappedNCGMinter, slackWebClient: SlackWebClient, monitorStateStore: IMonitorStateStore, exchangeHistoryStore: IExchangeHistoryStore, explorerUrl: string, etherscanUrl: string, exchangeFeeRatio: Decimal, limitationPolicy: LimitationPolicy, addressBanPolicy: IAddressBanPolicy, integration: Integration) { this._ncgTransfer = ncgTransfer; this._wrappedNcgTransfer = wrappedNcgTransfer; this._slackWebClient = slackWebClient; @@ -51,6 +54,7 @@ export class NCGTransferredEventObserver implements IObserver<{ blockHash: Block this._exchangeFeeRatio = exchangeFeeRatio; this._limitationPolicy = limitationPolicy; this._addressBanPolicy = addressBanPolicy; + this._integration = integration; } async notify(data: { blockHash: BlockHash, events: (NCGTransferredEvent & TransactionLocation)[] }): Promise { @@ -155,10 +159,18 @@ export class NCGTransferredEventObserver implements IObserver<{ blockHash: Block }); } catch (e) { console.log("EERRRR", e) + // TODO: it should be replaced with `Integration` Slack implementation. await this._slackWebClient.chat.postMessage({ channel: "#nine-chronicles-bridge-bot", ...new WrappingFailureEvent(this._explorerUrl, sender, String(recipient), amountString, txId, String(e)).render() }); + await this._integration.error("Unexpected error during wrapping NCG", { + errorMessage: String(e), + sender, + recipient, + txId, + amountString, + }); } } diff --git a/bridge/test/observers/__snapshots__/burn-event-observer.spec.ts.snap b/bridge/test/observers/__snapshots__/burn-event-observer.spec.ts.snap index 2202bb7..ca6760e 100644 --- a/bridge/test/observers/__snapshots__/burn-event-observer.spec.ts.snap +++ b/bridge/test/observers/__snapshots__/burn-event-observer.spec.ts.snap @@ -1,5 +1,20 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`EthereumBurnEventObserver notify pagerduty 9c transfer error message - snapshot 1`] = ` +Array [ + Array [ + "Unexpected error during unwrapping NCG", + Object { + "amountString": "1.00", + "errorMessage": "Error: mockNcgTransfer.transfer error", + "recipient": "0x6d29f9923C86294363e59BAaA46FcBc37Ee5aE2e", + "sender": "0x2734048eC2892d111b4fbAB224400847544FC872", + "transactionHash": "TX-ID", + }, + ], +] +`; + exports[`EthereumBurnEventObserver notify slack 9c transfer error message - snapshot 1`] = ` Array [ Array [ diff --git a/bridge/test/observers/__snapshots__/nine-chronicles.spec.ts.snap b/bridge/test/observers/__snapshots__/nine-chronicles.spec.ts.snap index d120512..e178955 100644 --- a/bridge/test/observers/__snapshots__/nine-chronicles.spec.ts.snap +++ b/bridge/test/observers/__snapshots__/nine-chronicles.spec.ts.snap @@ -1,5 +1,20 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`NCGTransferredEventObserver notify pagerduty ethereum transfer error message - snapshot 1`] = ` +Array [ + Array [ + "Unexpected error during wrapping NCG", + Object { + "amountString": "100.23", + "errorMessage": "Error: mockWrappedNcgMinter.mint error", + "recipient": "0x4029bC50b4747A037d38CF2197bCD335e22Ca301", + "sender": "0x2734048eC2892d111b4fbAB224400847544FC872", + "txId": "TX-ID", + }, + ], +] +`; + exports[`NCGTransferredEventObserver notify slack ethereum transfer error message - snapshot 1`] = ` Array [ Array [ diff --git a/bridge/test/observers/burn-event-observer.spec.ts b/bridge/test/observers/burn-event-observer.spec.ts index 589db85..8dcc0ce 100644 --- a/bridge/test/observers/burn-event-observer.spec.ts +++ b/bridge/test/observers/burn-event-observer.spec.ts @@ -5,6 +5,7 @@ import { WebClient as SlackWebClient } from "@slack/web-api"; import { TxId } from "../../src/types/txid"; import { EthereumBurnEventObserver } from "../../src/observers/burn-event-observer"; import { TransactionLocation } from '../../src/types/transaction-location'; +import { Integration } from '../../src/integrations'; jest.mock("@slack/web-api", () => { return { @@ -38,7 +39,11 @@ describe(EthereumBurnEventObserver.name, () => { store: jest.fn(), }; - const observer = new EthereumBurnEventObserver(mockNcgTransfer, mockSlackWebClient, mockMonitorStateStore, "https://explorer.libplanet.io/9c-internal", "https://ropsten.etherscan.io"); + const mockIntegration: jest.Mocked = { + error: jest.fn(), + }; + + const observer = new EthereumBurnEventObserver(mockNcgTransfer, mockSlackWebClient, mockMonitorStateStore, "https://explorer.libplanet.io/9c-internal", "https://ropsten.etherscan.io", mockIntegration); describe(EthereumBurnEventObserver.prototype.notify.name, () => { it("should record the block hash even if there is no events", () => { @@ -166,5 +171,39 @@ describe(EthereumBurnEventObserver.name, () => { expect(mockSlackWebClient.chat.postMessage.mock.calls).toMatchSnapshot(); }); + + it("pagerduty 9c transfer error message - snapshot", async () => { + mockNcgTransfer.transfer.mockImplementationOnce((address, amount, memo) => { + throw new Error("mockNcgTransfer.transfer error"); + }); + + await observer.notify({ + blockHash: "BLOCK-HASH", + events: [ + { + blockHash: "BLOCK-HASH", + address: "0x4029bC50b4747A037d38CF2197bCD335e22Ca301", + logIndex: 0, + blockNumber: 0, + event: "Burn", + raw: { + data: "", + topics: [], + }, + signature: "", + transactionIndex: 0, + transactionHash: "TX-ID", + txId: "TX-ID", + returnValues: { + _sender: "0x2734048eC2892d111b4fbAB224400847544FC872", + _to: "0x6d29f9923C86294363e59BAaA46FcBc37Ee5aE2e", + amount: 1000000000000000000 + } + } + ], + }); + + expect(mockIntegration.error.mock.calls).toMatchSnapshot(); + }); }) }) diff --git a/bridge/test/observers/nine-chronicles.spec.ts b/bridge/test/observers/nine-chronicles.spec.ts index 94b0752..f76bd59 100644 --- a/bridge/test/observers/nine-chronicles.spec.ts +++ b/bridge/test/observers/nine-chronicles.spec.ts @@ -8,6 +8,7 @@ import { WebClient as SlackWebClient } from "@slack/web-api"; import { TxId } from "../../src/types/txid"; import { IExchangeHistoryStore } from "../../src/interfaces/exchange-history-store"; import { IAddressBanPolicy } from "../../src/policies/address-ban"; +import { Integration } from "../../src/integrations"; jest.mock("@slack/web-api", () => { return { @@ -62,7 +63,11 @@ describe(NCGTransferredEventObserver.name, () => { isBannedAddress: jest.fn().mockImplementation(address => address === BANNED_ADDRESS), }; - const observer = new NCGTransferredEventObserver(mockNcgTransfer, mockWrappedNcgMinter, mockSlackWebClient, mockMonitorStateStore, mockExchangeHistoryStore, "https://explorer.libplanet.io/9c-internal", "https://ropsten.etherscan.io", exchangeFeeRatio, limitationPolicy, addressBanPolicy); + const mockIntegration: jest.Mocked = { + error: jest.fn(), + }; + + const observer = new NCGTransferredEventObserver(mockNcgTransfer, mockWrappedNcgMinter, mockSlackWebClient, mockMonitorStateStore, mockExchangeHistoryStore, "https://explorer.libplanet.io/9c-internal", "https://ropsten.etherscan.io", exchangeFeeRatio, limitationPolicy, addressBanPolicy, mockIntegration); describe(NCGTransferredEventObserver.prototype.notify.name, () => { it("should record the block hash even if there is no events", () => { @@ -447,5 +452,27 @@ describe(NCGTransferredEventObserver.name, () => { expect(mockSlackWebClient.chat.postMessage.mock.calls).toMatchSnapshot(); }); + + it("pagerduty ethereum transfer error message - snapshot", async () => { + mockWrappedNcgMinter.mint.mockImplementationOnce((address, amount) => { + throw new Error("mockWrappedNcgMinter.mint error"); + }); + + await observer.notify({ + blockHash: "BLOCK-HASH", + events: [ + { + amount: "100.23", + memo: "0x4029bC50b4747A037d38CF2197bCD335e22Ca301", + blockHash: "BLOCK-HASH", + txId: "TX-ID", + recipient: "0x6d29f9923C86294363e59BAaA46FcBc37Ee5aE2e", + sender: "0x2734048eC2892d111b4fbAB224400847544FC872", + }, + ], + }); + + expect(mockIntegration.error.mock.calls).toMatchSnapshot(); + }); }) }) diff --git a/bridge/yarn.lock b/bridge/yarn.lock index 3675204..99fa160 100644 --- a/bridge/yarn.lock +++ b/bridge/yarn.lock @@ -1961,6 +1961,14 @@ strict-event-emitter-types "^2.0.0" ws "^7.0.0" +"@pagerduty/pdjs@^2.2.3": + version "2.2.3" + resolved "https://registry.yarnpkg.com/@pagerduty/pdjs/-/pdjs-2.2.3.tgz#6efe281b356cb825b71e119db1842594aea0397b" + integrity sha512-d3cwxN7DYwUNERfGYjqo+7uthWJ6ONdbU0O5x9q12/TkuXFbGVdxiT40jKcfHg4MfoAXNUfzN7h3IfhrY9FFrA== + dependencies: + browser-or-node "^1.3.0" + cross-fetch "^3.0.6" + "@planetarium/aws-kms-provider@^0.3.4": version "0.3.4" resolved "https://registry.yarnpkg.com/@planetarium/aws-kms-provider/-/aws-kms-provider-0.3.4.tgz#dbf084daa3aa459969539437dbaf0f7d8344f6f3" @@ -3003,6 +3011,11 @@ brorand@^1.0.1, brorand@^1.1.0: resolved "https://registry.yarnpkg.com/brorand/-/brorand-1.1.0.tgz#12c25efe40a45e3c323eb8675a0a0ce57b22371f" integrity sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8= +browser-or-node@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/browser-or-node/-/browser-or-node-1.3.0.tgz#f2a4e8568f60263050a6714b2cc236bb976647a7" + integrity sha512-0F2z/VSnLbmEeBcUrSuDH5l0HxTXdQQzLjkmBR4cYfvg1zJrKSlmIZFqyFR8oX0NrwPhy3c3HQ6i3OxMbew4Tg== + browser-process-hrtime@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626" @@ -3529,6 +3542,13 @@ cross-fetch@^2.1.0, cross-fetch@^2.1.1: node-fetch "2.1.2" whatwg-fetch "2.0.4" +cross-fetch@^3.0.6: + version "3.1.4" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.4.tgz#9723f3a3a247bf8b89039f3a380a9244e8fa2f39" + integrity sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ== + dependencies: + node-fetch "2.6.1" + cross-spawn@^6.0.5: version "6.0.5" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" @@ -6690,7 +6710,7 @@ node-fetch@2.1.2: resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.1.2.tgz#ab884e8e7e57e38a944753cec706f788d1768bb5" integrity sha1-q4hOjn5X44qUR1POxwb3iNF2i7U= -node-fetch@^2.6.0, node-fetch@^2.6.1: +node-fetch@2.6.1, node-fetch@^2.6.0, node-fetch@^2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==