Merge pull request #11 from kianenigma/kiz-v1

A lot of more features for v1

.eslintrc

@@ -10,9 +10,11 @@
 		"plugin:@typescript-eslint/recommended"
 	],
 	"rules": {
-		"indent": [
-			"error",
-			"tab"
-		]
+		"@typescript-eslint/ban-ts-comment": "warn",
+		"@typescript-eslint/no-empty-function": "warn"
+		// "indent": [
+		// 	"error",
+		// 	"tab"
+		// ]
 	}
 }

.github/workflows/build.yaml

@@ -0,0 +1,25 @@
+name: Polkadot Basic Notifications CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [14.x, 16.x, 18.x]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+    - run: yarn install
+    - run: yarn build
+    - run: yarn test

.gitignore

@@ -7,3 +7,6 @@ config-p*.json
 *.log
 *.key
 *.ignore.json
+dont-care
+batch-*
+test-data

README.md

@@ -1,10 +1,19 @@
 # Polkadot Basic Notifications 🔴 📣
 
-A *dead-simple*, yet *highly **effective*** notification system for Polkadot and its parachains (or,
+A _dead-simple_, yet \*highly **effective\*** notification system for Polkadot and its parachains (or,
 any substrate-based chain).
 
 ![Untitled(2)](https://user-images.githubusercontent.com/5588131/158027440-a819bad8-c28a-4662-9c5a-b2f850f6ee36.png)
 
+Supported reporters:
+
+- Email (with optional end to end encryption)
+- Telegram
+- Matrix
+- Filesystem
+
+Lastly, as of the recent 1.0 release, this exporter also supports **report batching**. In essence, multiple notifications are stored in a temporary storage, and are are batch-reported all at once. This is quite useful for e.g. your staking rewards, not receiving a single email
+reward, rather a batch once a day/week.
 
 ## Configuration
 
@@ -19,87 +28,120 @@ These configurations can be provided either as JSON or YAML. See [Examples](./ex
 
 ```javascript
 {
-	// your list of accounts. For chains that use ethereum based accounts (e.g. moonbeam),
-	// just use your account's public key as hex (`0xabc..`).
-	//
-	// If the list is empty, then no account filter is applied. This means that all events and
-	// transactions will match.
-	"accounts": [
-		{ "address": "<ss58_address>", "nickname": "<account_nickname>" },
-	],
-	// a list of ws-endpoint to which we start to listen. For example, Polkadot's is "wss://rpc.
-	// polkadot.io". The cool thing here is that ANY substrate-based chain will work, so you can add
-	// accounts from parachains (Acala, Statemine), solo-chains (Aleph-zero), or even ethereum-based
-	// chains like moonbeam.
-	"endpoints": [
-		"wss://rpc.polkadot.io",
-		"wss://statemine-rpc.polkadot.io",
-		"wss://acala-polkadot.api.onfinality.io/public-ws",
-		"wss://wss.api.moonbeam.network",
-		"wss://ws.azero.dev"
-	],
-	// a case-sensitive list of methods that you want to subscribe to to. A 'method' is either the
-	// name of a transaction (usually lower_snake_case) or an event name (usually lowerCamelCase).
-	// Correct values are: 'all', or { 'ignore': .. }, or { 'only': .. }. 'Ignore' implies
-	// "everything is monitored except the given".
-	"method_subscription": {
-		"only": [
-			{
-				"pallet": "balances",
-				"method": "transfer"
-			},
-			{
-				"pallet": "electionProvierMultiPhase",
-				"method": "*"
-			},
-			{
-				"pallet": "*",
-				"method": "remark"
-			}
-		]
-	},
-	// This is where you specify which reporters you want to use.
-	"reporters": {
-		// if provided, report all events to a matrix room.
-		"matrix": {
-			// the user if of some user from which you will send the message.
-			"userId": "@your-username:matrix.org",
-			// the access token of the aforementioned user.
-			"accessToken": "..",
-			// the id of the room to which you will send the message.
-			"roomId": "..",
-			// the serve in which your user exist.
-			"server": "https://matrix.org"
-		},
-
-		// if provided, report all events to a set of email addresses.
-		"email": {
-			// the address from which you send the emails. It must be owned by the `transporter.auth` credentials once authenticated with `transporter.host`.
-			"from": "from@polkadot-basic-notification.xyz",
-			// The list of addresses that get notified.
-			"to": ["from1@dot-basic-notification.xyz", "from2@dot-basic-notification.xyz"],
-			// optional: if provided, your messages will be encrypted, but the formatting might not be as good.
-			"gpgpubkey": "./pub.key",
-			// this must be exactly the same object as used in the nodemailer library. See here for // more information: https://nodemailer.com/smtp/
-			"transporter": {
-				"host": "smtp.youremail.org",
-				"port": 587,
-				"secure": false,
-				"auth": {
-					"user": "...",
-					"pass": "..."
-				}
-			}
-		},
-
-		// if provided, writes all reports to the file at the given path. The file is appended to
-		"fs": {
-			"path": "./out1.log"
-		},
-
-		// enabling this will print all reports to console as well.
-		"console": {},
-	}
+  // your list of accounts. For chains that use ethereum based accounts (e.g. moonbeam),
+  // just use your account's public key as hex (`0xabc..`).
+  //
+  // If the list is empty, then no account filter is applied. This means that all events and
+  // transactions will match.
+  "accounts": [
+    { "address": "<ss58_address>", "nickname": "<account_nickname>" },
+  ],
+  // a list of ws-endpoint to which we start to listen. For example, Polkadot's is "wss://rpc.
+  // polkadot.io". The cool thing here is that ANY substrate-based chain will work, so you can add
+  // accounts from parachains (Acala, Statemine), solo-chains (Aleph-zero), or even ethereum-based
+  // chains like moonbeam.
+  "endpoints": [
+    "wss://rpc.polkadot.io",
+    "wss://statemine-rpc.polkadot.io",
+    "wss://acala-polkadot.api.onfinality.io/public-ws",
+    "wss://wss.api.moonbeam.network",
+    "wss://ws.azero.dev"
+  ],
+  // a case-sensitive list of methods that you want to subscribe to to. A 'method' is either the
+  // name of a transaction (usually lower_snake_case) or an event name (usually lowerCamelCase).
+  // 3 types exist here: 'Only', 'ignore', or 'all'.
+  "method_subscription": {
+    // chose one of the below 3 options:
+    // Listen to all methods. Useful to track all interactions around an account.
+    { 'type': 'all' }
+    // Listen only to these methods:
+    {
+      'type': 'only'
+      "only": [
+        {
+          "pallet": "balances",
+          "method": "transfer"
+        },
+        {
+          "pallet": "electionProvierMultiPhase",
+          "method": "*"
+        },
+        {
+          "pallet": "*",
+          "method": "remark"
+        }
+      ]
+    }
+    // listen to all except these methods. Useful to reduce the noise of 'all' type.
+    {
+      'type': 'ignore'
+      "ignore": [..]
+    }
+  },
+
+  // This is where you specify which reporters you want to use.
+  "reporters": {
+    // if provided, report all events to a matrix room.
+    "matrix": {
+      // the user if of some user from which you will send the message.
+      "userId": "@your-username:matrix.org",
+      // the access token of the aforementioned user.
+      "accessToken": "..",
+      // the id of the room to which you will send the message.
+      "roomId": "..",
+      // the serve in which your user exist.
+      "server": "https://matrix.org"
+    },
+
+    // if provided, report all events to a set of email addresses.
+    "email": {
+      // the address from which you send the emails. It must be owned by the `transporter.auth`
+      //credentials once authenticated with `transporter.host`.
+      "from": "from@polkadot-basic-notification.xyz",
+      // The list of addresses that get notified.
+      "to": ["from1@dot-basic-notification.xyz", "from2@dot-basic-notification.xyz"],
+      // optional: if provided, your messages will be encrypted, but the formatting might not be as good.
+      "gpgpubkey": "./pub.key",
+      // this must be exactly the same object as used in the nodemailer library. See here for
+      // more information: https://nodemailer.com/smtp/
+      "transporter": {
+        "host": "smtp.youremail.org",
+        "port": 587,
+        "secure": false,
+        "auth": {
+          "user": "...",
+          "pass": "..."
+        }
+      }
+    },
+
+    "telegram": {
+      // the id of the chat to which you want to send the message. The bot/account of choice
+      // must be allowed to send messages to this chat.
+      // see: https://stackoverflow.com/questions/32423837/telegram-bot-how-to-get-a-group-chat-id
+      "chatId": "123455",
+      // the access token of the bot.
+      "botToken": "..."
+    }
+
+    // if provided, writes all reports to the file at the given path. The file is appended to
+    "fs": {
+      "path": "./out1.log"
+    },
+
+    // enabling this will print all reports to console as well.
+    "console": {},
+
+    // any of the reporters can be batched. For example, a batched telegram reporter that send
+    // an update once per-day looks like this:
+    "telegram": {
+      "chatId": "123455",
+      "botToken": "...",
+      // batching interval is in seconds. If `misc` is set to true, miscellaneous messages
+      // (such as the startup report) are forwarded immediately, rather than being batched.
+      "batch": { "interval": 86400, "misc": true }
+    }
+  }
 }
 
 ```
@@ -126,30 +168,3 @@ $ docker build . -t polkadot-basic-notification -f builder.Dockerfile
 $ # note how the config file must be passed as an environment variable.
 $ docker run -e CONF=config.json polkadot-basic-notification
 ```
-
-## Under The Hood
-
-The underlying workings of this program is as follows: We have a list of accounts which we want to
-monitor, stored as ss58 string representation. The script then listens to incoming blocks of any
-given chain, and does a full-text search of the account strings in the `stringified` representation
-of both the transactions in the block, and entire events that are emitted at this block.
-
-This is super simple, yet enough to detect any interaction to or from your accounts of interest.
-Some covered examples are:
-
-- Any transaction signed by your accounts is detected, successful or unsuccessful.
-- Your staking rewards are detected both via the `Rewarded` and `Deposited` events.
-- Any transfer to your account is detected, both since your account will be an argument of the
-  `transfer` transaction, and the `Deposited` event.
-
-Nonetheless, the list goes way beyond this. The only known shortcoming of this is the lack of
-support for `pallet-indices`, which is essentially an alternative, shorter way to identify accounts.
-
-Any of such events creates a `report`. Any block that contains a non-zero number of reports is
-passed to an arbitrary number of `Reporter`s for delivery. The `Reporter`s are essentially the
-transport mechanism, i.e. how you want to be notified. Current implementations are:
-
-1. Matrix, using `matrix-js-sdk`.
-2. Email, optionally supporting GPG encryption as well.
-3. File system, writing to a file.
-4. Console, only sensible for testing.

examples/config-dev-all.json

@@ -5,10 +5,10 @@
 			"nickname": "wnd-god"
 		}
 	],
-	"endpoints": [
-		"wss://westend-rpc.polkadot.io"
-	],
-	"method_subscription": "all",
+	"endpoints": ["wss://westend-rpc.polkadot.io", "wss://ws.azero.dev"],
+	"method_subscription": {
+		"type": "all"
+	},
 	"api_subscription": "head",
 	"reporters": {
 		"console": {}

examples/config-dev-balances.json

@@ -1,9 +1,8 @@
 {
 	"accounts": [],
-	"endpoints": [
-		"wss://westend-rpc.polkadot.io"
-	],
+	"endpoints": ["wss://westend-rpc.polkadot.io"],
 	"method_subscription": {
+		"type": "only",
 		"only": [
 			{
 				"pallet": "balances",
@@ -15,4 +14,4 @@
 	"reporters": {
 		"console": {}
 	}
-}
+}

examples/config-dev-batched.json

@@ -0,0 +1,16 @@
+{
+	"accounts": [
+		{
+			"address": "5FUDdxaaZfye6ogJgqHh3Usqd6WN6q8aApFH4XNjU9iDvC49",
+			"nickname": "wnd-god"
+		}
+	],
+	"endpoints": ["wss://westend-rpc.polkadot.io"],
+	"method_subscription": {
+		"type": "all"
+	},
+	"api_subscription": "head",
+	"reporters": {
+		"console": { "batch": { "interval": 10, "misc": true } }
+	}
+}

examples/config-dev-filter.json

@@ -5,10 +5,9 @@
 			"nickname": "wnd-staking-miner"
 		}
 	],
-	"endpoints": [
-		"wss://westend-rpc.polkadot.io"
-	],
+	"endpoints": ["wss://westend-rpc.polkadot.io"],
 	"method_subscription": {
+		"type": "only",
 		"only": [
 			{
 				"pallet": "balances",
@@ -28,4 +27,4 @@
 	"reporters": {
 		"console": {}
 	}
-}
+}

examples/config-dev-filter.yaml

@@ -5,6 +5,7 @@ accounts:
 endpoints:
 - wss://westend-rpc.polkadot.io
 method_subscription:
+  type: only
   only:
   - pallet: balances
     method: transfer