makeomatic · pajgo · Oct 26, 2020 · Sep 25, 2020
diff --git a/package.json b/package.json
@@ -32,6 +32,7 @@
     "@hapi/hapi": "^19.1.1",
     "@hapi/vision": "^6.0.0",
     "@microfleet/core": "^17.3.3",
+    "@microfleet/plugin-consul": "^2.2.12",
     "@microfleet/transport-amqp": "^15.2.1",
     "@microfleet/validation": "^9.0.1",
     "bluebird": "^3.7.2",
@@ -44,7 +45,7 @@
     "flake-idgen": "^1.1.0",
     "get-stdin": "^8.0.0",
     "get-value": "^3.0.1",
-    "got": "^11.3.0",
+    "got": "^11.7.0",
     "handlebars": "^4.7.3",
     "ioredis": "^4.17.3",
     "is": "^3.3.0",
@@ -58,6 +59,7 @@
     "ms-mailer-templates": "^1.16.0",
     "ms-token": "^4.1.0",
     "otplib": "^12.0.1",
+    "p-retry": "^4.2.0",
     "password-generator": "^2.3.2",
     "prom-client": "^12.0.0",
     "qs": "^6.9.1",
@@ -103,6 +105,7 @@
     "json": "^9.0.6",
     "md5": "^2.2.1",
     "mocha": "^8.0.1",
+    "nock": "^13.0.4",
     "nyc": "^15.1.0",
     "puppeteer": "4.0.0",
     "rimraf": "^3.0.2",

diff --git a/rfcs/cloudflare-access-list.md b/rfcs/cloudflare-access-list.md
@@ -0,0 +1,93 @@
+# Cloudflare Access List
+
+Provides an ability to manage Cloudflare IP list contents and controls TTL of the records.
+
+## Why
+
+In some cases, some groups of users should be able to access the service under any circumstances. So we should configure the Cloudflare firewall and provide rules that will allow us to skip some checks for some IP addresses.
+
+Eg.: `If IP is in $some_list passthrough RateLimit checks`.
+
+We can create a bunch of rules like: `IP.src in {127.0.0.1 127.0.1.1/24}`, but it's hard to manage, and there is an undocumented `4kb` length limit, so we won't be able to add not so many rules.
+
+That's why we should use the `IP List` option.[https://blog.cloudflare.com/introducing-ip-lists/]
+
+## Known limitations
+
+  * We can have only a limited amount of the Firewall rules.
+  * Only Firewall rules have `passthrough` action. IP Access Rules do not provide such an ability.
+  * Each Ip List should contain a maximum of 1000 records, so this module provides the ability to use multiple lists.
+  * Maximum list count depends on Cloudflare subscription.
+  * Maximum firewall rule count depends on Cloudflare subscription.
+  * Lists should already exist.
+  * Maximum amount of API requests is 1200 in 5 minutes.
+
+## General
+
+* `CloudflareClient` - Auth configuration and additional response processing for Cloudflare API requests.
+
+### CloudflareIpList
+
+Handles IP and Ip List records registry and list synchronization.
+
+On `addIp`:
+
+1. Tries to get the list entries from Cloudflare and stores them in the cache.
+2. Finds first free Ip List.
+3. Adds IP address to the selected Ip List.
+4. Adds record into Redis Hash that stores mapping between IP address and assigned list.
+5. Increases cached list counters.
+
+On `touchIp`:
+
+1. Calls API like `addIp` and changes `updated_on` property of the record in IpList.
+
+### CloudflareWorker
+
+Only one instance should execute All synchronization tasks, so `@microfleet/plugin-consul` used.
+
+Executes AccessList cleanup procedures every `n` seconds:
+
+* Deletes outdated records from Ip Lists.
+* Synchronizes remote IP Lists with local Redis IP -> IpList mappings.
+
+### `cf.add-to-access-list` Action
+```javascript
+amqp.publishAndWait('cf.add-to-access-list', {
+  remoteip: '10.1.1.1',
+  comment: 'optional comment',
+})
+```
+
+Action performs checks whether `remoteip` already in some lists. If IP does not exists calls `CloudflareIpList.addIP()` otherwise `CloudflareIpList.touchIp()`.
+
+### Configuration
+
+Please refer [schemas/config.json#/definitions/cfAccessList](../schemas/config.json) for information.
+
+```javascript
+module.exports = {
+  cfAccessList: {
+    enabled: true,
+    auth: {
+      serviceKey: 'valid-service-key'
+    },
+    accessList: {
+      ttl: thirtyDays,
+      listCacheTTL: fifteenMinutes,
+    },
+    worker: {
+      enabled: true,
+      concurrency: 5,
+      cleanupInterval: halfAnHour,
+    },
+    api: {
+      retry: {
+        retries: 20,
+        factor: 0.5,
+      },
+    },
+  },
+};
+
+```
diff --git a/schemas/cf.add-to-access-list.json b/schemas/cf.add-to-access-list.json
@@ -0,0 +1,14 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "remoteip": {
+      "type": "string",
+      "format": "ipv4"
+    },
+    "comment": {
+      "type": "string",
+      "maxLength": 256
+    }
+  }
+}
diff --git a/schemas/config.json b/schemas/config.json
@@ -635,6 +635,9 @@
           "type": "boolean"
         }
       }
+    },
+    "cfAccessList": {
+      "$ref": "#/definitions/cfAccessList"
     }
   },
   "definitions": {
@@ -699,6 +702,103 @@
           "$ref": "#/definitions/slidingWindowRateLimiter"
         }
       }
+    },
+    "cfAccessList": {
+      "type": "object",
+      "description": "Cloudflare account and access list configuration",
+      "required": [ "enabled" ],
+      "if": {"properties": { "enabled": { "const": true } } },
+      "then": {
+        "required": [ "auth", "accessList", "worker" ],
+        "properties": {
+          "enabled": { "type": "boolean" },
+          "auth": {
+            "description": "API access configuration [https://api.cloudflare.com/#getting-started-requests]",
+            "anyOf": [
+              {
+                "type": "object",
+                "additionalProperties": false,
+                "required": [ "token" ],
+                "properties": {
+                  "token": { "type": "string", "minLength": 10, "maxLength": 128 }
+                }
+              },
+              {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["serviceKey"],
+                "properties": {
+                  "serviceKey": { "type": "string", "minLength": 10, "maxLength": 128 }
+                }
+              },
+              {
+                "type": "object",
+                "additionalProperties": false,
+                "required": [ "email", "key" ],
+                "properties": {
+                  "email": { "type": "string", "format": "email" },
+                  "key": { "type": "string", "minLength": 10, "maxLength": 128 }
+                }
+              }
+            ]
+          },
+          "api": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "retry": {
+                "type": "object",
+                "description": "Bulk operation wait configuration [https://www.npmjs.com/package/p-retry#options]"
+              }
+            }
+          },
+          "accessList": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [ "accountId", "ttl", "listCacheTTL" ],
+            "properties": {
+              "prefix": {
+                "type": "string",
+                "description": "Use only lists that start with a prefix"
+              },
+              "accountId": {
+                "type": "string",
+                "description": "Cloudflare account id with managed lists",
+                "minLength": 10,
+                "maxLength": 128
+              },
+              "ttl": {
+                "type": "number",
+                "description": "Milliseconds to hold the IP address in access list",
+                "minimum": 60000
+              },
+              "listCacheTTL": {
+                "type": "number",
+                "description": "Milliseconds to wait before dropping cached lists information",
+                "minimum": 15000
+              }
+            }
+          },
+          "worker": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [ "cleanupInterval", "concurrency", "enabled" ],
+            "properties": {
+              "enabled": { "type": "boolean" },
+              "cleanupInterval": {
+                "type": "number",
+                "description": "Milliseconds to wait between lists cleanup",
+                "minimum": 2000
+              },
+              "concurrency": {
+                "type": "number",
+                "description": "The number of lists to process in parallel",
+                "minimum": 1
+              }
+            }
+          }
+        }
+      }
     }
   }
 }
diff --git a/src/actions/cf/add-to-access-list.js b/src/actions/cf/add-to-access-list.js
@@ -0,0 +1,14 @@
+const { ActionTransport } = require('@microfleet/core');
+
+async function addToAccessList({ params }) {
+  const { cfAccessList } = this;
+  const { remoteip: ip, comment } = params;
+  const listId = await cfAccessList.findRuleListId(ip);
+  const ipEntry = { ip, comment };
+
+  return listId ? cfAccessList.touchIP(ipEntry, listId) : cfAccessList.addIP(ipEntry);
+}
+
+addToAccessList.transports = [ActionTransport.amqp, ActionTransport.internal];
+
+module.exports = addToAccessList;
diff --git a/src/actions/login.js b/src/actions/login.js
@@ -217,7 +217,11 @@ async function login({ params, locals }) {
   isBanned(internalData);
 
   // retrieves complete information and returns it
-  return getUserInfo(ctx, internalData);
+  const userInfo = await getUserInfo(ctx, internalData);
+
+  await this.hook('users:login', userInfo, ctx);
+
+  return userInfo;
 }
 
 login.mfa = MFA_TYPE_OPTIONAL;

diff --git a/src/configs/cloudflare-list.js b/src/configs/cloudflare-list.js
@@ -0,0 +1,24 @@
+const thirtyDays = 30 * 24 * 60 * 60 * 1000;
+const fifteenMinutes = 15 * 60 * 1000;
+const halfAnHour = 30 * 60 * 1000;
+
+module.exports = {
+  cfAccessList: {
+    enabled: false,
+    accessList: {
+      ttl: thirtyDays,
+      listCacheTTL: fifteenMinutes,
+    },
+    worker: {
+      enabled: true,
+      concurrency: 5,
+      cleanupInterval: halfAnHour,
+    },
+    api: {
+      retry: {
+        retries: 20,
+        factor: 0.5,
+      },
+    },
+  },
+};
diff --git a/src/configs/consul.js b/src/configs/consul.js
@@ -0,0 +1,3 @@
+module.exports = {
+  consul: {},
+};
diff --git a/src/custom/cappasity-cf-access-list.js b/src/custom/cappasity-cf-access-list.js
@@ -0,0 +1,16 @@
+/**
+ * Checks whether user has paid plan and adds to Cloudflare list
+ * @param  {Object} metadata
+ * @return {Promise}
+ */
+module.exports = async function addIpToAccessList({ user }, ctx) {
+  const { amqp, config: { router: { routes }, cfAccessList: { enabled } } } = this;
+  if (!enabled) return;
+
+  const { remoteip, audience } = ctx;
+  const route = `${routes.prefix}.cf.add-to-access-list`;
+
+  if (remoteip !== false && user.metadata[audience].plan !== 'free') {
+    await amqp.publish(route, { remoteip }, { confirm: true, mandatory: true });
+  }
+};
diff --git a/src/users.js b/src/users.js
@@ -10,6 +10,7 @@ const Flakeless = require('ms-flakeless');
 const conf = require('./config');
 const get = require('./utils/get-value');
 const attachPasswordKeyword = require('./utils/password-validator');
+const { CloudflareWorker } = require('./utils/cloudflare/worker');
 
 /**
  * @namespace Users
@@ -140,6 +141,23 @@ class Users extends Microfleet {
       }, 'tbits');
     }
 
+    if (this.config.cfAccessList.enabled) {
+      if (!this.hasPlugin('consul')) {
+        const consul = require('@microfleet/plugin-consul');
+        this.initPlugin(consul, this.config.consul);
+      }
+
+      this.addConnector(ConnectorsTypes.application, () => {
+        this.cfWorker = new CloudflareWorker(this, this.config.cfAccessList);
+        this.cfAccessList = this.cfWorker.cfList;
+        this.cfWorker.start();
+      });
+
+      this.addDestructor(ConnectorsTypes.application, () => {
+        this.cfWorker.stop();
+      });
+    }
+
     // init account seed
     this.addConnector(ConnectorsTypes.application, () => (
       this.initAdminAccounts()