Merge pull request #1346 from firewalla/beta_7_0

1.961 - Beta Patch

alarm/PolicyManager2.js

@@ -93,6 +93,16 @@ class PolicyManager2 {
     }
     return instance;
   }
+
+  shouldFilter(rule) {
+    // this is to filter legacy schedule rules that is not compatible with current system any more
+    // all legacy rules should already been migrated in OldDataCleanSensor, any leftovers should be bug
+    // and here is a protection for that
+    if(rule.cronTime && rule.cronTime.startsWith("* *")) {
+      return true;
+    }   
+    return false;   
+  }
 
   setupPolicyQueue() {
     this.queue = new Queue('policy', {
@@ -118,6 +128,11 @@ class PolicyManager2 {
       const oldPolicy = this.jsonToPolicy(event.oldPolicy)
       const action = event.action
 
+      if(this.shouldFilter(policy)) {
+        done();
+        return;
+      }
+
       switch(action) {
       case "enforce": {
         return async(() => {

control/CategoryBlock.js

@@ -62,7 +62,9 @@ class CategoryBlock {
 
     // this policy has scope
     if(options.macSet) {
-//      await categoryUpdater.iptablesBlockCategoryPerDevice(category, options.macSet);
+      // block in filter
+      await categoryUpdater.iptablesBlockCategoryPerDevice(category, options.macSet);
+      // block in nat
       await categoryUpdater.iptablesBlockCategoryPerDeviceNew(category, options.macSet);
     } else {
       // global policy
@@ -81,8 +83,9 @@ class CategoryBlock {
 
     // this policy has scope
     if(options.macSet) {
-      // TBD
-//      await categoryUpdater.iptablesUnblockCategoryPerDevice(category, options.macSet);
+      // filter table
+      await categoryUpdater.iptablesUnblockCategoryPerDevice(category, options.macSet);
+      // nat table
       await categoryUpdater.iptablesUnblockCategoryPerDeviceNew(category, options.macSet);
     } else {
       // global policy

extension/dnsmasq/dnsmasq.js

@@ -1099,11 +1099,15 @@ module.exports = class DNSMASQ {
       bone.log("error", {
         version: sysManager.version(),
         type: 'DNSMASQ CRASH',
-        msg: "dnsmasq failed to restart after 5 retries",
+        msg: `dnsmasq failed to restart after ${this.failCount} retries`,
       }, null);
     } else {
-      let {stdout, stderr} = await execAsync("ps aux | grep dns[m]asq");
-      log.info("dnsmasq running status: \n", stdout, {})
+      try {
+        let {stdout, stderr} = await execAsync("ps aux | grep dns[m]asq");
+        log.info("dnsmasq running status: \n", stdout, stderr)
+      } catch(e) {
+        log.error("Failed to query process list of dnsmasq", e)
+      }
 
       // restart this service, something is wrong
       try {

extension/upnp/upnp.js

@@ -60,38 +60,39 @@ module.exports = class {
             instance = this;
             this.refreshTimers = {};
 
-            upnpIntervalHandler = setInterval(
-                () => {
-                    log.info("UPnP periodical check starts")
-                    if (upnpMappings.isEmpty) {
-                        log.info("No mapping registered.")
-                        return;
-                    }
-                    upnpClient.getMappings((err, results) => {
-                        if (err) {
-                            log.error("Failed to get current mappings", err);
-                            return;
-                        }
-                        log.info("Current mappings: ", results);
-
-                        upnpMappings.forEach((check) => {
-                            log.info("Checking registered mapping:", check);
-                            if (_.isEmpty(
-                                results.find((m) => mappingCompare(m, check))
-                            )) {
-                                log.info("Mapping no longer exists, adding back to router...")
-                                let { protocol, localPort, externalPort, description } = check;
-                                this.addPortMappingUPNP(protocol, localPort, externalPort, description)
-                            } else {
-                                log.info("Mapping still exists")
-                            }
-                        })
-                    })
-                },
-                upnpCheckInterval
-            )
+            if (process.title === "FireMain") {
+                upnpIntervalHandler = setInterval(
+                  () => {
+                      log.info("UPnP periodical check starts")
+                      if (upnpMappings.isEmpty) {
+                          log.info("No mapping registered.")
+                          return;
+                      }
+                      upnpClient.getMappings((err, results) => {
+                          if (err) {
+                              log.error("Failed to get current mappings", err);
+                              return;
+                          }
+                          log.info("Current mappings: ", results);
+
+                          upnpMappings.forEach((check) => {
+                              log.info("Checking registered mapping:", check);
+                              if (_.isEmpty(
+                                  results.find((m) => mappingCompare(m, check))
+                              )) {
+                                  log.info("Mapping no longer exists, adding back to router...")
+                                  let { protocol, localPort, externalPort, description } = check;
+                                  this.addPortMappingUPNP(protocol, localPort, externalPort, description)
+                              } else {
+                                  log.info("Mapping still exists")
+                              }
+                          })
+                      })
+                  },
+                  upnpCheckInterval
+              )
+            }
         }
-
         return instance;
     }
 

net2/BroDetect.js

@@ -930,8 +930,8 @@ module.exports = class {
                 if (port_flow == null) {
                     port_flow = {
                         sp: [obj['id.orig_p']],
-                        ob: Number(flowspec.ob),
-                        rb: Number(flowspec.rb),
+                        ob: Number(obj.orig_bytes),
+                        rb: Number(obj.resp_bytes),
                         ct: 1
                     };
                     flowspec.pf[portflowkey] = port_flow;

net2/Firewalla.js

@@ -112,12 +112,25 @@ function isDevelopmentVersion() {
 function isBeta() {
   let branch = getBranch()
   if(branch.match(/^beta_.*/)) {
-    return true
+    if(branch === 'beta_5_0') {
+      return false;
+    } else {
+      return true;
+    }
   } else {
     return false
   }  
 }
 
+function isAlpha() {
+  let branch = getBranch()
+  if(branch === 'beta_7_0') {
+    return true
+  } else {
+    return false
+  } 
+}
+
 function isProduction() {
   let branch = getBranch()
   if(branch.match(/^release_.*/)) {
@@ -140,6 +153,9 @@ function isProductionOrBeta() {
 function getReleaseType() {
   if(isProduction()) {
     return "prod"
+  } else if(isAlpha()) {
+    return "beta"; // TODO: will change to alpha when all app side codes are ready for alpha release
+    //return "alpha";
   } else if(isBeta()) {
     return "beta"
   } else if (isDevelopmentVersion()) {

net2/FlowManager.js

@@ -1063,7 +1063,7 @@ module.exports = class FlowManager {
       //     let key = o.sh+":"+o.dh+":"+o.fd;
       let flow = conndb[key];
       if (flow == null) {
-        conndb[key] = o;
+        conndb[key] = JSON.parse(JSON.stringify(o));  // this object may be presented multiple times in conndb due to different dst ports. Copy is needed to avoid interference between each other.
       } else {
         this.mergeFlow(flow, o);
       }

net2/HostManager.js

@@ -2303,6 +2303,12 @@ module.exports = class HostManager {
             let since = Date.now()/1000-60*60*24*7; // one week
             rclient.multi(multiarray).exec((err, replies) => {
                 _async.eachLimit(replies,2, (o, cb) => {
+                    if (!o) {
+                        // defensive programming
+                        cb();
+                        return;
+                    }
+
                     if (sysManager.isLocalIP(o.ipv4Addr) && o.lastActiveTimestamp>since) {
                         //log.info("Processing GetHosts ",o);
                         if (o.ipv4) {

net2/config.json

@@ -1,5 +1,5 @@
 {
-  "version": 1.959,
+  "version": 1.961,
   "firewallaBoneServerURL": "https://firewalla.encipher.io/bone/api/v3",
   "firewallaBoneFallbackServerURL": "https://firewalla.encipher.io/bone/api/v3",
   "firewallaBoneServerDevURL": "https://firewalla.encipher.io/bone/api/v0",
@@ -167,6 +167,14 @@
       "syssumflow": {
         "expires": -1,
         "count": 200
+      },
+      "categoryflow": {
+        "expires": 86400,
+        "count": 1000
+      },
+      "appflow": {
+        "expires": 86400,
+        "count": 1000
       }
     },
     "DeviceOfflineSensor": {

scripts/bootstrap.sha256sum

@@ -1,4 +1,4 @@
-2c870b6959d38009e9530c5982cb21e5bda0bd084c71f7bc488ecbc2cd58d6e9  check_fix_network.sh
+a15fd9b7c706e34d2561b78676c903ebcc9b8f7ffc704676be275abf884a7a9d  check_fix_network.sh
 988f0452446b544e00103d37b4f8cd350cf855c1cdb00367ad8c4c85d6869e6d  fireupgrade.sh
 39539f51a04e9e699bfae4886b137496112ca0dff4faae78ed2fb0234491c7e6  check_reset.sh
 3257a1b5c7736488bf8a8e9cef22eef90f718dad8daccff956dd3f80286da32f  ../etc/fireupgrade2.service

scripts/check_fix_network.sh

@@ -74,10 +74,11 @@ set_value() {
     saved_value=$2
     case ${kind} in
         ip)
+            /sbin/ip addr flush dev eth0 # flush legacy ips on eth0
             /sbin/ip addr replace ${saved_value} dev eth0
             ;;
         gw)
-            /sbin/route add default gw ${saved_value} eth0
+            /sbin/ip route replace default via ${saved_value} dev eth0 # upsert current default route
             ;;
     esac
 }

sensor/OldDataCleanSensor.js

@@ -40,6 +40,8 @@ let Promise = require('bluebird');
 let async = require('asyncawait/async');
 let await = require('asyncawait/await');
 
+const migrationPrefix = "oldDataMigration";
+
 let fConfig = require('../net2/config.js').getConfig();
 
 function arrayDiff(a, b) {
@@ -136,21 +138,18 @@ class OldDataCleanSensor extends Sensor {
 
   async cleanHourlyStats() {
     // FIXME: not well coded here, deprecated code
-    let keys = await rclient.keysAsync("stats:hour*");
-    let expireDate = Date.now() / 1000 - 60 * 60 * 24 * 30 * 6;
-    for (let j in keys) {
-      rclient.zscan(keys[j],0,(err,data)=>{
-        if (data && data.length==2) {
-          let array = data[1];
-          for (let i=0;i<array.length;i++) {
-            if (array[i]<expireDate) {
-              rclient.zrem(keys[j],array[i]);
-            }
-            i += Number(1);
-          }
-        }
+    let keys = await rclient.keysAsync("stats:hour:*");
+    const expireDate = Date.now() / 1000 - 60 * 60 * 24 * 2;
+    for (const key of keys) {
+      const timestamps = await rclient.zrangeAsync(key, 0, -1);
+      const expiredTimestamps = timestamps.filter((timestamp) => {
+        return Number(timestamp) < expireDate;
       });
+      if(expiredTimestamps.length > 0) {
+        await rclient.zremAsync([key, ...expiredTimestamps]);
+      }
     }
+
     // expire legacy stats:last24 keys if its expiration is not set
     keys = await rclient.keysAsync("stats:last24:*");
     for (let j in keys) {
@@ -187,6 +186,16 @@ class OldDataCleanSensor extends Sensor {
       return Promise.resolve();
   }
 
+  async cleanFlowX509() {
+    const flows = await rclient.keysAsync("flow:x509:*");
+    for(const flow of flows) {
+      const ttl = await rclient.ttlAsync(flow);
+      if(ttl === -1) {
+        await rclient.expireAsync(flow, 600); // 600 is default expire time if expire is not set
+      }
+    }
+  }
+
   cleanHostData(type, keyPattern, defaultExpireInterval) {
     let expireInterval = (this.config[type] && this.config[type].expires) ||
       defaultExpireInterval;
@@ -330,14 +339,17 @@ class OldDataCleanSensor extends Sensor {
       await (this.regularClean("software", "software:*"));
       await (this.regularClean("monitor", "monitor:flow:*"));
       await (this.regularClean("alarm", "alarm:ip4:*"));
-      await (this.regularClean("sumflow", "sumflow:*"));
-      await (this.regularClean("aggrflow", "aggrflow:*"));
+//      await (this.regularClean("sumflow", "sumflow:*"));
+//      await (this.regularClean("aggrflow", "aggrflow:*"));
       await (this.regularClean("syssumflow", "syssumflow:*"));
+      await (this.regularClean("categoryflow", "categoryflow:*"));
+      await (this.regularClean("appflow", "appflow:*"));
       await (this.cleanHourlyStats());
       await (this.cleanUserAgents());
       await (this.cleanHostData("host:ip4", "host:ip4:*", 60*60*24*30));
       await (this.cleanHostData("host:ip6", "host:ip6:*", 60*60*24*30));
       await (this.cleanHostData("host:mac", "host:mac:*", 60*60*24*365));
+      await (this.cleanFlowX509());
 
       await (this.cleanupAlarmExtendedKeys());
 
@@ -394,13 +406,39 @@ class OldDataCleanSensor extends Sensor {
     })
   }
 
+  async legacySchedulerMigration() {
+    const key = `${migrationPrefix}:legacySchedulerMigration`;
+    const result = await rclient.typeAsync(key);
+    if(result !== "none") {
+      return;
+    }
+
+    const policyRules = await pm2.loadActivePolicysAsync();
+    for(const rule of policyRules) {
+      if(rule.cronTime === "* * * * 1" && rule.duration === "432000") {
+        rule.cronTime = "0 0 * * 1,2,3,4,5";
+        rule.duration = "86390";
+        await pm2.updatePolicyAsync(rule);
+      } else if(rule.cronTime === "* * * * 6" && rule.duration === "172800") {
+        rule.cronTime = "0 0 * * 0,6";
+        rule.duration = "86390";
+        await pm2.updatePolicyAsync(rule);
+      }
+    }
+
+    await rclient.setAsync(key, "1");
+    return;
+  }
+
   run() {
     super.run();
 
     this.listen();
 
     this.hostPolicyMigration()
 
+    this.legacySchedulerMigration();
+
     setTimeout(() => {
       this.scheduledJob();
       this.oneTimeJob()

vpn/ovpngen.sh

@@ -33,6 +33,11 @@ TA="ta.key"
 #Build the client key and then encrypt the key
 sudo chmod 777 -R /etc/openvpn
 cd /etc/openvpn/easy-rsa
+# Change nextUpdate in openssl crl to 3600 days
+if [ -f /etc/openvpn/easy-rsa/openssl-1.0.0.cnf ]; then
+  sudo sed -i 's/default_crl_days= [0-9]*/default_crl_days= 3600/' /etc/openvpn/easy-rsa/openssl-1.0.0.cnf
+fi
+
 source ./vars
 if [ -f ~/ovpns/.ovpn.cn ]; then
   # Invalidate previous profile