From 4c3bb5e74af614d9395a2693d2cf1163516fb5c1 Mon Sep 17 00:00:00 2001 From: xinnige Date: Thu, 18 Apr 2024 18:01:04 +0800 Subject: [PATCH 1/2] Fix netgear router http-brute --- extension/nmap/scripts/http-brute.nse | 168 ++++++++++++++++++++++++++ net2/config.json | 2 +- sensor/InternalScanSensor.js | 106 +++++++++++++--- tests/test_internal_scan_sensor.js | 82 +++++++++---- 4 files changed, 312 insertions(+), 46 deletions(-) create mode 100644 extension/nmap/scripts/http-brute.nse diff --git a/extension/nmap/scripts/http-brute.nse b/extension/nmap/scripts/http-brute.nse new file mode 100644 index 0000000000..8a52dbb8da --- /dev/null +++ b/extension/nmap/scripts/http-brute.nse @@ -0,0 +1,168 @@ +local brute = require "brute" +local creds = require "creds" +local http = require "http" +local nmap = require "nmap" +local shortport = require "shortport" +local string = require "string" +local stdnse = require "stdnse" + +description = [[ +Performs brute force password auditing against http basic, digest and ntlm authentication. + +This script uses the unpwdb and brute libraries to perform password +guessing. Any successful guesses are stored in the nmap registry, using +the creds library, for other scripts to use. +]] + +--- +-- @usage +-- nmap --script http-brute -p 80 +-- +-- @output +-- PORT STATE SERVICE REASON +-- 80/tcp open http syn-ack +-- | http-brute: +-- | Accounts: +-- | user:user - Valid credentials +-- |_ Statistics: Performed 123 guesses in 1 seconds, average tps: 123 +-- +-- +-- @args http-brute.path points to the path protected by authentication (default: /) +-- @args http-brute.hostname sets the host header in case of virtual hosting +-- @args http-brute.method sets the HTTP method to use (default: GET) +-- +-- @xmloutput +-- +--
+-- Valid credentials +-- user +-- user +--
+-- +-- Performed 123 guesses in 1 seconds, average +-- tps: 123 + +-- +-- Version 0.1 +-- Created 07/30/2010 - v0.1 - created by Patrik Karlsson +-- Version 0.2 +-- 07/26/2012 - v0.2 - added digest auth support (Piotr Olma) +-- Version 0.3 +-- Created 06/20/2015 - added ntlm auth support (Gyanendra Mishra) + + +author = {"Patrik Karlsson", "Piotr Olma", "Gyanendra Mishra"} +license = "Same as Nmap--See https://nmap.org/book/man-legal.html" +categories = {"intrusive", "brute"} + + +portrule = shortport.port_or_service( {80, 443}, {"http", "https"}, "tcp", "open") + +Driver = { + + new = function(self, host, port, opts) + local o = {host=host, port=port, path=opts.path, method=opts.method, authmethod=opts.authmethod} + setmetatable(o, self) + self.__index = self + o.hostname = stdnse.get_script_args("http-brute.hostname") + return o + end, + + connect = function( self ) + -- This will cause problems, as there is no way for us to "reserve" + -- a socket. We may end up here early with a set of credentials + -- which won't be guessed until the end, due to socket exhaustion. + return true + end, + + get_opts = function( self ) + -- we need to supply the no_cache directive, or else the http library + -- incorrectly tells us that the authentication was successful + local opts = { + auth = { }, + no_cache = true, + bypass_cache = true, + header = { + -- nil just means not set, so default http.lua behavior + Host = self.hostname, + } + } + if self.authmethod == "digest" then + opts.auth.digest = true + elseif self.authmethod == "ntlm" then + opts.auth.ntlm = true + end + return opts + end, + + login = function( self, username, password ) + local opts_table = self:get_opts() + opts_table.auth.username = username + opts_table.auth.password = password + + local response = http.generic_request( self.host, self.port, self.method, self.path, opts_table) + + if not response.status then + local err = brute.Error:new(response["status-line"]) + err:setRetry(true) + return false, err + end + + -- Checking for ~= 401 *should* work to + -- but gave me a number of false positives last time I tried. + -- We decided to change it to ~= 4xx. + if ( response.status < 400 or response.status > 499 ) then + if ( response.body and string.find(response.body, ' 401 Authorization')) then + return false, brute.Error:new( "Incorrect password" ) + end + return true, creds.Account:new( username, password, creds.State.VALID) + end + return false, brute.Error:new( "Incorrect password" ) + end, + + disconnect = function( self ) + return true + end, + + check = function( self ) + return true + end, + +} + + +action = function( host, port ) + local status, result + local path = stdnse.get_script_args("http-brute.path") or "/" + local method = string.upper(stdnse.get_script_args("http-brute.method") or "GET") + + if ( not(path) ) then + return stdnse.format_output(false, "No path was specified (see http-brute.path)") + end + + local response = http.generic_request( host, port, method, path, { no_cache = true } ) + + if ( response.status ~= 401 ) then + return (" \n Path \"%s\" does not require authentication"):format(path) + end + + -- check if digest or ntlm auth is required + local authmethod = "basic" + local h = response.header['www-authenticate'] + if h then + h = h:lower() + if string.find(h, 'digest.-realm') then + authmethod = "digest" + end + if string.find(h, 'ntlm') then + authmethod = "ntlm" + end + end + + local engine = brute.Engine:new(Driver, host, port, {method=method, path=path, authmethod=authmethod}) + engine.options.script_name = SCRIPT_NAME + + status, result = engine:start() + + return result +end \ No newline at end of file diff --git a/net2/config.json b/net2/config.json index a217c56828..a5fb2af133 100644 --- a/net2/config.json +++ b/net2/config.json @@ -109,7 +109,7 @@ }, "AdblockPlugin": {}, "WireGuardPlugin": {}, - "InternalScanSensor": { "skip_verify": false}, + "InternalScanSensor": { "skip_verify": false, "strict_http": true}, "ExternalScanSensor": { "scanCooldown": 300 }, diff --git a/sensor/InternalScanSensor.js b/sensor/InternalScanSensor.js index 2b014ea493..ac2e901ae6 100644 --- a/sensor/InternalScanSensor.js +++ b/sensor/InternalScanSensor.js @@ -19,6 +19,7 @@ const Sensor = require('./Sensor.js').Sensor; const platformLoader = require('../platform/PlatformLoader.js'); const platform = platformLoader.getPlatform(); const firewalla = require('../net2/Firewalla.js'); +const fpath = require('path'); const fs = require('fs'); const fsp = require('fs').promises; const util = require('util'); @@ -26,11 +27,12 @@ const bone = require("../lib/Bone.js"); const rclient = require('../util/redis_manager.js').getRedisClient(); const cp = require('child_process'); const execAsync = util.promisify(cp.exec); -const scanDictPath = `${firewalla.getHiddenFolder()}/run/scan_dict`; +const scanConfigPath = `${firewalla.getHiddenFolder()}/run/scan_config`; const HostManager = require("../net2/HostManager.js"); const hostManager = new HostManager(); const IdentityManager = require('../net2/IdentityManager.js'); const xml2jsonBinary = firewalla.getFirewallaHome() + "/extension/xml2json/xml2json." + firewalla.getPlatform(); +const httpBruteScript = firewalla.getFirewallaHome() + "/extension/nmap/scripts/http-brute.nse"; const _ = require('lodash'); const bruteConfig = require('../extension/nmap/bruteConfig.json'); const AsyncLock = require('../vendor_lib/async-lock'); @@ -371,10 +373,9 @@ class InternalScanSensor extends Sensor { await rclient.setAsync(dictShaKey, boneShaData); log.info(`Loading dictionary from cloud...`); - //const data = require('./scan_dict.json'); if (data && data != '[]') { try { - await mkdirp(scanDictPath); + await mkdirp(scanConfigPath); } catch (err) { log.error("Error when mkdir:", err); return; @@ -390,6 +391,10 @@ class InternalScanSensor extends Sensor { // process extraConfig (http-form-brute) await this._process_dict_extras(dictData.extraConfig); + } else { + // cleanup + await this._cleanup_dict_creds(); + await this._cleanup_dict_extras(); } } } @@ -400,10 +405,13 @@ class InternalScanSensor extends Sensor { const commonCreds = dictData.commonCreds && dictData.commonCreds.creds || []; const customCreds = dictData.customCreds; + let newCredFnames = []; + let newUserFnames = []; + let newPwdFnames = []; if (customCreds) { for (const key of Object.keys(customCreds)) { - // eg. {firewalla}/run/scan_dict/*_users.lst + // eg. {firewalla}/run/scan_config/*_users.lst let scanUsers = customCreds[key].usernames || []; if (_.isArray(scanUsers) && _.isArray(commonUser)) { scanUsers.push.apply(scanUsers, commonUser); @@ -411,11 +419,12 @@ class InternalScanSensor extends Sensor { if (scanUsers.length > 0) { const txtUsers = _.uniqWith(scanUsers, _.isEqual).join("\n"); if (txtUsers.length > 0) { - await fsp.writeFile(scanDictPath + "/" + key.toLowerCase() + "_users.lst", txtUsers); + newUserFnames.push(key.toLowerCase() + "_users.lst"); + await fsp.writeFile(scanConfigPath + "/" + key.toLowerCase() + "_users.lst", txtUsers); } } - // eg. {firewalla}/run/scan_dict/*_pwds.lst + // eg. {firewalla}/run/scan_config/*_pwds.lst let scanPwds = customCreds[key].passwords || []; if (_.isArray(scanPwds) && _.isArray(commonPwds)) { scanPwds.push.apply(scanPwds, commonPwds); @@ -423,11 +432,12 @@ class InternalScanSensor extends Sensor { if (scanPwds.length > 0) { const txtPwds = _.uniqWith(scanPwds, _.isEqual).join("\n"); if (txtPwds.length > 0) { - await fsp.writeFile(scanDictPath + "/" + key.toLowerCase() + "_pwds.lst", txtPwds); + newPwdFnames.push(key.toLowerCase() + "_pwds.lst"); + await fsp.writeFile(scanConfigPath + "/" + key.toLowerCase() + "_pwds.lst", txtPwds); } } - // eg. {firewalla}/run/scan_dict/*_creds.lst + // eg. {firewalla}/run/scan_config/*_creds.lst let scanCreds = customCreds[key].creds || []; if (_.isArray(scanCreds) && _.isArray(commonCreds)) { scanCreds.push.apply(scanCreds, commonCreds); @@ -435,18 +445,58 @@ class InternalScanSensor extends Sensor { if (scanCreds.length > 0) { const txtCreds = _.uniqWith(scanCreds.map(i => i.user+'/'+i.password), _.isEqual).join("\n"); if (txtCreds.length > 0) { - await fsp.writeFile(scanDictPath + "/" + key.toLowerCase() + "_creds.lst", txtCreds); + newCredFnames.push(key.toLowerCase() + "_creds.lst"); + await fsp.writeFile(scanConfigPath + "/" + key.toLowerCase() + "_creds.lst", txtCreds); } } } } + // remove outdated *.lst + await this._clean_diff_creds(scanConfigPath, '_users.lst', newUserFnames); + await this._clean_diff_creds(scanConfigPath, '_pwds.lst', newPwdFnames); + await this._clean_diff_creds(scanConfigPath, '_creds.lst', newCredFnames); } async _process_dict_extras(extraConfig) { if (!extraConfig) { return; } - await rclient.hsetAsync("sys:config", 'weak_password_scan', JSON.stringify(extraConfig)); + await rclient.hsetAsync('sys:config', 'weak_password_scan', JSON.stringify(extraConfig)); + } + + async _clean_diff_creds(dir, suffix, newFnames) { + const fnames = await this._list_suffix_files(scanConfigPath, suffix); + const diff = fnames.filter(x => !newFnames.includes(x)); + const rmFiles = diff.map(file => {return fpath.join(dir, file)}); + log.debug(`rm diff files *${suffix}`, rmFiles); + for (const filepath of rmFiles) { + await execAsync(`rm -f ${filepath}`).catch(err => {log.warn(`fail to rm ${filepath},`, err.stderr)}); + } + } + + async _cleanup_dict_creds() { + await this._remove_suffix_files(scanConfigPath, '_creds.lst'); + await this._remove_suffix_files(scanConfigPath, '_users.lst'); + await this._remove_suffix_files(scanConfigPath, '_pwds.lst'); + } + + async _list_suffix_files(dir, suffix) { + const filenames = await fsp.readdir(dir); + const fnames = filenames.filter(name => {return name.endsWith(suffix)}); + log.debug(`ls ${dir} *${suffix}`, fnames); + return fnames; + } + + async _remove_suffix_files(dir, suffix) { + const filenames = await this._list_suffix_files(dir, suffix); + const rmFiles = filenames.map(file => {return fpath.join(dir, file)}); + for (const filepath of rmFiles) { + await execAsync(`rm -f ${filepath}`).catch(err => {log.warn(`fail to rm ${filepath},`, err.stderr)}); + } + } + + async _cleanup_dict_extras() { + await rclient.hdelAsync('sys:config', 'weak_password_scan'); } _getCmdStdout(cmd, subTask) { @@ -494,11 +544,22 @@ class InternalScanSensor extends Sensor { return util.format('sudo timeout 5430s nmap -p %s %s %s -oX - | %s', port, cmdArg.join(' '), ipAddr, xml2jsonBinary); } - _genNmapCmd_default(ipAddr, port, scripts) { + async _genNmapCmd_default(ipAddr, port, scripts) { let nmapCmdList = []; for (const bruteScript of scripts) { let cmdArg = []; - cmdArg.push(util.format('--script %s', bruteScript.scriptName)); + // customized http-brute + let customHttpBrute = false; + if (this.config.strict_http === true && bruteScript.scriptName == 'http-brute') { + if (await fsp.access(httpBruteScript, fs.constants.F_OK).then(() => true).catch((err) => false)) { + customHttpBrute = true; + } + } + if (customHttpBrute === true) { + cmdArg.push(util.format('--script %s', httpBruteScript)); + } else { + cmdArg.push(util.format('--script %s', bruteScript.scriptName)); + } if (bruteScript.otherArgs) { cmdArg.push(bruteScript.otherArgs); } @@ -523,7 +584,7 @@ class InternalScanSensor extends Sensor { scriptArgs.push(bruteScript.scriptArgs); } if (bruteScript.scriptName.indexOf("brute") > -1) { - const credsFile = scanDictPath + "/" + serviceName.toLowerCase() + "_creds.lst"; + const credsFile = scanConfigPath + "/" + serviceName.toLowerCase() + "_creds.lst"; if (await fsp.access(credsFile, fs.constants.F_OK).then(() => true).catch((err) => false)) { scriptArgs.push("brute.mode=creds,brute.credfile=" + credsFile); needCustom = true; @@ -569,12 +630,12 @@ class InternalScanSensor extends Sensor { scriptArgs.push(bruteScript.scriptArgs); } if (bruteScript.scriptName.indexOf("brute") > -1) { - const scanUsersFile = scanDictPath + "/" + serviceName.toLowerCase() + "_users.lst"; + const scanUsersFile = scanConfigPath + "/" + serviceName.toLowerCase() + "_users.lst"; if (await fsp.access(scanUsersFile, fs.constants.F_OK).then(() => true).catch((err) => false)) { scriptArgs.push("userdb=" + scanUsersFile); needCustom = true; } - const scanPwdsFile = scanDictPath + "/" + serviceName.toLowerCase() + "_pwds.lst"; + const scanPwdsFile = scanConfigPath + "/" + serviceName.toLowerCase() + "_pwds.lst"; if (await fsp.access(scanPwdsFile, fs.constants.F_OK).then(() => true).catch((err) => false)) { scriptArgs.push("passdb=" + scanPwdsFile); needCustom = true; @@ -617,7 +678,7 @@ class InternalScanSensor extends Sensor { const extraConfig = JSON.parse(data); // 1. compose default userdb/passdb (NO apply extra configs) - const defaultCmds = this._genNmapCmd_default(ipAddr, port, scripts); + const defaultCmds = await this._genNmapCmd_default(ipAddr, port, scripts); if (defaultCmds.length > 0) { nmapCmdList = nmapCmdList.concat(defaultCmds); } @@ -725,7 +786,7 @@ class InternalScanSensor extends Sensor { async recheckWeakPassword(ipAddr, port, scriptName, weakPassword) { switch (scriptName) { case "http-brute": - const credfile = scanDictPath + "/" + ipAddr + "_" + port + "_credentials.lst"; + const credfile = scanConfigPath + "/" + ipAddr + "_" + port + "_credentials.lst"; const {username, password} = weakPassword; return await this.httpbruteCreds(ipAddr, port, username, password, credfile); default: @@ -735,7 +796,7 @@ class InternalScanSensor extends Sensor { // check if username/password valid credentials async httpbruteCreds(ipAddr, port, username, password, credfile) { - credfile = credfile || scanDictPath + "/tmp_credentials.lst"; + credfile = credfile || scanConfigPath + "/tmp_credentials.lst"; let creds; if (password == '') { creds = `${username}/`; @@ -757,7 +818,14 @@ class InternalScanSensor extends Sensor { } } - const cmd = `sudo nmap -p ${port} --script http-brute --script-args unpwdb.timelimit=10s,brute.mode=creds,brute.credfile=${credfile} ${ipAddr} | grep "Valid credentials" | wc -l` + let scriptName = 'http-brute'; + if (this.config.strict_http === true ) { + if (await fsp.access(httpBruteScript, fs.constants.F_OK).then(() => true).catch((err) => false)) { + scriptName = httpBruteScript; + } + } + + const cmd = `sudo nmap -p ${port} --script ${scriptName} --script-args unpwdb.timelimit=10s,brute.mode=creds,brute.credfile=${credfile} ${ipAddr} | grep "Valid credentials" | wc -l` const result = await execAsync(cmd); if (result.stderr) { log.warn(`fail to running command: ${cmd} (user/pass=${username}/${password}), err: ${result.stderr}`); diff --git a/tests/test_internal_scan_sensor.js b/tests/test_internal_scan_sensor.js index ca56ea0ffa..89994c2556 100644 --- a/tests/test_internal_scan_sensor.js +++ b/tests/test_internal_scan_sensor.js @@ -56,7 +56,7 @@ const bruteConfig = { }} const extraConfig = { - 'http-form-brute': [{path: '/oauth', passvar: 'token'}, {uservar: 'name'}], + 'http-form-brute': [{},{path: '/oauth', passvar: 'token', uservar: 'username'}, {uservar: 'name'}], }; describe('Test InternalScanSensor', function() { @@ -90,11 +90,11 @@ describe('Test InternalScanSensor', function() { }; await this.plugin._process_dict_creds(data); - const usercount = await execAsync('grep -c "" ~/.firewalla/run/scan_dict/ssh_users.lst'); + const usercount = await execAsync('grep -c "" ~/.firewalla/run/scan_config/ssh_users.lst'); expect(usercount.stdout.trim()).to.be.equal('4'); - const passcount = await execAsync('grep -c "" ~/.firewalla/run/scan_dict/ssh_pwds.lst'); + const passcount = await execAsync('grep -c "" ~/.firewalla/run/scan_config/ssh_pwds.lst'); expect(passcount.stdout.trim()).to.be.equal('5'); - const credcount = await execAsync('grep -c "" ~/.firewalla/run/scan_dict/ssh_creds.lst'); + const credcount = await execAsync('grep -c "" ~/.firewalla/run/scan_config/ssh_creds.lst'); expect(credcount.stdout.trim()).to.be.equal('1'); }); @@ -109,12 +109,6 @@ describe('Test InternalScanSensor', function() { await this.plugin._process_dict_extras(extras); - const users = await execAsync('sudo cat /usr/share/nmap/nselib/data/custom_usernames.lst'); - expect(users.stdout.trim()).to.be.equal('admin\nuser\nusername'); - - const passes = await execAsync('sudo cat /usr/share/nmap/nselib/data/custom_passwords.lst'); - expect(passes.stdout.trim()).to.be.equal('admin\nadmin123'); - const data = await rclient.hgetAsync('sys:config', 'weak_password_scan'); expect(data).to.be.equal('{"http-form-brute":[{"path":"/oauth"},{"uservar":"name"}]}'); }); @@ -147,36 +141,46 @@ describe('Test InternalScanSensor', function() { expect(results).to.be.eql(exp); }); - it('should generate nmap default', () => { - const httpcmds = this.plugin._genNmapCmd_default('192.168.196.105', 80, bruteConfig['tcp_80'].scripts); + it('should generate nmap default', async() => { + const httpcmds = await this.plugin._genNmapCmd_default('192.168.196.105', 80, bruteConfig['tcp_80'].scripts); expect(httpcmds.map(i=>i.cmd)).to.eql([ 'sudo timeout 5430s nmap -p 80 --script http-brute --script-args unpwdb.timelimit=60m 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', ]); - const telcmds = this.plugin._genNmapCmd_default('192.168.196.105', 23, bruteConfig['tcp_23'].scripts); + const telcmds = await this.plugin._genNmapCmd_default('192.168.196.105', 23, bruteConfig['tcp_23'].scripts); expect(telcmds.map(i=>i.cmd)).to.eql([ 'sudo timeout 5430s nmap -p 23 --script telnet-brute -v 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', ]); + + this.plugin.config = {strict_http: true}; + const customhttpcmds = await this.plugin._genNmapCmd_default('192.168.196.105', 80, bruteConfig['tcp_80'].scripts); + expect(customhttpcmds.map(i=>i.cmd)).to.eql([ + 'sudo timeout 5430s nmap -p 80 --script /home/pi/firewalla/extension/nmap/scripts/http-brute.nse --script-args unpwdb.timelimit=60m 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + ]); + + this.plugin.config = {}; }); it('should generate nmap credfile', async () => { const httpcmds = await this.plugin._genNmapCmd_credfile('192.168.196.105', 80, 'HTTP', bruteConfig['tcp_80'].scripts, extraConfig); expect(httpcmds.map(i=>i.cmd)).to.eql([ - 'sudo timeout 5430s nmap -p 80 --script http-brute --script-args unpwdb.timelimit=60m,brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_dict/http_creds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', - 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_dict/http_creds.lst,http-form-brute.path=/oauth,http-form-brute.passvar=token 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', - 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_dict/http_creds.lst,http-form-brute.uservar=name 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-brute --script-args unpwdb.timelimit=60m,brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_config/http_creds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_config/http_creds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_config/http_creds.lst,http-form-brute.path=/oauth,http-form-brute.uservar=username,http-form-brute.passvar=token 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_config/http_creds.lst,http-form-brute.uservar=name 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', ]); const httpbrutecmds = await this.plugin._genNmapCmd_credfile('192.168.196.105', 80, 'HTTP', bruteConfig['tcp_80'].scripts, null); expect(httpbrutecmds.map(i=>i.cmd)).to.eql([ - 'sudo timeout 5430s nmap -p 80 --script http-brute --script-args unpwdb.timelimit=60m,brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_dict/http_creds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', - 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_dict/http_creds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-brute --script-args unpwdb.timelimit=60m,brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_config/http_creds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_config/http_creds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', ]); const telcmds = await this.plugin._genNmapCmd_credfile('192.168.196.105', 23, 'TELNET', bruteConfig['tcp_23'].scripts); expect(telcmds.map(i=>i.cmd)).to.eql([ - 'sudo timeout 5430s nmap -p 23 --script telnet-brute -v --script-args brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_dict/telnet_creds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64' + 'sudo timeout 5430s nmap -p 23 --script telnet-brute -v --script-args brute.mode=creds,brute.credfile=/home/pi/.firewalla/run/scan_config/telnet_creds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64' ]); const nocmds = await this.plugin._genNmapCmd_credfile('192.168.196.105', 53, 'DNS', bruteConfig['tcp_53'].scripts); @@ -186,21 +190,22 @@ describe('Test InternalScanSensor', function() { it('should generate nmap userpass', async () => { const httpcmds = await this.plugin._genNmapCmd_userpass('192.168.196.105', 80, 'HTTP', bruteConfig['tcp_80'].scripts, extraConfig); expect(httpcmds.map(i=>i.cmd)).to.eql([ - 'sudo timeout 5430s nmap -p 80 --script http-brute --script-args unpwdb.timelimit=60m,userdb=/home/pi/.firewalla/run/scan_dict/http_users.lst,passdb=/home/pi/.firewalla/run/scan_dict/http_pwds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', - 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,userdb=/home/pi/.firewalla/run/scan_dict/http_users.lst,passdb=/home/pi/.firewalla/run/scan_dict/http_pwds.lst,http-form-brute.path=/oauth,http-form-brute.passvar=token 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', - 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,userdb=/home/pi/.firewalla/run/scan_dict/http_users.lst,passdb=/home/pi/.firewalla/run/scan_dict/http_pwds.lst,http-form-brute.uservar=name 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-brute --script-args unpwdb.timelimit=60m,userdb=/home/pi/.firewalla/run/scan_config/http_users.lst,passdb=/home/pi/.firewalla/run/scan_config/http_pwds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,userdb=/home/pi/.firewalla/run/scan_config/http_users.lst,passdb=/home/pi/.firewalla/run/scan_config/http_pwds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,userdb=/home/pi/.firewalla/run/scan_config/http_users.lst,passdb=/home/pi/.firewalla/run/scan_config/http_pwds.lst,http-form-brute.path=/oauth,http-form-brute.uservar=username,http-form-brute.passvar=token 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,userdb=/home/pi/.firewalla/run/scan_config/http_users.lst,passdb=/home/pi/.firewalla/run/scan_config/http_pwds.lst,http-form-brute.uservar=name 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', ]); const httpbrutecmds = await this.plugin._genNmapCmd_userpass('192.168.196.105', 80, 'HTTP', bruteConfig['tcp_80'].scripts); expect(httpbrutecmds.map(i=>i.cmd)).to.eql([ - 'sudo timeout 5430s nmap -p 80 --script http-brute --script-args unpwdb.timelimit=60m,userdb=/home/pi/.firewalla/run/scan_dict/http_users.lst,passdb=/home/pi/.firewalla/run/scan_dict/http_pwds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', - 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,userdb=/home/pi/.firewalla/run/scan_dict/http_users.lst,passdb=/home/pi/.firewalla/run/scan_dict/http_pwds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-brute --script-args unpwdb.timelimit=60m,userdb=/home/pi/.firewalla/run/scan_config/http_users.lst,passdb=/home/pi/.firewalla/run/scan_config/http_pwds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m,userdb=/home/pi/.firewalla/run/scan_config/http_users.lst,passdb=/home/pi/.firewalla/run/scan_config/http_pwds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', ]); const telcmds = await this.plugin._genNmapCmd_userpass('192.168.196.105', 23, 'TELNET', bruteConfig['tcp_23'].scripts); expect(telcmds.map(i=>i.cmd)).to.eql([ - 'sudo timeout 5430s nmap -p 23 --script telnet-brute -v --script-args userdb=/home/pi/.firewalla/run/scan_dict/telnet_users.lst,passdb=/home/pi/.firewalla/run/scan_dict/telnet_pwds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64' + 'sudo timeout 5430s nmap -p 23 --script telnet-brute -v --script-args userdb=/home/pi/.firewalla/run/scan_config/telnet_users.lst,passdb=/home/pi/.firewalla/run/scan_config/telnet_pwds.lst 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64' ]); const nocmds = await this.plugin._genNmapCmd_userpass('192.168.196.105', 53, 'DNS', bruteConfig['tcp_53'].scripts); @@ -219,7 +224,7 @@ describe('Test InternalScanSensor', function() { }); it('should check http brute creds', async() => { - const ret = await this.plugin.httpbruteCreds('127.0.0.1', 8080, 'admin', '', '/home/pi/.firewalla/run/scan_dict/127.0.0.1_8080_credentials.lst'); + const ret = await this.plugin.httpbruteCreds('127.0.0.1', 8080, 'admin', '', '/home/pi/.firewalla/run/scan_config/127.0.0.1_8080_credentials.lst'); expect(ret).to.equal(false); }); @@ -228,10 +233,35 @@ describe('Test InternalScanSensor', function() { expect(result).to.equal(false); }); + it('should clean up local scan config redis', async() => { + await rclient.hsetAsync('sys:config', 'weak_password_scan', '{"http-form-brute":[{"path":"/oauth"},{"uservar":"name"}]}').catch( + (err) => {log.warn('hset err', err.stderr)}); + + await this.plugin._cleanup_dict_extras(); + const data = await rclient.hgetAsync('sys:config', 'weak_password_scan'); + expect(data).to.be.null; + }); + + + it('should clean up local scan config dir', async() => { + await execAsync('touch /home/pi/.firewalla/run/scan_config/ssh_tests.lst /home/pi/.firewalla/run/scan_config/http_tests.lst /home/pi/.firewalla/run/scan_config/telnet_tests.lst').catch( + (err) => {log.warn('touch err', err.stderr)}); + + await this.plugin._clean_diff_creds('/home/pi/.firewalla/run/scan_config/', '_tests.lst', ['ssh_tests.lst']); + const credFiles = await this.plugin._list_suffix_files('/home/pi/.firewalla/run/scan_config/', '_tests.lst'); + expect(credFiles).to.be.eql(['ssh_tests.lst']); + await execAsync('rm -f /home/pi/.firewalla/run/scan_config/ssh_tests.lst').catch((err) => {}); + + await this.plugin._cleanup_dict_creds(); + const files = await this.plugin._list_suffix_files('/home/pi/.firewalla/run/scan_config/', '.lst'); + expect(files).to.be.empty; + }); + it('should check dictionary', async() => { await this.plugin.checkDictionary(); }); + // A connected device is expected to run a http-server with basic-auth enabled // e.g. tiny-http-server --authfile userpass.txt --port 80 --bind 0.0.0.0 --directory html it.skip('should nmap guess passwords with weak password enviroment', async() => { From 7eac87ea7f147928b7667997cbc46c52b58e0d4e Mon Sep 17 00:00:00 2001 From: xinnige Date: Wed, 24 Apr 2024 15:45:24 +0800 Subject: [PATCH 2/2] Dynamic load http-brute script --- extension/nmap/scripts/http-brute.nse | 168 -------------------------- platform/all/files/assets.lst | 1 + sensor/InternalScanSensor.js | 2 +- tests/test_internal_scan_sensor.js | 3 +- 4 files changed, 4 insertions(+), 170 deletions(-) delete mode 100644 extension/nmap/scripts/http-brute.nse diff --git a/extension/nmap/scripts/http-brute.nse b/extension/nmap/scripts/http-brute.nse deleted file mode 100644 index 8a52dbb8da..0000000000 --- a/extension/nmap/scripts/http-brute.nse +++ /dev/null @@ -1,168 +0,0 @@ -local brute = require "brute" -local creds = require "creds" -local http = require "http" -local nmap = require "nmap" -local shortport = require "shortport" -local string = require "string" -local stdnse = require "stdnse" - -description = [[ -Performs brute force password auditing against http basic, digest and ntlm authentication. - -This script uses the unpwdb and brute libraries to perform password -guessing. Any successful guesses are stored in the nmap registry, using -the creds library, for other scripts to use. -]] - ---- --- @usage --- nmap --script http-brute -p 80 --- --- @output --- PORT STATE SERVICE REASON --- 80/tcp open http syn-ack --- | http-brute: --- | Accounts: --- | user:user - Valid credentials --- |_ Statistics: Performed 123 guesses in 1 seconds, average tps: 123 --- --- --- @args http-brute.path points to the path protected by authentication (default: /) --- @args http-brute.hostname sets the host header in case of virtual hosting --- @args http-brute.method sets the HTTP method to use (default: GET) --- --- @xmloutput --- ---
--- Valid credentials --- user --- user ---
--- --- Performed 123 guesses in 1 seconds, average --- tps: 123 - --- --- Version 0.1 --- Created 07/30/2010 - v0.1 - created by Patrik Karlsson --- Version 0.2 --- 07/26/2012 - v0.2 - added digest auth support (Piotr Olma) --- Version 0.3 --- Created 06/20/2015 - added ntlm auth support (Gyanendra Mishra) - - -author = {"Patrik Karlsson", "Piotr Olma", "Gyanendra Mishra"} -license = "Same as Nmap--See https://nmap.org/book/man-legal.html" -categories = {"intrusive", "brute"} - - -portrule = shortport.port_or_service( {80, 443}, {"http", "https"}, "tcp", "open") - -Driver = { - - new = function(self, host, port, opts) - local o = {host=host, port=port, path=opts.path, method=opts.method, authmethod=opts.authmethod} - setmetatable(o, self) - self.__index = self - o.hostname = stdnse.get_script_args("http-brute.hostname") - return o - end, - - connect = function( self ) - -- This will cause problems, as there is no way for us to "reserve" - -- a socket. We may end up here early with a set of credentials - -- which won't be guessed until the end, due to socket exhaustion. - return true - end, - - get_opts = function( self ) - -- we need to supply the no_cache directive, or else the http library - -- incorrectly tells us that the authentication was successful - local opts = { - auth = { }, - no_cache = true, - bypass_cache = true, - header = { - -- nil just means not set, so default http.lua behavior - Host = self.hostname, - } - } - if self.authmethod == "digest" then - opts.auth.digest = true - elseif self.authmethod == "ntlm" then - opts.auth.ntlm = true - end - return opts - end, - - login = function( self, username, password ) - local opts_table = self:get_opts() - opts_table.auth.username = username - opts_table.auth.password = password - - local response = http.generic_request( self.host, self.port, self.method, self.path, opts_table) - - if not response.status then - local err = brute.Error:new(response["status-line"]) - err:setRetry(true) - return false, err - end - - -- Checking for ~= 401 *should* work to - -- but gave me a number of false positives last time I tried. - -- We decided to change it to ~= 4xx. - if ( response.status < 400 or response.status > 499 ) then - if ( response.body and string.find(response.body, ' 401 Authorization')) then - return false, brute.Error:new( "Incorrect password" ) - end - return true, creds.Account:new( username, password, creds.State.VALID) - end - return false, brute.Error:new( "Incorrect password" ) - end, - - disconnect = function( self ) - return true - end, - - check = function( self ) - return true - end, - -} - - -action = function( host, port ) - local status, result - local path = stdnse.get_script_args("http-brute.path") or "/" - local method = string.upper(stdnse.get_script_args("http-brute.method") or "GET") - - if ( not(path) ) then - return stdnse.format_output(false, "No path was specified (see http-brute.path)") - end - - local response = http.generic_request( host, port, method, path, { no_cache = true } ) - - if ( response.status ~= 401 ) then - return (" \n Path \"%s\" does not require authentication"):format(path) - end - - -- check if digest or ntlm auth is required - local authmethod = "basic" - local h = response.header['www-authenticate'] - if h then - h = h:lower() - if string.find(h, 'digest.-realm') then - authmethod = "digest" - end - if string.find(h, 'ntlm') then - authmethod = "ntlm" - end - end - - local engine = brute.Engine:new(Driver, host, port, {method=method, path=path, authmethod=authmethod}) - engine.options.script_name = SCRIPT_NAME - - status, result = engine:start() - - return result -end \ No newline at end of file diff --git a/platform/all/files/assets.lst b/platform/all/files/assets.lst index ace156195c..d05b54c4ab 100644 --- a/platform/all/files/assets.lst +++ b/platform/all/files/assets.lst @@ -1,3 +1,4 @@ /home/pi/.firewalla/run/device-detector-regexes.tar.gz /all/device-detector-regexes.tar.gz 644 ':' 'cd /home/pi/.firewalla/run/; tar -zxf device-detector-regexes.tar.gz; redis-cli publish "TO.FireMain" '\''{"type":"DeviceDetector:RegexUpdated","toProcess":"FireMain"}'\' /home/pi/.firewalla/run/assets/detect.tar.gz /all/detect.tar.gz 644 ':' 'cd /home/pi/.firewalla/run/assets; tar -zxf detect.tar.gz; redis-cli publish "assets:updated" "detect/"' /home/pi/.firewalla/run/noise_domains.tar.gz /all/noise_domains.tar.gz 644 'mkdir -p /home/pi/.firewalla/run/noise_domains; rm -f /home/pi/.firewalla/run/noise_domains/*; ' ' cd /home/pi/.firewalla/run/; tar xzf noise_domains.tar.gz -C noise_domains; ' +/home/pi/.firewalla/run/assets/http-brute.nse /all/http-brute.nse 644 diff --git a/sensor/InternalScanSensor.js b/sensor/InternalScanSensor.js index ac2e901ae6..d009505ad6 100644 --- a/sensor/InternalScanSensor.js +++ b/sensor/InternalScanSensor.js @@ -32,7 +32,7 @@ const HostManager = require("../net2/HostManager.js"); const hostManager = new HostManager(); const IdentityManager = require('../net2/IdentityManager.js'); const xml2jsonBinary = firewalla.getFirewallaHome() + "/extension/xml2json/xml2json." + firewalla.getPlatform(); -const httpBruteScript = firewalla.getFirewallaHome() + "/extension/nmap/scripts/http-brute.nse"; +const httpBruteScript = firewalla.getHiddenFolder() + "/run/assets/http-brute.nse"; const _ = require('lodash'); const bruteConfig = require('../extension/nmap/bruteConfig.json'); const AsyncLock = require('../vendor_lib/async-lock'); diff --git a/tests/test_internal_scan_sensor.js b/tests/test_internal_scan_sensor.js index 89994c2556..33ede8a05b 100644 --- a/tests/test_internal_scan_sensor.js +++ b/tests/test_internal_scan_sensor.js @@ -154,9 +154,10 @@ describe('Test InternalScanSensor', function() { ]); this.plugin.config = {strict_http: true}; + await execAsync("cp /usr/share/nmap/scripts/http-brute.nse /home/pi/.firewalla/run/assets/http-brute.nse;"); const customhttpcmds = await this.plugin._genNmapCmd_default('192.168.196.105', 80, bruteConfig['tcp_80'].scripts); expect(customhttpcmds.map(i=>i.cmd)).to.eql([ - 'sudo timeout 5430s nmap -p 80 --script /home/pi/firewalla/extension/nmap/scripts/http-brute.nse --script-args unpwdb.timelimit=60m 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', + 'sudo timeout 5430s nmap -p 80 --script /home/pi/.firewalla/run/assets/http-brute.nse --script-args unpwdb.timelimit=60m 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', 'sudo timeout 5430s nmap -p 80 --script http-form-brute --script-args unpwdb.timelimit=60m 192.168.196.105 -oX - | /home/pi/firewalla/extension/xml2json/xml2json.x86_64', ]);