web_exploit_detector.js

import os      from 'os';
import path    from 'path';
import Promise from 'bluebird';

const fs = Promise.promisifyAll(require('graceful-fs'));

let crypto = null;
try {
  crypto = require('crypto');
} catch (err) {
  crypto = null;
}

export const rules = require('../rules');

export function loadExceptions(logger) {
  const ex_file = path.join(os.homedir(), 'wed-exceptions.json');
  return fs.accessAsync(ex_file, fs.R_OK)
    .then(res => {
      if (res)
        return [];
      return fs.readFileAsync(ex_file)
        .then(res => {
          return JSON.parse(res.toString());
        })
        .catch(err => {
          logger(1, `ERROR: unable to load exceptions: ${err.toString()}`);
          return [];
        });
    })
    .catch(() => {
      return [];
    });
}

export function executeTests(settings) {
  let exceptions = [];

  if (!settings.logger)
    settings.logger = (level, msg) => msg;

  return loadExceptions(settings.logger)
    .then(res => {
      exceptions = res;
    })

    .then(() => {
      settings.logger(3, 'Getting file list...');
      return getFileList(settings.webroot, settings.logger);
    })

    .then(files => {

      // Complete processing of all rules
      //  - Parallel mode: return the result of Promise.all()
      //  - Serial mode: use mapSeries() to call each function in the array in
      //    turn in order to process the rule
      const _processPromises = (promises) =>
        settings.parallel ?
          Promise.all(promises) :
          Promise.mapSeries(
            promises,
            func => func()
          );

      const activeRules = rules

        // Remove debug rules if not in debug mode
        .filter(rule => settings.debug ? true : rule.tags.filter(tag => tag === 'debug').length === 0)

        // Remove deprecated rules unless "include_deprecated" is set
        .filter(rule => settings.include_deprecated ? true : !rule.deprecated)

        // Remove all rules not in "tags" (if set)
        .filter(rule => {
          if (!settings.tags)
            return true;
          const tags_arr = settings.tags.split(',').map(x => x.trim().toLowerCase());
          return rule.tags.filter(tag => tags_arr.filter(x => x === tag).length > 0).length > 0;
        })

        // Remove all rules with tags included in "exclude_tags" (if set)
        .filter(rule => {
          if (!settings.exclude_tags)
            return true;
          const tags_arr = settings.exclude_tags.split(',').map(x => x.trim().toLowerCase());
          return rule.tags.filter(tag => tags_arr.filter(x => x === tag).length > 0).length === 0;
        });

      settings.logger(3, `Processing ${activeRules.length} rules ${settings.parallel ? 'in parallel ' : ''}on ${files.length} files...`);

      // Process depending on current settings.mode
      //  - Parallel mode: map each rule to the result of processing the rule
      //    (a Promise) directly
      //  - Serial mode: map each rule to a function that can be called to
      //    start rule processing
      return _processPromises(
        files.length === 0 ?
          [] :
          (settings.parallel ?
            files.map(file => processRulesOnFile(file, activeRules, exceptions[file], settings.logger)) :
            files.map(file => () => processRulesOnFile(file, activeRules, exceptions[file], settings.logger))
          )
      )
        // Re-group "file => matched rules" to "rule => matched files" and
        // remove rules with no matches
        .then(res => {
          const ruleMatches = res
            .reduce((acc, filerec) => {
              if (filerec.matches.length === 0)
                return acc;
              filerec.matches.forEach(ruleMatch => {
                if (acc[ruleMatch.id])
                  acc[ruleMatch.id].matches.push(filerec.file);
                else
                  acc[ruleMatch.id] = {rule: ruleMatch, matches: [filerec.file]};
              });
              return acc;
            }, {});
          return Object.keys(ruleMatches).map(ruleID => ruleMatches[ruleID]);
        });

    });
}

export function getFileList(path, logger = null) {
  return readDirRecursive(path, logger)
    .catch(err => {
      if (logger)
        logger(2, `WARN: Error reading webroot directory: ${err.toString()}`);
      return [];
    });
}

function readDirRecursive(basePath, logger = null) {
  const list = [];
  return fs.readdirAsync(basePath)
    .then(res => {
      return Promise.all(res.map(x => {
        const filePath = path.join(basePath, x);
        return fs.statAsync(filePath).then(stat => ({file: filePath, stat}));
      }));
    })
    .then(res => {
      res.filter(x => !x.stat.isDirectory()).forEach(x => list.push(x.file));
      return Promise.all(res.filter(x => x.stat.isDirectory()).map(x => readDirRecursive(x.file, logger)))
        .then(lists => lists.forEach(x => x.forEach(y => list.push(y))));
    })
    .then(() => {
      return list;
    })
    .catch(err => {
      if (logger)
        logger(1, `ERROR: unable to process ${basePath}: ${err.toString()}`);
      return list;
    });
}

function processSingleRule_path(file, rule) {
  if (!rule.tests || !rule.tests.path)
    return Promise.resolve(false);

  if (typeof rule.tests.path === 'function')
  {
    const testResult = rule.tests.path(file.toString());
    if (testResult.then)
      return testResult;
    else
      return Promise.resolve(testResult);
  }

  return Promise.resolve(!!file.toString().match(rule.tests.path));
}

function processSingleRule_content(file, rule, contents) {
  if (!rule.tests || !rule.tests.content)
    return Promise.resolve(false);

  if (typeof rule.tests.content === 'function')
  {
    const testResult = rule.tests.content(contents, file);
    if (testResult.then)
      return testResult;
    else
      return Promise.resolve(testResult);
  }

  return Promise.resolve(!!contents.match(rule.tests.content));
}

export function processRulesOnFile(file, rules, exceptions, logger = null) {
  let filteredRules = rules;

  // Remove rules that list this file as excluded if possible
  if (exceptions && exceptions.length > 0)
    filteredRules = rules.filter(r => exceptions.filter(ex => ex === r.id).length === 0);

  // Process all path tests first
  return Promise.all(
    filteredRules.map(r => processSingleRule_path(file, r).then(res => ({rule: r, matched: res})))
  )

    // Check to see if any of the files have path checks that matched for rules
    // that have a content check. For those that do, read the contents and
    // process all of the rule content checks too.
    .then(res => {
      const contentChecks = res
        .filter(x => x.matched)
        .filter(x => x.rule.tests && x.rule.tests.content !== null)
        .map(x => x.rule);
      if (contentChecks.length > 0)
      {
        return fs.readFileAsync(file)
          .then(content => {
            const content_str = content.toString();
            return Promise.all(
              res.map(pathResult =>
                pathResult.matched && pathResult.rule.tests.content !== null ?
                  processSingleRule_content(file, pathResult.rule, content_str)
                    .then(contentRes => ({rule: pathResult.rule, matched: contentRes})) :
                  pathResult
              )
            );
          })
          .catch(err => {
            if (logger)
              logger(2, `WARN: unable to read file: ${err.toString()}`);
            return res.map(pathResult => ({
              rule: pathResult.rule,
              matched: pathResult.rule.tests.content !== null ? false : pathResult.matched,
            }));
          });
      }
      else
        return res;
    })

    .then(res => ({
      file,
      matches: res.filter(x => x.matched).map(x => x.rule),
    }));
}

export function formatResult(res) {
  return res.matches.length > 0 ?
    `${res.rule.deprecated ? 'DEPRECATED RULE:' : 'Rule:'} ${res.rule.name} (${res.matches.length} matches)\n` +
      (res.rule.urls && res.rule.urls.length > 0 ? `References: ${res.rule.urls.join(', ')}\n` : '') +
      res.matches.map(m => ' - ' + m).join('\n') + '\n'
    : null;
}

export function getFileHash(file) {
  const hashName = 'sha256';
  let   hash     = null;

  if (!crypto)
    return Promise.reject(new Error('Crypto support not available in this version of Node.js'));

  try {
    hash = crypto.createHash(hashName);
  } catch (err) {
    return Promise.reject(new Error(`${hashName} hash is unavailable in this version of Node.js`));
  }

  return fs.readFileAsync(file)
    .then(res => hash.update(res).digest('hex'));
}

export function getTreeSnapshot(settings) {
  if (!settings.logger)
    settings.logger = (level, msg) => msg;

  if (!settings.webroot)
    return Promise.resolve([]);

  return getFileList(settings.webroot, settings.logger)
    .then(fileList => {
      settings.logger(4, `Processing ${fileList.length} files...`);
      return Promise.all(
        fileList.map(f =>
          getFileHash(f)
            .then(hash => ({
              file: settings.relative ? f.replace(path.join(settings.webroot, ' ').trim(), '') : f,
              hash
            }))
        )
      );
    });
}