Automatic generation of Magento (1 and 2) whitelists

Author: jeroenvermeulen

php-malware-finder/utils/generate_whitelist.py

@@ -24,7 +24,7 @@
 output_list = list()
 
 for curdir, dirnames, filenames in os.walk(sys.argv[2]):
-    for filename in fnmatch.filter(filenames, '*.ph*'):
+    for filename in filenames:
         fname = os.path.join(curdir, filename)
         if 0 < os.stat(fname).st_size < 5 * 1024 * 1024:
             matches = rules.match(fname, fast=True)
@@ -36,7 +36,6 @@
 
 if output_list:
     output_rule = 'import "hash"\n\nrule %s\n{\n\tcondition:\n\t\t/* %s */\n\t\t' % (sys.argv[1].split(' ')[0], sys.argv[1])
-    output_list.append(output_list.pop().replace(' or ', '    '))
     output_rule += '\n\t\t'.join(output_list)
-    output_rule += '\n}'
+    output_rule += '\n\t\tfalse\n}'
     print(output_rule)

php-malware-finder/utils/magento1_whitelist.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+# Quit script if something goes wrong
+set  -o errexit  -o nounset  -o pipefail;
+
+SCRIPTDIR="$( dirname "$(readlink -f "$0")" )";
+OUTFILE="${SCRIPTDIR}/../whitelists/magento1ce.yar";
+TMPFILE="${OUTFILE}.new";
+
+# First empty the target whitelist so we can completely generate a new one
+cat <<EOF >"${OUTFILE}";
+private rule Magento1Ce : ECommerce
+{
+	condition:
+		false
+}
+EOF
+
+# Create a temporary directory and make sure it is empty
+GENTEMPDIR="$( mktemp -d --suffix="_gen_whitelist_m1" )";
+
+# Add header to whitelist tempfile
+cat <<EOF | tee "${TMPFILE}";
+private rule Magento1Ce : ECommerce
+{
+	condition:
+EOF
+
+# Fetch tags (releases) from Github repo
+TAGS=$( git ls-remote --tags https://github.com/OpenMage/magento-mirror.git | cut -d '/' -f3 | grep -P '^[\d\.]+$' );
+
+# Foreach tag (release)
+while read -r TAG; do
+    # Download tarball of release
+    wget "https://github.com/OpenMage/magento-mirror/archive/${TAG}.tar.gz" -O "${GENTEMPDIR}/${TAG}.tgz";
+    # Unpack tarball
+    tar -C "${GENTEMPDIR}" -xpzf "${GENTEMPDIR}/${TAG}.tgz";
+    # Add version comment to whitelist tempfile
+    echo "		/* Magento CE ${TAG} */" | tee -a "${TMPFILE}";
+    # Generate whitelist for version, add output to whitelist tempfile
+    ${SCRIPTDIR}/generate_whitelist.py "Magento CE ${TAG}" "${GENTEMPDIR}/magento-mirror-${TAG}" | grep 'hash.sha1' | sed "s|// ${GENTEMPDIR}/magento-mirror-${TAG}/|// |" | tee -a "${TMPFILE}";
+    # Add white line, with indent
+    echo "		" | tee -a "${TMPFILE}";
+done <<< "${TAGS}";
+
+# Add footer to whitelist tempfile
+cat <<EOF | tee -a "${TMPFILE}";
+		false
+}
+EOF
+
+# Copy temporary file to target whitelist while removing duplicate lines except empty ones
+cat "${TMPFILE}" | awk 'match($0,/^\s*$/)||!seen[$0]++' > "${OUTFILE}";
+
+# Clean up
+rm "${TMPFILE}";
+rm -rf "${GENTEMPDIR}";

php-malware-finder/utils/magento2_whitelist.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+# Quit script if something goes wrong
+set  -o errexit  -o nounset  -o pipefail;
+
+SCRIPTDIR="$( dirname "$(readlink -f "$0")" )";
+OUTFILE="${SCRIPTDIR}/../whitelists/magento2.yar";
+TMPFILE="${OUTFILE}.new";
+
+# First empty the target whitelist so we can completely generate a new one
+cat <<EOF >"${OUTFILE}";
+private rule Magento2 : ECommerce
+{
+	condition:
+		false
+}
+EOF
+
+# Create a temporary directory and make sure it is empty
+GENTEMPDIR="$( mktemp -d --suffix="_gen_whitelist_m2" )";
+
+# Composer access tokens
+if [ ! -f "${HOME}/.composer/auth.json" ]; then
+    echo -e "\nYou have no '.composer/auth.json' in your home dir. We will create it from a template and open an editor.";
+    echo -e "Press [Enter] to continue. Press Ctrl-C if you wish to leave.";
+    read;
+    mkdir -p "${HOME}/.composer";
+    cat <<EOF >"${HOME}/.composer/auth.json"    
+{
+  "INFO_GITHUB": "==== GET TOKEN: https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/ ====",
+  "github-oauth": {
+    "github.com": "---github-token-goes-here---"
+  },
+  "INFO_MAGENTO": "==== GET TOKEN: https://devdocs.magento.com/guides/v2.0/install-gde/prereq/connect-auth.html ====",
+  "http-basic": {
+    "repo.magento.com": {
+      "username": "---public-key-goes-here---",
+      "password": "---private-key-goes-here---"
+    }
+  }
+}
+EOF
+    editor "${HOME}/.composer/auth.json";
+fi
+
+# Add header to whitelist tempfile
+cat <<EOF | tee "${TMPFILE}";
+private rule Magento2 : ECommerce
+{
+	condition:
+EOF
+
+# Fetch tags (releases) from Github repo
+TAGS=$( git ls-remote --tags https://github.com/magento/magento2.git | cut -d '/' -f3 | grep -P '^[\d\.]+$' | sort -V );
+
+# Foreach tag (release)
+while read -r TAG; do
+    # Download tarball of release
+    wget "https://github.com/magento/magento2/archive/${TAG}.tar.gz" -O "${GENTEMPDIR}/${TAG}.tgz";
+    # Unpack tarball
+    tar -C "${GENTEMPDIR}" -xpzf "${GENTEMPDIR}/${TAG}.tgz";
+    # Run 'composer install' inside unpacked release
+    SOURCEDIR="${GENTEMPDIR}/magento2-${TAG}";
+    composer --working-dir="${SOURCEDIR}" -- install;
+    # Add version comment to whitelist
+    echo "		/* Magento2 ${TAG} */" | tee -a "${TMPFILE}";
+    # Generate whitelist for version, add output to whitelist tempfile
+    ${SCRIPTDIR}/generate_whitelist.py "Magento2 ${TAG}" "${SOURCEDIR}" | grep 'hash.sha1' | sed "s|// ${SOURCEDIR}/|// |" | tee -a "${TMPFILE}";
+    # Add white line, with indent
+    echo "		" | tee -a "${TMPFILE}";
+done <<< "${TAGS}";
+
+# Add footer to whitelist tempfile
+cat <<EOF | tee -a "${TMPFILE}";
+		false
+}
+EOF
+
+# Copy temporary file to target whitelist while removing duplicate lines except empty ones
+cat "${TMPFILE}" | awk 'match($0,/^\s*$/)||!seen[$0]++' > "${OUTFILE}";
+
+# Clean up
+rm "${TMPFILE}";
+rm -rf "${GENTEMPDIR}";

php-malware-finder/whitelist.yar

@@ -8,6 +8,7 @@ include "whitelists/drupal.yar"
 include "whitelists/wordpress.yar"
 include "whitelists/symfony.yar"
 include "whitelists/phpmyadmin.yar"
+include "whitelists/magento1ce.yar"
 include "whitelists/magento2.yar"
 include "whitelists/prestashop.yar"
 include "whitelists/custom.yar"
@@ -116,6 +117,7 @@ private rule IsWhitelisted
         Wordpress or
         Prestashop or
         Magento or
+        Magento1Ce or
         Magento2 or
         Drupal or
         Roundcube or