Add the moz_ingest integration tests to CI

build/run.sh

@@ -8,8 +8,9 @@
 
 set -e
 
-hindsight_dir="$PWD"
-lsb_dir="$(dirname "$hindsight_dir")/lua_sandbox"
+root_dir="$PWD"
+lsb_dir="$(dirname "$root_dir")/lua_sandbox"
+hs_dir="$(dirname "$root_dir")/hindsight"
 
 if [ ! -d "$lsb_dir" ]; then
     (   set -x;
@@ -18,6 +19,13 @@ if [ ! -d "$lsb_dir" ]; then
     )
 fi
 
+if [ ! -d "$hs_dir" ]; then
+    (   set -x;
+        git clone https://github.com/mozilla-services/hindsight "$hs_dir"
+        git rev-parse HEAD
+    )
+fi
+
 . "$lsb_dir/build/functions.sh"
 
 if [ "$1" != "build" -o $# -ge 2 ]; then
@@ -26,22 +34,38 @@ if [ "$1" != "build" -o $# -ge 2 ]; then
 fi
 
 build_lua_sandbox_extensions() {
+    if [[ "$DISTRO" =~ ^(centos:7|fedora:latest|ubuntu:latest) ]]; then
+        integration_tests="true"
+    fi
+
+    echo "+cd $hs_dir"
+    cd "$hs_dir"
+    build
+    if [ "$DISTRO" = "fedora:latest" ]; then
+        (   set -x;
+            $as_root yum install -y lua
+        )
+    fi
+    install_packages_from_dir ./release
+    echo "+cd $root_dir"
+    cd "$root_dir"
+
     local packages
     local cmake_args
     local ext
     packages="git c++-compiler"
     cmake_args=""
-    # todo add support for aws geoip jose kafka parquet snappy systemd
+    # todo add support for aws geoip jose kafka parquet snappy systemd moz_pioneer moz_security
     for ext in bloom_filter circular_buffer cjson compat cuckoo_filter \
-            elasticsearch heka hyperloglog lfs lpeg lsb moz_ingest maxminddb \
-            moz_pioneer moz_security moz_telemetry openssl postgres rjson sax \
+            elasticsearch heka hyperloglog lfs lpeg lsb maxminddb moz_ingest \
+            moz_telemetry openssl postgres rjson sax \
             socket ssl struct syslog zlib; do
         case "$ext" in
             maxminddb)
-                if [[ "$DISTRO" =~ ^(centos|fedora|ubuntu:12.04|ubuntu:latest) ]]; then
+                if [ "$integration_tests" = "true" ]; then
                     if [ "$CPACK_GENERATOR" = "DEB" ]; then
                         (   set -x;
-                            $as_root apt-get install -y software-properties-common python-software-properties
+                            $as_root apt-get install -y software-properties-common
                             $as_root add-apt-repository -y ppa:maxmind/ppa
                         )
                         packages="$packages libmaxminddb-dev libmaxminddb0"
@@ -64,8 +88,10 @@ build_lua_sandbox_extensions() {
             postgres)
                 if [ "$CPACK_GENERATOR" = "DEB" ]; then
                     packages="$packages libpq-dev postgresql-server-dev-all"
+                    cmake_args="$cmake_args -DEXT_$ext=false"
                 elif [ "$CPACK_GENERATOR" = "RPM" ]; then
                     packages="$packages postgresql-devel"
+                    cmake_args="$cmake_args -DEXT_$ext=false"
                 else
                     cmake_args="$cmake_args -DEXT_$ext=false"
                 fi
@@ -96,8 +122,10 @@ build_lua_sandbox_extensions() {
             zlib)
                 if [ "$CPACK_GENERATOR" = "DEB" ]; then
                     packages="$packages zlib1g-dev"
+                    cmake_args="$cmake_args -DEXT_$ext=true"
                 elif [ "$CPACK_GENERATOR" = "RPM" ]; then
                     packages="$packages zlib-devel"
+                    cmake_args="$cmake_args -DEXT_$ext=true"
                 else
                     cmake_args="$cmake_args -DEXT_$ext=false"
                 fi
@@ -123,6 +151,14 @@ build_lua_sandbox_extensions() {
         ctest -V
         make packages
     )
+
+    if [ "$integration_tests" = "true" ]; then
+        (   set -x
+            cd release
+            install_packages_from_dir .
+            ctest -V -C integration
+        )
+    fi
 }
 
 build_function="build_lua_sandbox_extensions"

moz_ingest/CMakeLists.txt

@@ -9,7 +9,12 @@ set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-rjson (>= 1.1.1), ${PACKAGE_
 string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS})
 include(sandbox_module)
 
-add_test(NAME ${PROJECT_NAME}_hindsight
-  COMMAND run.sh
-  CONFIGURATIONS hindsight
-  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/hindsight)
+add_test(NAME ${PROJECT_NAME}_decoder_common
+  COMMAND ../run.sh
+  CONFIGURATIONS integration
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/integration/decoder_common)
+
+add_test(NAME ${PROJECT_NAME}_decoder_json
+  COMMAND ../run.sh
+  CONFIGURATIONS integration
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/integration/decoder_json)

moz_ingest/tests/hindsight/run/analysis/verify_decoder.lua → moz_ingest/tests/integration/decoder_common/run/analysis/verify_decoder.lua

@@ -26,7 +26,7 @@ local messages = {
         docType = "foobar",
         DecodeErrorType = "skipped",
         DecodeError = "no sub decoder",
-        geoCity = "San Francisco",
+        geoCity = "Milton",
         geoCountry = "US",
         docVersion = 1
         }
@@ -35,7 +35,7 @@ local messages = {
         documentId = "0055FAC4-8A1A-4FCA-B380-EBFDC8571A01",
         docType = "foobar",
         duplicateDelta = 0,
-        geoCity = "San Francisco",
+        geoCity = "Milton",
         geoCountry = "US",
         docVersion = 1
         }
@@ -57,9 +57,9 @@ local messages = {
         docType = "widget",
         DecodeErrorType = "skipped",
         DecodeError = "no sub decoder",
-        geoCity = "New York",
+        geoCity = "Milton",
         geoCountry = "US",
-        geoISP = "Time Warner Cable",
+        geoISP = "Century Link",
         docVersion = 99
         }
     },

moz_ingest/tests/hindsight/run/input/generate_data.lua → moz_ingest/tests/integration/decoder_common/run/input/generate_data.lua

@@ -13,7 +13,7 @@ local msg = {
     Type   = "common.raw",
     Hostname = "example.com",
     Fields = {
-        remote_addr = "192.30.255.112",
+        remote_addr = "216.160.83.56",
         uri         = nil,
     }
 }
@@ -41,7 +41,7 @@ function process_message()
 
     -- override remote_addr with X-Forwarded-For
     msg.Fields.uri = "/submit/common/widget/99/0055FAC4-8A1A-4FCA-B380-EBFDC8571A03"
-    msg.Fields["X-Forwarded-For"] = "127.0.0.1, 72.229.28.185"
+    msg.Fields["X-Forwarded-For"] = "127.0.0.1, 216.160.83.56"
     inject_message(msg)
 
     -- valid

moz_ingest/tests/hindsight/run/input/test_decoder.cfg → moz_ingest/tests/integration/decoder_common/run/input/test_decoder.cfg

@@ -7,9 +7,9 @@ decoders_moz_ingest_common = {
     error_on_missing_sub_decoder = true,
 
     -- String used to specify GeoIP city database location on disk.
-    city_db_file = "/usr/share/geoip/GeoIP2-City.mmdb", -- optional, if not specified no geoip lookup is performed
+    city_db_file = "../../../../maxminddb/tests/GeoIP2-City-Test.mmdb", -- optional, if not specified no geoip lookup is performed
 
-    isp_db_file = "/usr/share/geoip/GeoIP2-ISP.mmdb", -- optional
+    isp_db_file = "../../../../maxminddb/tests/GeoIP2-ISP-Test.mmdb", -- optional
     isp_docTypes = {widget = true}, -- docTypes to perform ISP geoip lookups on, must be set if isp_db_file is defined
 
     -- number of items in the de-duping cuckoo filter

moz_ingest/tests/hindsight/run/analysis/verify_json_decoder.cfg → moz_ingest/tests/integration/decoder_json/run/analysis/verify_decoder.cfg

@@ -1,3 +1,3 @@
-filename = "verify_json_decoder.lua"
+filename = "verify_decoder.lua"
 message_matcher = "Logger != 'moz_ingest' && Logger != 'common'"
 ticker_interval = 0

moz_ingest/tests/hindsight/run/analysis/verify_json_decoder.lua → moz_ingest/tests/integration/decoder_json/run/analysis/verify_decoder.lua

@@ -12,7 +12,7 @@ local messages = {
     {Logger = "foo", Type = "validated", Hostname = "example.com", Fields = {
         docVersion = 1,
         docType = "bar",
-        geoCity = "New York",
+        geoCity = "Milton",
         geoCountry = "US",
         documentId = "0055FAC4-8A1A-4FCA-B380-EBFDC8571A01",
         submission = [[{"exampleString":"string one"}]],
@@ -27,7 +27,7 @@ local messages = {
         docType = "bar",
         DecodeErrorType = "json",
         DecodeError = "invalid submission: failed to parse offset:0 Invalid value.",
-        geoCity = "New York",
+        geoCity = "Milton",
         geoCountry = "US",
         docVersion = 1,
         content = ""
@@ -39,7 +39,7 @@ local messages = {
         docType = "bar",
         DecodeErrorType = "json",
         DecodeError = "namespace: foo schema: bar version: 1 error: SchemaURI: # Keyword: required DocumentURI: #",
-        geoCity = "New York",
+        geoCity = "Milton",
         geoCountry = "US",
         docVersion = 1,
         content = [[{"xString":"string one"}]]
@@ -51,7 +51,7 @@ local messages = {
         docType = "bar",
         DecodeErrorType = "json",
         DecodeError = "namespace: bar schema: bar version: 1 error: schema not found",
-        geoCity = "New York",
+        geoCity = "Milton",
         geoCountry = "US",
         docVersion = 1,
         content = "{}"

moz_ingest/tests/integration/decoder_json/run/input/generate_data.lua

@@ -0,0 +1,44 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this
+-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+--[[
+# Generates test data for moz_ingest_common decoder
+--]]
+
+require "string"
+
+local msg = {
+    Logger = "moz_ingest",
+    Type   = "json.raw",
+    Hostname = "example.com",
+    Fields = {
+        remote_addr = "216.160.83.56",
+        uri         = nil,
+    }
+}
+
+function process_message()
+    -- valid
+    msg.Fields.uri = "/submit/foo/bar/1/0055FAC4-8A1A-4FCA-B380-EBFDC8571A01"
+    msg.Fields.content = [[{"exampleString":"string one"}]]
+    msg.Fields["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0"
+    inject_message(msg)
+
+    -- fails parsing
+    msg.Fields.uri = "/submit/foo/bar/1/0055FAC4-8A1A-4FCA-B380-EBFDC8571A02"
+    msg.Fields.content = ""
+    inject_message(msg)
+
+    -- fails validation
+    msg.Fields.uri = "/submit/foo/bar/1/0055FAC4-8A1A-4FCA-B380-EBFDC8571A03"
+    msg.Fields.content = [[{"xString":"string one"}]]
+    inject_message(msg)
+
+    -- fails schema lookup
+    msg.Fields.uri = "/submit/bar/bar/1/0055FAC4-8A1A-4FCA-B380-EBFDC8571A01"
+    msg.Fields.content = "{}"
+    inject_message(msg)
+
+    return 0
+end

moz_ingest/tests/integration/decoder_json/run/input/generate_data.off

@@ -0,0 +1 @@
+filename = "generate_data.lua"

moz_ingest/tests/hindsight/run/input/test_json_decoder.cfg → moz_ingest/tests/integration/decoder_json/run/input/test_decoder.cfg

@@ -1,4 +1,4 @@
-filename = "test_json_decoder.lua"
+filename = "test_decoder.lua"
 
 decoders_moz_ingest_common = {
     sub_decoders = {
@@ -7,7 +7,7 @@ decoders_moz_ingest_common = {
     error_on_missing_sub_decoder = true,
 
     -- String used to specify GeoIP city database location on disk.
-    city_db_file = "/usr/share/geoip/GeoIP2-City.mmdb", -- optional, if not specified no geoip lookup is performed
+    city_db_file = "../../../../maxminddb/tests/GeoIP2-City-Test.mmdb", -- optional, if not specified no geoip lookup is performed
 
     -- number of items in the de-duping cuckoo filter
     cf_items = 1000, -- optional, if not provided de-duping is disabled

moz_ingest/tests/hindsight/run/input/test_json_decoder.lua → moz_ingest/tests/integration/decoder_json/run/input/test_decoder.lua

@@ -15,7 +15,7 @@ local hsr = create_stream_reader(read_config("Logger"))
 local is_running = is_running
 
 function process_message()
-    fh = assert(io.open("json.hpb", "rb")) -- closed on plugin shutdown
+    fh = assert(io.open("input.hpb", "rb")) -- closed on plugin shutdown
 
     local found, bytes, read
     local cnt = 0

moz_ingest/tests/hindsight/hindsight.cfg → moz_ingest/tests/integration/hindsight.cfg

@@ -1,6 +1,6 @@
 output_path             = "output"
 output_size             =  1024 * 1024 * 1024
-sandbox_load_path       = nil 
+sandbox_load_path       = nil
 sandbox_run_path        = "run"
 analysis_threads        = 1
 analysis_lua_path       = "/usr/lib/luasandbox/modules/?.lua"

moz_ingest/tests/integration/run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+rm -rf output
+hindsight_cli ../hindsight.cfg 7