From 3c36141007ddb559396f4450d25789f148a6b512 Mon Sep 17 00:00:00 2001 From: rmerriman Date: Thu, 7 Apr 2016 14:45:38 -0500 Subject: [PATCH] METRON-102: Unit Tests for parsers and enrichment adapters --- .../enrichment}/EnrichmentHelper.java | 5 +- .../Metron-EnrichmentAdapters/pom.xml | 13 +- .../enrichment/adapters/jdbc/JdbcAdapter.java | 4 + .../adapters/cif/CIFHbaseAdapterTest.java | 225 ------------------ .../adapters/geo/GeoAdapterTest.java | 94 ++++++++ .../adapters/geo/GeoMysqlAdapterTest.java | 185 -------------- .../host/HostFromJSONListAdapterTest.java | 83 +++++++ .../HostFromPropertiesFileAdapterTest.java | 106 +++++++++ .../adapters/jdbc/MySqlConfigTest.java | 44 ++++ .../simplehbase/SimpleHBaseAdapterTest.java | 117 +++++++++ .../simplehbase/SimpleHBaseConfigTest.java | 42 ++++ .../threatintel/ThreatIntelAdapterTest.java | 148 ++++++++++++ .../threatintel/ThreatIntelConfigTest.java | 53 +++++ .../adapters/whois/WhoisHBaseAdapterTest.java | 166 ------------- .../parsing/parsers/GrokParserTest.java | 114 +++++++++ metron-streaming/Metron-Testing/pom.xml | 5 + .../EnrichmentIntegrationTest.java | 4 +- .../Metron-TestingUtilities/pom.xml | 73 ++++++ .../integration/util/mock/MockHTable.java | 0 metron-streaming/pom.xml | 1 + 20 files changed, 898 insertions(+), 584 deletions(-) rename metron-streaming/{Metron-Testing/src/main/java/org/apache/metron/integration/util => Metron-Common/src/main/java/org/apache/metron/hbase/converters/enrichment}/EnrichmentHelper.java (84%) delete mode 100644 metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapterTest.java create mode 100644 metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoAdapterTest.java delete mode 100644 metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoMysqlAdapterTest.java create mode 100644 metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java create mode 100644 metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java create mode 100644 metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfigTest.java create mode 100644 metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java create mode 100644 metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfigTest.java create mode 100644 metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java create mode 100644 metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfigTest.java delete mode 100644 metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/whois/WhoisHBaseAdapterTest.java create mode 100644 metron-streaming/Metron-MessageParsers/src/test/java/org/apache/metron/parsing/parsers/GrokParserTest.java create mode 100644 metron-streaming/Metron-TestingUtilities/pom.xml rename metron-streaming/{Metron-Testing => Metron-TestingUtilities}/src/main/java/org/apache/metron/integration/util/mock/MockHTable.java (100%) diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/EnrichmentHelper.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/enrichment/EnrichmentHelper.java similarity index 84% rename from metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/EnrichmentHelper.java rename to metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/enrichment/EnrichmentHelper.java index e06a2a5cb9..a3d1b66514 100644 --- a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/EnrichmentHelper.java +++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/enrichment/EnrichmentHelper.java @@ -15,13 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.integration.util; +package org.apache.metron.hbase.converters.enrichment; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; -import org.apache.metron.hbase.converters.enrichment.EnrichmentConverter; -import org.apache.metron.hbase.converters.enrichment.EnrichmentKey; -import org.apache.metron.hbase.converters.enrichment.EnrichmentValue; import org.apache.metron.reference.lookup.LookupKV; import java.io.IOException; diff --git a/metron-streaming/Metron-EnrichmentAdapters/pom.xml b/metron-streaming/Metron-EnrichmentAdapters/pom.xml index 7399ade522..2739c4e27c 100644 --- a/metron-streaming/Metron-EnrichmentAdapters/pom.xml +++ b/metron-streaming/Metron-EnrichmentAdapters/pom.xml @@ -112,12 +112,23 @@ junit ${global_junit_version} + + org.mockito + mockito-all + ${global_mockito_version} + test + commons-validator commons-validator 1.4.0 - + + org.apache.metron + Metron-TestingUtilities + 0.1BETA + test + diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java index b21044f48a..9233059571 100644 --- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java +++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java @@ -38,6 +38,10 @@ public abstract class JdbcAdapter implements EnrichmentAdapter, private JdbcConfig config; private String host; + public void setStatement(Statement statement) { + this.statement = statement; + } + public JdbcAdapter withJdbcConfig(JdbcConfig config) { this.config = config; this.host = config.getHost(); diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapterTest.java deleted file mode 100644 index 9e808eee4f..0000000000 --- a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapterTest.java +++ /dev/null @@ -1,225 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.enrichment.adapters.cif; - -import java.net.InetAddress; -import java.util.Properties; - -import org.apache.metron.enrichment.bolt.CacheKey; -import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; -import org.apache.metron.test.AbstractTestContext; -import org.junit.Assert; - - - /** - *
    - *
  • Title: CIFHbaseAdapterTest
    • - *
  • Description: Test Class for CIGFHbaseAdapter
    • - *
  • Created: Aug 7, 2014
    • - *
- * @version $Revision: 1.1 $ - */ -public class CIFHbaseAdapterTest extends AbstractTestContext { - - private static CIFHbaseAdapter cifHbaseAdapter=null; - - - - /** - * Constructs a new CIFHbaseAdapterTest instance. - * @param name - */ - - public CIFHbaseAdapterTest(String name) { - super(name); - - } - - - - /** - - * @throws java.lang.Exception - */ - protected static void setUpBeforeClass() throws Exception { - } - - /** - - * @throws java.lang.Exception - */ - protected static void tearDownAfterClass() throws Exception { - } - - /* - * (non-Javadoc) - * @see junit.framework.TestCase#setUp() - */ - - protected void setUp() throws Exception { - super.setUp(); - - Properties prop = super.getTestProperties(); - Assert.assertNotNull(prop); - - if(skipTests(this.getMode())){ - return;//skip tests - } - - String[] zk = prop.get("kafka.zk.list").toString().split(","); - - for(String z : zk) - { - InetAddress address = InetAddress.getByName(z); - boolean reachable = address.isReachable(100); - - if(!reachable) - { - this.setMode("local"); - //throw new Exception("Unable to reach zookeeper, skipping CIF adapter test"); - break; - } - - } - - if(skipTests(this.getMode())) - return;//skip tests - - System.out.println("kafka.zk.list ="+(String) prop.get("kafka.zk.list")); - System.out.println("kafka.zk.list ="+(String) prop.get("kafka.zk.port")); - System.out.println("kafka.zk.list ="+(String) prop.get("bolt.enrichment.cif.tablename")); - if(skipTests(this.getMode())){ - System.out.println("Local Mode Skipping tests !! "); - }else{ - cifHbaseAdapter=new CIFHbaseAdapter((String) prop.get("kafka.zk.list"), (String) prop.get("kafka.zk.port"),(String) prop.get("bolt.enrichment.cif.tablename")); - } - } - - /* - * (non-Javadoc) - * @see junit.framework.TestCase#tearDown() - */ - - protected void tearDown() throws Exception { - super.tearDown(); - cifHbaseAdapter=null; - } - - /** - * Test method for {@link EnrichmentAdapter#initializeAdapter()}. - */ - public void testInitializeAdapter() { - if(skipTests(this.getMode())){ - return;//skip tests - }else{ - Assert.assertTrue(cifHbaseAdapter.initializeAdapter()); - } - } - - /** - * Test method for {@link org.apache.metron.enrichment.adapters.cif.CIFHbaseAdapter#enrichByIP(java.lang.String)}. - */ - public void testEnrichByIP() { - if(skipTests(this.getMode())){ - return;//skip tests - }else{ - Assert.assertNull(cifHbaseAdapter.enrichByIP("11.1.1")); - } - } - - /** - * Test method for {@link org.apache.metron.enrichment.adapters.cif.CIFHbaseAdapter#enrichByDomain(java.lang.String)}. - */ - public void testEnrichByDomain() { - if(skipTests(this.getMode())){ - return;//skip tests - }else{ - Assert.assertNull(cifHbaseAdapter.enrichByIP("invaliddomain")); - } - } - - /** - * Test method for {@link org.apache.metron.enrichment.adapters.cif.CIFHbaseAdapter#enrichByEmail(java.lang.String)}. - */ - public void testEnrichByEmail() { - if(skipTests(this.getMode())){ - return;//skip tests - }else{ - Assert.assertNull(cifHbaseAdapter.enrichByIP("sample@invalid.com")); - } - } - - /** - * Test method for {@link org.apache.metron.enrichment.adapters.cif.CIFHbaseAdapter#CIFHbaseAdapter(java.lang.String, java.lang.String, java.lang.String)}. - */ - public void testCIFHbaseAdapter() { - if(skipTests(this.getMode())){ - return;//skip tests - }else{ - Assert.assertNotNull(cifHbaseAdapter); - } - } - - /** - * Test method for {@link org.apache.metron.enrichment.adapters.cif.CIFHbaseAdapter#enrich(CacheKey)}. - */ - public void testEnrich() { - if(skipTests(this.getMode())){ - return;//skip tests - }else{ - cifHbaseAdapter.initializeAdapter(); - Assert.assertNotNull(cifHbaseAdapter.enrich(new CacheKey("cif", "testinvalid.metadata", null))); - - Assert.assertNotNull(cifHbaseAdapter.enrich(new CacheKey("cif", "ivalid.ip", null))); - Assert.assertNotNull(cifHbaseAdapter.enrich(new CacheKey("cif", "1.1.1.10", null))); - } - } - - - /** - * Test method for {@link org.apache.metron.enrichment.adapters.cif.CIFHbaseAdapter#getCIFObject(java.lang.String)}. - */ - public void testGetCIFObject() { - if(skipTests(this.getMode())){ - return;//skip tests - }else{ - cifHbaseAdapter.initializeAdapter(); - Assert.assertNotNull(cifHbaseAdapter.getCIFObject("testkey")); - } - } - /** - * Returns the cifHbaseAdapter. - * @return the cifHbaseAdapter. - */ - - public static CIFHbaseAdapter getCifHbaseAdapter() { - return CIFHbaseAdapterTest.cifHbaseAdapter; - } - - /** - * Sets the cifHbaseAdapter. - * @param cifHbaseAdapter the cifHbaseAdapter. - */ - - public static void setCifHbaseAdapter(CIFHbaseAdapter cifHbaseAdapter) { - - CIFHbaseAdapterTest.cifHbaseAdapter = cifHbaseAdapter; - } - -} - diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoAdapterTest.java new file mode 100644 index 0000000000..ec90c49578 --- /dev/null +++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoAdapterTest.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.enrichment.adapters.geo; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.enrichment.bolt.CacheKey; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.sql.ResultSet; +import java.sql.Statement; + +import static org.mockito.Mockito.when; + +public class GeoAdapterTest { + + + private String ip = "72.163.4.161"; + + + /** + * { + * "locID":"1", + * "country":"test country", + * "city":"test city", + * "postalCode":"test zip", + * "latitude":"test latitude", + * "longitude":"test longitude", + * "dmaCode":"test dma", + * "location_point":"test longitude,test latitude" + * } + */ + @Multiline + private String expectedMessageString; + + private JSONObject expectedMessage; + + @Mock + Statement statetment; + @Mock + ResultSet resultSet, resultSet1; + + + @Before + public void setup() throws Exception { + JSONParser jsonParser = new JSONParser(); + expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString); + MockitoAnnotations.initMocks(this); + when(statetment.executeQuery("select IPTOLOCID(\"CacheKey{field='dummy', value='72.163.4.161'}\") as ANS")).thenReturn(resultSet); + when(statetment.executeQuery("select * from location where locID = 1")).thenReturn(resultSet1); + when(resultSet.next()).thenReturn(Boolean.TRUE, Boolean.FALSE); + when(resultSet.getString("ANS")).thenReturn("1"); + when(resultSet1.next()).thenReturn(Boolean.TRUE, Boolean.FALSE); + when(resultSet1.getString("locID")).thenReturn("1"); + when(resultSet1.getString("country")).thenReturn("test country"); + when(resultSet1.getString("city")).thenReturn("test city"); + when(resultSet1.getString("postalCode")).thenReturn("test zip"); + when(resultSet1.getString("latitude")).thenReturn("test latitude"); + when(resultSet1.getString("longitude")).thenReturn("test longitude"); + when(resultSet1.getString("dmaCode")).thenReturn("test dma"); + } + + + @Test + public void testEnrich() throws Exception { + GeoAdapter geo = new GeoAdapter(); + geo.setStatement(statetment); + JSONObject actualMessage = geo.enrich(new CacheKey("dummy", ip, null)); + Assert.assertNotNull(actualMessage.get("locID")); + Assert.assertEquals(expectedMessage, actualMessage); + } + +} + diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoMysqlAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoMysqlAdapterTest.java deleted file mode 100644 index cc3fb6fb89..0000000000 --- a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoMysqlAdapterTest.java +++ /dev/null @@ -1,185 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.enrichment.adapters.geo; - -import java.net.URL; -import java.util.Properties; - -import org.apache.metron.enrichment.adapters.jdbc.JdbcAdapter; -import org.apache.metron.enrichment.adapters.jdbc.MySqlConfig; -import org.apache.metron.enrichment.bolt.CacheKey; -import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; -import org.json.simple.JSONObject; - -import org.apache.metron.test.AbstractSchemaTest; -import org.junit.Assert; - - -/** - *
    - *
  • Title: GeoMySqlAdapterTest
    • - *
  • Description: Tests for GeoMySqlAdapter
    • - *
  • Created: Aug 25, 2014
    • - *
- * @version $Revision: 1.1 $ - */ -public class GeoMysqlAdapterTest extends AbstractSchemaTest { - - private static JdbcAdapter geoMySqlAdapter=null; - private static boolean connected=false; - - /** - * Constructs a new GeoMysqlAdapterTest instance. - * @param name - */ - - public GeoMysqlAdapterTest(String name) { - super(name); - } - - /** - - * @throws java.lang.Exception - */ - protected static void setUpBeforeClass() throws Exception { - } - - /** - - * @throws java.lang.Exception - */ - protected static void tearDownAfterClass() throws Exception { - } - - /* - * (non-Javadoc) - * @see junit.framework.TestCase#setUp() - */ - - protected void setUp() throws Exception { - super.setUp(); - Properties prop = super.getTestProperties(); - assertNotNull(prop); - System.out.println("username ="+(String)prop.get("mysql.username")); - if(skipTests(this.getMode())){ - System.out.println(getClass().getName()+" Skipping Tests !!Local Mode"); - return;//skip tests - }else{ - MySqlConfig mySqlConfig = new MySqlConfig(); - mySqlConfig.setHost((String)prop.get("mysql.ip")); - mySqlConfig.setPort(new Integer((String) prop.get("mysql.port"))); - mySqlConfig.setUsername((String)prop.get("mysql.username")); - mySqlConfig.setPassword((String)prop.get("mysql.password")); - mySqlConfig.setTable((String)prop.get("bolt.enrichment.geo.adapter.table")); - JdbcAdapter geoAdapter = new GeoAdapter().withJdbcConfig(mySqlConfig); - GeoMysqlAdapterTest.setGeoMySqlAdapter(geoAdapter); - connected =geoMySqlAdapter.initializeAdapter(); - Assert.assertTrue(connected); - URL schema_url = getClass().getClassLoader().getResource( - "TestSchemas/GeoMySqlSchema.json"); - super.setSchemaJsonString(super.readSchemaFromFile(schema_url)); - } - } - - /* - * (non-Javadoc) - * @see junit.framework.TestCase#tearDown() - */ - - protected void tearDown() throws Exception { - super.tearDown(); - GeoMysqlAdapterTest.setGeoMySqlAdapter(null); - } - - /** - * Test method for {@link org.apache.metron.enrichment.adapters.geo.GeoAdapter#enrich(java.lang.String)}. - */ - public void testEnrich() { - if(skipTests(this.getMode())){ - return;//skip tests - }else{ - - try { - JSONObject json = geoMySqlAdapter.enrich(new CacheKey("dummy", "72.163.4.161", null)); - - //assert Geo Response is not null - System.out.println("json ="+json); - assertNotNull(json); - - Assert.assertEquals(true, super.validateJsonData(super.getSchemaJsonString(), json.toString())); - //assert LocId is not null - assertNotNull(json.get("locID")); - - //assert right LocId is being returned - Assert.assertEquals("4522",json.get("locID")); - } catch (Exception e) { - e.printStackTrace(); - Assert.assertTrue(false); - } - } - } - - /** - * Test method for {@link EnrichmentAdapter#initializeAdapter()}. - */ - public void testInitializeAdapter() { - if(skipTests(this.getMode())){ - return;//skip tests - }else{ - boolean connected =geoMySqlAdapter.initializeAdapter(); - Assert.assertTrue(connected); - } - } - - /** - * Test method for - * - * - * - * - * - * {@link org.apache.metron.enrichment.adapters.geo.GeoAdapter}. - */ - public void testGeoMysqlAdapter() { - if(skipTests(this.getMode())){ - return;//skip tests - }else{ - Assert.assertTrue(connected); - } - } - - /** - * Returns the geoMySqlAdapter. - * @return the geoMySqlAdapter. - */ - - public static JdbcAdapter getGeoMySqlAdapter() { - return geoMySqlAdapter; - } - - /** - * Sets the geoMySqlAdapter. - * @param geoMySqlAdapter the geoMySqlAdapter. - */ - - public static void setGeoMySqlAdapter(JdbcAdapter geoMySqlAdapter) { - - GeoMysqlAdapterTest.geoMySqlAdapter = geoMySqlAdapter; - } -} - diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java new file mode 100644 index 0000000000..c448f1e45c --- /dev/null +++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.enrichment.adapters.host; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.enrichment.bolt.CacheKey; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class HostFromJSONListAdapterTest { + + + /** + * [ + * {"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"}, + * {"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"}, + * {"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}, + * {"ip":"10.0.2.15", "local":"YES", "type":"printer", "asset_value" : "important"} + * ] + */ + @Multiline + private String expectedKnownHostsString; + + /** + * { + * "known_info.local":"YES", + * "known_info.type":"printer", + * "known_info.asset_value" : "important" + * } + */ + @Multiline + private String expectedMessageString; + + private JSONObject expectedMessage; + private String ip = "10.0.2.15"; + private String ip1 = "10.0.22.22"; + + + @Before + public void parseJSON() throws ParseException { + JSONParser jsonParser = new JSONParser(); + expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString); + } + + @Test + public void testEnrich() throws Exception { + HostFromJSONListAdapter hja = new HostFromJSONListAdapter(expectedKnownHostsString); + JSONObject actualMessage = hja.enrich(new CacheKey("dummy", ip, null)); + Assert.assertNotNull(actualMessage); + Assert.assertEquals(expectedMessage, actualMessage); + actualMessage = hja.enrich(new CacheKey("dummy", ip1, null)); + JSONObject emptyJson = new JSONObject(); + Assert.assertEquals(emptyJson, actualMessage); + } + + + @Test + public void testInitializeAdapter() throws Exception { + HostFromJSONListAdapter hja = new HostFromJSONListAdapter(expectedKnownHostsString); + Assert.assertTrue(hja.initializeAdapter()); + } + +} + diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java new file mode 100644 index 0000000000..ea5cabe79e --- /dev/null +++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.enrichment.adapters.host; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.enrichment.bolt.CacheKey; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + + +public class HostFromPropertiesFileAdapterTest { + + /** + * [ + * {"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"}, + * {"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"}, + * {"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}, + * {"ip":"10.0.2.15", "local":"YES", "type":"printer", "asset_value" : "important"} + * ] + */ + @Multiline + private String expectedKnownHostsString; + + /** + * { + * "known_info": + * {"asset_value":"important", + * "type":"printer","local":"YES" + * } + * } + */ + @Multiline + private String expectedMessageString; + + private JSONObject expectedMessage; + private String ip = "10.0.2.15"; + private String ip1 = "10.0.22.22"; + + @Before + public void parseJSON() throws ParseException { + JSONParser jsonParser = new JSONParser(); + expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString); + } + + @Test + public void testEnrich() throws Exception { + Map mapKnownHosts = new HashMap<>(); + JSONArray jsonArray = (JSONArray) JSONValue.parse(expectedKnownHostsString); + Iterator jsonArrayIterator = jsonArray.iterator(); + while(jsonArrayIterator.hasNext()) { + JSONObject jsonObject = (JSONObject) jsonArrayIterator.next(); + String host = (String) jsonObject.remove("ip"); + mapKnownHosts.put(host, jsonObject); + } + HostFromPropertiesFileAdapter hfa = new HostFromPropertiesFileAdapter(mapKnownHosts); + JSONObject actualMessage = hfa.enrich(new CacheKey("dummy", ip, null)); + Assert.assertNotNull(actualMessage); + Assert.assertEquals(expectedMessage, actualMessage); + actualMessage = hfa.enrich(new CacheKey("dummy", ip1, null)); + JSONObject emptyJson = new JSONObject(); + Assert.assertEquals(emptyJson, actualMessage); + } + + + @Test + public void testInitializeAdapter() throws Exception { + Map mapKnownHosts = new HashMap<>(); + HostFromPropertiesFileAdapter hfa = new HostFromPropertiesFileAdapter(mapKnownHosts); + Assert.assertFalse(hfa.initializeAdapter()); + JSONArray jsonArray = (JSONArray) JSONValue.parse(expectedKnownHostsString); + Iterator jsonArrayIterator = jsonArray.iterator(); + while(jsonArrayIterator.hasNext()) { + JSONObject jsonObject = (JSONObject) jsonArrayIterator.next(); + String host = (String) jsonObject.remove("ip"); + mapKnownHosts.put(host, jsonObject); + } + hfa = new HostFromPropertiesFileAdapter(mapKnownHosts); + Assert.assertTrue(hfa.initializeAdapter()); + } + +} + diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfigTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfigTest.java new file mode 100644 index 0000000000..2ae2ff0a7c --- /dev/null +++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfigTest.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.enrichment.adapters.jdbc; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class MySqlConfigTest { + + private String sampleURL = "jdbc:mysql://10.22.0.214:3306/GEO?user=root&password=hadoop123"; + private MySqlConfig conn; + + @Before + public void setupJdbc() { + conn = new MySqlConfig(); + conn.setHost("10.22.0.214"); + conn.setPort(3306); + conn.setTable("GEO"); + conn.setUsername("root"); + conn.setPassword("hadoop123"); + } + + @Test + public void testGetJdbcUrl() throws Exception { + Assert.assertEquals(sampleURL, conn.getJdbcUrl()); + } + +} diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java new file mode 100644 index 0000000000..a652e95142 --- /dev/null +++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.enrichment.adapters.simplehbase; + + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.domain.SensorEnrichmentConfig; +import org.apache.metron.enrichment.bolt.CacheKey; +import org.apache.metron.hbase.converters.enrichment.EnrichmentKey; +import org.apache.metron.hbase.converters.enrichment.EnrichmentValue; +import org.apache.metron.hbase.lookup.EnrichmentLookup; +import org.apache.metron.hbase.converters.enrichment.EnrichmentHelper; +import org.apache.metron.integration.util.mock.MockHTable; +import org.apache.metron.reference.lookup.LookupKV; +import org.apache.metron.reference.lookup.accesstracker.BloomAccessTracker; +import org.apache.metron.reference.lookup.accesstracker.PersistentAccessTracker; +import org.apache.metron.utils.JSONUtils; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +public class SimpleHBaseAdapterTest { + + private String cf = "cf"; + private String atTableName = "tracker"; + private final String hbaseTableName = "enrichments"; + private EnrichmentLookup lookup; + private static final String PLAYFUL_CLASSIFICATION_TYPE = "playful_classification"; + private static final Map PLAYFUL_ENRICHMENT = new HashMap() {{ + put("orientation", "north"); + }}; + + /** + * { + * "10.0.2.3.orientation":"north" + * } + */ + @Multiline + private String expectedMessageString; + + /** + * { + * "index": "bro", + * "batchSize": 5, + * "enrichmentFieldMap": { + * "geo": ["ip_dst_addr", "ip_src_addr"], + * "host": ["host"] + * }, + * "fieldToEnrichmentTypeMap": { + * "ip_dst_addr" : [ "10.0.2.3" ], + * "ip_src_addr" : [ "10.3.30.120" ] + * } + * } + */ + @Multiline + private String sourceConfigStr; + + private JSONObject expectedMessage; + + @Before + public void setup() throws Exception { + final MockHTable trackerTable = (MockHTable) MockHTable.Provider.addToCache(atTableName, cf); + final MockHTable hbaseTable = (MockHTable) MockHTable.Provider.addToCache(hbaseTableName, cf); + EnrichmentHelper.INSTANCE.load(hbaseTable, cf, new ArrayList>() {{ + add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3") + , new EnrichmentValue(PLAYFUL_ENRICHMENT) + ) + ); + }}); + BloomAccessTracker bat = new BloomAccessTracker(hbaseTableName, 100, 0.03); + PersistentAccessTracker pat = new PersistentAccessTracker(hbaseTableName, "0", trackerTable, cf, bat, 0L); + lookup = new EnrichmentLookup(hbaseTable, cf, pat); + JSONParser jsonParser = new JSONParser(); + expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString); + } + + @Test + public void testEnrich() throws Exception { + SimpleHBaseAdapter sha = new SimpleHBaseAdapter(); + sha.lookup = lookup; + SensorEnrichmentConfig broSc = JSONUtils.INSTANCE.load(sourceConfigStr, SensorEnrichmentConfig.class); + JSONObject actualMessage = sha.enrich(new CacheKey("test", "test", broSc)); + Assert.assertEquals(actualMessage, new JSONObject()); + actualMessage = sha.enrich(new CacheKey("ip_dst_addr", "10.0.2.3", broSc)); + Assert.assertNotNull(actualMessage); + Assert.assertEquals(expectedMessage, actualMessage); + } + + @Test(expected = Exception.class) + public void testInitializeAdapter() { + SimpleHBaseConfig config = new SimpleHBaseConfig(); + SimpleHBaseAdapter sha = new SimpleHBaseAdapter(config); + sha.initializeAdapter(); + } + +} diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfigTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfigTest.java new file mode 100644 index 0000000000..832a939932 --- /dev/null +++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfigTest.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.enrichment.adapters.simplehbase; + +import org.apache.metron.hbase.HTableProvider; +import org.apache.metron.hbase.TableProvider; +import org.junit.Assert; +import org.junit.Test; + +public class SimpleHBaseConfigTest { + + + private String cf ="cf"; + private String table = "threatintel"; + private TableProvider provider; + + @Test + public void test(){ + SimpleHBaseConfig shc = new SimpleHBaseConfig(); + shc.withHBaseCF(cf); + shc.withHBaseTable(table); + provider = new HTableProvider(); + Assert.assertEquals(cf, shc.getHBaseCF()); + Assert.assertEquals(table, shc.getHBaseTable()); + } + +} diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java new file mode 100644 index 0000000000..49fe7915cb --- /dev/null +++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.enrichment.adapters.threatintel; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.metron.domain.SensorEnrichmentConfig; +import org.apache.metron.enrichment.bolt.CacheKey; +import org.apache.metron.hbase.TableProvider; +import org.apache.metron.hbase.converters.enrichment.EnrichmentKey; +import org.apache.metron.hbase.converters.enrichment.EnrichmentValue; +import org.apache.metron.hbase.lookup.EnrichmentLookup; +import org.apache.metron.hbase.converters.enrichment.EnrichmentHelper; +import org.apache.metron.integration.util.mock.MockHTable; +import org.apache.metron.reference.lookup.LookupKV; +import org.apache.metron.reference.lookup.accesstracker.BloomAccessTracker; +import org.apache.metron.reference.lookup.accesstracker.PersistentAccessTracker; +import org.apache.metron.utils.JSONUtils; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; + + +public class ThreatIntelAdapterTest { + + public static class ExceptionProvider implements TableProvider { + + public ExceptionProvider() {}; + + @Override + public HTableInterface getTable(Configuration config, String tableName) throws IOException { + throw new IOException(); + } + } + + private String cf = "cf"; + private String atTableName = "tracker"; + private static final String MALICIOUS_IP_TYPE = "malicious_ip"; + private final String threatIntelTableName = "threat_intel"; + private EnrichmentLookup lookup; + + /** + * { + * "10.0.2.3":"alert" + * } + */ + @Multiline + private String expectedMessageString; + + /** + * { + * "index": "bro", + * "batchSize": 5, + * "enrichmentFieldMap": { + * "geo": ["ip_dst_addr", "ip_src_addr"], + * "host": ["host"] + * }, + * "threatIntelFieldMap": { + * "hbaseThreatIntel": ["ip_dst_addr", "ip_src_addr"] + * }, + * "fieldToThreatIntelTypeMap": { + * "ip_dst_addr" : [ "10.0.2.3" ], + * "ip_src_addr" : [ "malicious_ip" ] + * } + * } + */ + @Multiline + private static String sourceConfigStr; + + private JSONObject expectedMessage; + + @Before + public void setup() throws Exception { + + final MockHTable trackerTable = (MockHTable) MockHTable.Provider.addToCache(atTableName, cf); + final MockHTable threatIntelTable = (MockHTable) MockHTable.Provider.addToCache(threatIntelTableName, cf); + EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList>() {{ + add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3"), new EnrichmentValue(new HashMap()))); + }}); + + BloomAccessTracker bat = new BloomAccessTracker(threatIntelTableName, 100, 0.03); + PersistentAccessTracker pat = new PersistentAccessTracker(threatIntelTableName, "0", trackerTable, cf, bat, 0L); + lookup = new EnrichmentLookup(threatIntelTable, cf, pat); + JSONParser jsonParser = new JSONParser(); + expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString); + } + + + @Test + public void testEnrich() throws Exception { + ThreatIntelAdapter tia = new ThreatIntelAdapter(); + tia.lookup = lookup; + SensorEnrichmentConfig broSc = JSONUtils.INSTANCE.load(sourceConfigStr, SensorEnrichmentConfig.class); + JSONObject actualMessage = tia.enrich(new CacheKey("ip_dst_addr", "10.0.2.3", broSc)); + Assert.assertNotNull(actualMessage); + Assert.assertEquals(expectedMessage, actualMessage); + } + + @Test(expected = IllegalStateException.class) + public void testInitializeAdapter() { + + String cf = "cf"; + String table = "threatintel"; + String trackCf = "cf"; + String trackTable = "Track"; + double falsePositive = 0.03; + int expectedInsertion = 1; + long millionseconds = (long) 0.1; + + ThreatIntelConfig config = new ThreatIntelConfig(); + config.withHBaseCF(cf); + config.withHBaseTable(table); + config.withExpectedInsertions(expectedInsertion); + config.withFalsePositiveRate(falsePositive); + config.withMillisecondsBetweenPersists(millionseconds); + config.withTrackerHBaseCF(trackCf); + config.withTrackerHBaseTable(trackTable); + config.withProviderImpl(ExceptionProvider.class.getName()); + + ThreatIntelAdapter tia = new ThreatIntelAdapter(config); + tia.initializeAdapter(); + + } + + +} diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfigTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfigTest.java new file mode 100644 index 0000000000..58fd80353e --- /dev/null +++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfigTest.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.enrichment.adapters.threatintel; + +import org.junit.Assert; +import org.junit.Test; + +public class ThreatIntelConfigTest { + + private String cf = "cf"; + private String table = "threatintel"; + private String trackCf = "cf"; + private String trackTable = "Track"; + private double falsePositive = 0.03; + private int expectedInsertion = 1; + private long millionseconds = (long) 0.1; + + @Test + public void test() { + ThreatIntelConfig tic = new ThreatIntelConfig(); + tic.withHBaseCF(cf); + tic.withHBaseTable(table); + tic.withExpectedInsertions(expectedInsertion); + tic.withFalsePositiveRate(falsePositive); + tic.withMillisecondsBetweenPersists(millionseconds); + tic.withTrackerHBaseCF(trackCf); + tic.withTrackerHBaseTable(trackTable); + + Assert.assertEquals(cf, tic.getHBaseCF()); + Assert.assertEquals(table, tic.getHBaseTable()); + Assert.assertEquals(trackCf, tic.getTrackerHBaseCF()); + Assert.assertEquals(trackTable, tic.getTrackerHBaseTable()); + Assert.assertEquals(expectedInsertion, tic.getExpectedInsertions()); + Assert.assertEquals(millionseconds, tic.getMillisecondsBetweenPersists()); + + } + +} diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/whois/WhoisHBaseAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/whois/WhoisHBaseAdapterTest.java deleted file mode 100644 index 60dd6564a2..0000000000 --- a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/whois/WhoisHBaseAdapterTest.java +++ /dev/null @@ -1,166 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.enrichment.adapters.whois; - -import java.net.InetAddress; -import java.util.Properties; - -import org.apache.metron.enrichment.bolt.CacheKey; -import org.apache.metron.enrichment.interfaces.EnrichmentAdapter; -import org.json.simple.JSONObject; - -import org.apache.metron.test.AbstractTestContext; -import org.junit.Assert; - - /** - *
    - *
  • Title:
    • - *
  • Description:
    • - *
  • Created: Aug 25, 2014
    • - *
- * @version $Revision: 1.1 $ - */ -public class WhoisHBaseAdapterTest extends AbstractTestContext { - - private static WhoisHBaseAdapter whoisHbaseAdapter=null; - private static boolean connected=false; - /** - * Constructs a new WhoisHBaseAdapterTest instance. - * @param name - */ - - public WhoisHBaseAdapterTest(String name) { - super(name); - } - - /** - - * @throws java.lang.Exception - */ - protected static void setUpBeforeClass() throws Exception { - } - - /** - - * @throws java.lang.Exception - */ - protected static void tearDownAfterClass() throws Exception { - } - - /* - * (non-Javadoc) - * @see junit.framework.TestCase#setUp() - */ - - protected void setUp() throws Exception { - super.setUp(); - Properties prop = super.getTestProperties(); - Assert.assertNotNull(prop); - - if(skipTests(this.getMode())){ - return;//skip tests - } - - String[] zk = prop.get("kafka.zk.list").toString().split(","); - - for(String z : zk) - { - InetAddress address = InetAddress.getByName(z); - boolean reachable = address.isReachable(100); - - if(!reachable) - { - this.setMode("local"); - break; - //throw new Exception("Unable to reach zookeeper, skipping WHois adapter test"); - } - - System.out.println("kafka.zk.list ="+(String) prop.get("kafka.zk.list")); - System.out.println("kafka.zk.list ="+(String) prop.get("kafka.zk.port")); - System.out.println("kafka.zk.list ="+(String) prop.get("bolt.enrichment.cif.tablename")); - - } - - if(skipTests(this.getMode())){ - System.out.println("Local Mode Skipping tests !! "); - }else{ - whoisHbaseAdapter=new WhoisHBaseAdapter((String)prop.get("bolt.enrichment.whois.hbase.table.name"),(String)prop.get("kafka.zk.list"),(String)prop.get("kafka.zk.port")); - connected =whoisHbaseAdapter.initializeAdapter(); - Assert.assertTrue(connected); - } - - } - - /* - * (non-Javadoc) - * @see junit.framework.TestCase#tearDown() - */ - - protected void tearDown() throws Exception { - super.tearDown(); - } - - /** - * Test method for {@link EnrichmentAdapter#initializeAdapter()}. - */ - public void testInitializeAdapter() { - if(skipTests(this.getMode())){ - return;//skip tests - }else{ - Assert.assertTrue(connected); - } - } - - /** - * Test method for {@link org.apache.metron.enrichment.adapters.whois.WhoisHBaseAdapter#enrich(CacheKey)}. - */ - public void testEnrich() { - if(skipTests(this.getMode())){ - return;//skip tests - }else{ - JSONObject json = whoisHbaseAdapter.enrich(new CacheKey("whois", "72.163.4.161", null)); - - //assert Geo Response is not null - Assert.assertNotNull(json); - - //assert LocId is not null - Assert.assertNotNull(json.get("cisco.com")); - } - } - - - /** - * Returns the whoisHbaseAdapter. - * @return the whoisHbaseAdapter. - */ - - public static WhoisHBaseAdapter getWhoisHbaseAdapter() { - return whoisHbaseAdapter; - } - - /** - * Sets the whoisHbaseAdapter. - * @param whoisHbaseAdapter the whoisHbaseAdapter. - */ - - public static void setWhoisHbaseAdapter(WhoisHBaseAdapter whoisHbaseAdapter) { - - WhoisHBaseAdapterTest.whoisHbaseAdapter = whoisHbaseAdapter; - } -} - diff --git a/metron-streaming/Metron-MessageParsers/src/test/java/org/apache/metron/parsing/parsers/GrokParserTest.java b/metron-streaming/Metron-MessageParsers/src/test/java/org/apache/metron/parsing/parsers/GrokParserTest.java new file mode 100644 index 0000000000..81bf17bd1a --- /dev/null +++ b/metron-streaming/Metron-MessageParsers/src/test/java/org/apache/metron/parsing/parsers/GrokParserTest.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsing.parsers; + +import com.google.common.collect.MapDifference; +import com.google.common.collect.Maps; +import junit.framework.Assert; +import org.adrianwalker.multilinestring.Multiline; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Before; +import org.junit.Test; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GrokParserTest { + + public String expectedRaw = "2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle"; + + /** + * { + * "roct":0, + * "end_reason":"idle", + * "ip_dst_addr":"10.0.2.15", + * "iflags":"AS", + * "rpkt":0, + * "original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle", + * "tag":0, + * "risn":0, + * "ip_dst_port":39468, + * "ruflags":0, + * "app":0, + * "protocol":6 + * ,"isn":"22efa001", + * "uflags":0,"duration":"0.000", + * "oct":44, + * "ip_src_port":80, + * "end_time":"2016-01-28 15:29:48.512", + * "riflags":0,"start_time":"2016-01-28 15:29:48.512", + * "rtt":"0.000", + * "rtag":0, + * "pkt":1, + * "ip_src_addr":"216.21.170.221" + * } + */ + @Multiline + private String expectedParsedString; + + private JSONObject expectedParsed; + + @Before + public void parseJSON() throws ParseException { + JSONParser jsonParser = new JSONParser(); + expectedParsed = (JSONObject) jsonParser.parse(expectedParsedString); + } + + @Test + public void test() throws IOException, ParseException { + String metronHdfsHome = "../Metron-MessageParsers/src/main/resources"; + String grokHdfsPath = "/patterns/yaf"; + String patternLabel = "YAF_DELIMITED"; + GrokParser grokParser = new GrokParser(grokHdfsPath, patternLabel); + grokParser.withMetronHDFSHome(metronHdfsHome); + grokParser.init(); + byte[] rawMessage = expectedRaw.getBytes(); + List parsedList = grokParser.parse(rawMessage); + Assert.assertEquals(1, parsedList.size()); + compare(expectedParsed, parsedList.get(0)); + } + + public boolean compare(JSONObject expected, JSONObject actual) { + MapDifference mapDifferences = Maps.difference(expected, actual); + if (mapDifferences.entriesOnlyOnLeft().size() > 0) Assert.fail("Expected JSON has extra parameters: " + mapDifferences.entriesOnlyOnLeft()); + if (mapDifferences.entriesOnlyOnRight().size() > 0) Assert.fail("Actual JSON has extra parameters: " + mapDifferences.entriesOnlyOnRight()); + Map actualDifferences = new HashMap(); + if (mapDifferences.entriesDiffering().size() > 0) { + Map differences = Collections.unmodifiableMap(mapDifferences.entriesDiffering()); + for (Object key : differences.keySet()) { + Object expectedValueObject = expected.get(key); + Object actualValueObject = actual.get(key); + if (expectedValueObject instanceof Long || expectedValueObject instanceof Integer) { + Long expectedValue = Long.parseLong(expectedValueObject.toString()); + Long actualValue = Long.parseLong(actualValueObject.toString()); + if (!expectedValue.equals(actualValue)) { + actualDifferences.put(key, differences.get(key)); + } + } else { + actualDifferences.put(key, differences.get(key)); + } + } + } + if (actualDifferences.size() > 0) Assert.fail("Expected and Actual JSON values don't match: " + actualDifferences); + return true; + } +} diff --git a/metron-streaming/Metron-Testing/pom.xml b/metron-streaming/Metron-Testing/pom.xml index a8423229e4..d06953deb7 100644 --- a/metron-streaming/Metron-Testing/pom.xml +++ b/metron-streaming/Metron-Testing/pom.xml @@ -106,6 +106,11 @@ Metron-MessageParsers 0.1BETA + + org.apache.metron + Metron-TestingUtilities + 0.1BETA + diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java index fa140ed13f..2e4e0ccdaa 100644 --- a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java +++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java @@ -19,18 +19,16 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.*; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.metron.Constants; import org.apache.metron.domain.Configurations; -import org.apache.metron.domain.SensorEnrichmentConfig; import org.apache.metron.hbase.TableProvider; import org.apache.metron.hbase.converters.enrichment.EnrichmentKey; import org.apache.metron.hbase.converters.enrichment.EnrichmentValue; -import org.apache.metron.integration.util.EnrichmentHelper; +import org.apache.metron.hbase.converters.enrichment.EnrichmentHelper; import org.apache.metron.integration.util.TestUtils; import org.apache.metron.integration.util.UnitTestHelper; import org.apache.metron.integration.util.integration.ComponentRunner; diff --git a/metron-streaming/Metron-TestingUtilities/pom.xml b/metron-streaming/Metron-TestingUtilities/pom.xml new file mode 100644 index 0000000000..1c67e7cbb2 --- /dev/null +++ b/metron-streaming/Metron-TestingUtilities/pom.xml @@ -0,0 +1,73 @@ + + + + + 4.0.0 + + org.apache.metron + Metron-Streaming + 0.1BETA + + Metron-TestingUtilities + Metron Testing Utilities + + + + + log4j + log4j + 1.2.17 + + + junit + junit + ${global_junit_version} + + + + + com.google.guava + guava + ${global_hbase_guava_version} + + + org.apache.hbase + hbase-client + ${global_hbase_version} + + + log4j + log4j + + + + + org.apache.hadoop + hadoop-client + ${global_hadoop_version} + + + org.slf4j + slf4j-log4j12 + + + + + + + + + + diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/mock/MockHTable.java b/metron-streaming/Metron-TestingUtilities/src/main/java/org/apache/metron/integration/util/mock/MockHTable.java similarity index 100% rename from metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/mock/MockHTable.java rename to metron-streaming/Metron-TestingUtilities/src/main/java/org/apache/metron/integration/util/mock/MockHTable.java diff --git a/metron-streaming/pom.xml b/metron-streaming/pom.xml index 12568e1b82..2b40883042 100644 --- a/metron-streaming/pom.xml +++ b/metron-streaming/pom.xml @@ -80,6 +80,7 @@ Metron-Elasticsearch Metron-Solr Metron-Testing + Metron-TestingUtilities