From b93d95318fe86ccdaf55769e569608ca8f08ffdd Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 23 Aug 2017 13:49:35 -0400 Subject: [PATCH 1/6] Adjust the dependencies so the stellar functions are in metron-maas-common --- .gitignore | 2 +- metron-analytics/metron-maas-common/pom.xml | 11 +++++++++++ .../apache/metron/maas/functions/MaaSFunctions.java | 0 metron-analytics/metron-maas-service/pom.xml | 11 ----------- .../org/apache/metron/maas/service/runner/Runner.java | 6 +++++- 5 files changed, 17 insertions(+), 13 deletions(-) rename metron-analytics/{metron-maas-service => metron-maas-common}/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java (100%) diff --git a/.gitignore b/.gitignore index d505c9ca29..12fd7cd213 100644 --- a/.gitignore +++ b/.gitignore @@ -21,5 +21,5 @@ tmp/** tmp/**/* temp/** temp/**/* - +metron-interface/metron-alerts/node/ repodata/ diff --git a/metron-analytics/metron-maas-common/pom.xml b/metron-analytics/metron-maas-common/pom.xml index 13fb7b9606..6921e51aba 100644 --- a/metron-analytics/metron-maas-common/pom.xml +++ b/metron-analytics/metron-maas-common/pom.xml @@ -33,6 +33,17 @@ ${global_curator_version} + + org.apache.metron + stellar-common + ${project.parent.version} + + + org.apache.hadoop + hadoop-auth + + + commons-cli commons-cli diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java similarity index 100% rename from metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java rename to metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml index 0ac9bac570..4eeceae254 100644 --- a/metron-analytics/metron-maas-service/pom.xml +++ b/metron-analytics/metron-maas-service/pom.xml @@ -48,17 +48,6 @@ kryo-serializers ${global_kryo_serializers_version} - - org.apache.metron - stellar-common - ${project.parent.version} - - - org.apache.hadoop - hadoop-auth - - - org.apache.hadoop hadoop-yarn-server-common diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java index cc297d26f5..214b387c80 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java @@ -202,7 +202,11 @@ public static void main(String... argv) throws Exception { serviceDiscovery.start(); File cwd = new File(script).getParentFile(); - final String cmd = new File(cwd, script).getAbsolutePath(); + File scriptFile = new File(cwd, script); + if(scriptFile.exists() && !scriptFile.canExecute()) { + scriptFile.setExecutable(true); + } + final String cmd = scriptFile.getAbsolutePath(); try { p = new ProcessBuilder(cmd).directory(cwd).start(); From 9e99b887a4bfb92ed35b621bcca406522f08ae24 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 23 Aug 2017 14:02:21 -0400 Subject: [PATCH 2/6] Updating readme to be current. --- metron-analytics/metron-maas-service/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/metron-analytics/metron-maas-service/README.md b/metron-analytics/metron-maas-service/README.md index 5ed108bc6c..d59f5af407 100644 --- a/metron-analytics/metron-maas-service/README.md +++ b/metron-analytics/metron-maas-service/README.md @@ -170,8 +170,6 @@ Now that we have a deployed model, let's adjust the configurations for the Squid * Edit the squid enrichment configuration at `$METRON_HOME/config/zookeeper/enrichments/squid.json` (this file will not exist, so create a new one) to make the threat triage adjust the level of risk based on the model output: ``` { - "index": "squid", - "batchSize": 1, "enrichment" : { "fieldMap": {} }, From 0b38b920ce06e19979aeeed883dbdc6c3cd0ff2a Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 23 Aug 2017 16:35:08 -0400 Subject: [PATCH 3/6] Updated readme to be more generic. --- metron-analytics/metron-maas-service/README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/metron-analytics/metron-maas-service/README.md b/metron-analytics/metron-maas-service/README.md index d59f5af407..59f2acc0f2 100644 --- a/metron-analytics/metron-maas-service/README.md +++ b/metron-analytics/metron-maas-service/README.md @@ -126,16 +126,20 @@ Now let's install some prerequisites: Start Squid via `service squid start` Now that we have flask and jinja, we can create a mock DGA service to deploy with MaaS: -* Download the files in [this](https://gist.github.com/cestella/cba10aff0f970078a4c2c8cade3a4d1a) gist into the `/root/mock_dga` directory -* Make `rest.sh` executable via `chmod +x /root/mock_dga/rest.sh` +* Download the files in [this](https://gist.github.com/cestella/cba10aff0f970078a4c2c8cade3a4d1a) gist into the `$HOME/mock_dga` directory +* Make `rest.sh` executable via `chmod +x $HOME/mock_dga/rest.sh` This service will treat `yahoo.com` and `amazon.com` as legit and everything else as malicious. The contract is that the REST service exposes an endpoint `/apply` and returns back JSON maps with a single key `is_malicious` which can be `malicious` or `legit`. ## Deploy Mock DGA Service via MaaS +The following presumes that you are a logged in as a user who has a +home directory in HDFS under `/user/$USER`. If you do not, please create one +and ensure the permissions are set appropriate. + Now let's start MaaS and deploy the Mock DGA Service: * Start MaaS via `$METRON_HOME/bin/maas_service.sh -zq node1:2181` -* Start one instance of the mock DGA model with 512M of memory via `$METRON_HOME/bin/maas_deploy.sh -zq node1:2181 -lmp /root/mock_dga -hmp /user/root/models -mo ADD -m 512 -n dga -v 1.0 -ni 1` +* Start one instance of the mock DGA model with 512M of memory via `$METRON_HOME/bin/maas_deploy.sh -zq node1:2181 -lmp $HOME/mock_dga -hmp /user/$USER/models -mo ADD -m 512 -n dga -v 1.0 -ni 1` * As a sanity check: * Ensure that the model is running via `$METRON_HOME/bin/maas_deploy.sh -zq node1:2181 -mo LIST`. You should see `Model dga @ 1.0` be displayed and under that a url such as (but not exactly) `http://node1:36161` * Try to hit the model via curl: `curl 'http://localhost:36161/apply?host=caseystella.com'` and ensure that it returns a JSON map indicating the domain is malicious. From 405e31afdddc668e5c2cabfaff95518d7d231b64 Mon Sep 17 00:00:00 2001 From: cstella Date: Thu, 24 Aug 2017 09:28:00 -0400 Subject: [PATCH 4/6] Fixed some obvious mistakes and defended against some obvious mistakes that I ran into while testing this PR. --- .../org/apache/metron/maas/service/Client.java | 14 ++++++++++++++ .../org/apache/metron/maas/service/Constants.java | 5 ++++- .../apache/metron/maas/service/runner/Runner.java | 3 ++- .../apache/metron/maas/submit/ModelSubmission.java | 5 +++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java index c2d89069f6..646eb2e952 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java @@ -24,6 +24,7 @@ import java.util.function.Function; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import org.apache.commons.cli.*; import org.apache.commons.cli.CommandLine; @@ -558,6 +559,7 @@ public boolean run() throws IOException, YarnException { // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); + createMaaSDirectory(fs, appId.toString()); Path ajPath = addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); // Set the log4j properties if needed @@ -789,6 +791,17 @@ private void forceKillApplication(ApplicationId appId) yarnClient.killApplication(appId); } + private void createMaaSDirectory(FileSystem fs, String appId) throws IOException { + for(Path p : ImmutableList.of(new Path(fs.getHomeDirectory(), appName) + , new Path(fs.getHomeDirectory(), appName + "/" + appId) + ) + ) { + if(!fs.exists(p)) { + fs.setPermission(p, new FsPermission((short)0755)); + } + } + } + private Path addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map localResources, String resources) throws IOException { @@ -808,6 +821,7 @@ private Path addToLocalResources(FileSystem fs, String fileSrcPath, } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } + fs.setPermission(dst, new FsPermission((short)0755)); FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance( diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Constants.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Constants.java index ac2c950edb..d0325111e5 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Constants.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Constants.java @@ -31,5 +31,8 @@ public class Constants { * Environment key name denoting the timeline domain ID. */ public static final String TIMELINEDOMAIN = "TIMELINEDOMAIN"; - + /* + The filename which communicates the endpoint information for a deployed model + */ + public static final String ENDPOINT_DAT = "endpoint.dat"; } diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java index 214b387c80..8f0b9e512b 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java @@ -33,6 +33,7 @@ import org.apache.curator.x.discovery.*; import org.apache.curator.x.discovery.details.JsonInstanceSerializer; import org.apache.metron.maas.config.Endpoint; +import org.apache.metron.maas.service.Constants; import org.apache.metron.maas.util.ConfigUtil; import org.apache.metron.maas.config.MaaSConfig; import org.apache.metron.maas.config.ModelEndpoint; @@ -303,7 +304,7 @@ private static URL correctLocalUrl(String hostname, String tmpUrl) throws Malfor private static Endpoint readEndpoint(File cwd) throws Exception { String content = ""; - File f = new File(cwd, "endpoint.dat"); + File f = new File(cwd, Constants.ENDPOINT_DAT); for(int i = 0;i < NUM_ATTEMPTS;i++) { if(f.exists()) { try { diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java index ebfa904537..fcae40aeaa 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java @@ -33,6 +33,7 @@ import org.apache.log4j.PropertyConfigurator; import org.apache.metron.maas.config.*; import org.apache.metron.maas.discovery.ServiceDiscoverer; +import org.apache.metron.maas.service.Constants; import org.apache.metron.maas.service.Log4jPropertyHelper; import org.apache.metron.maas.util.ConfigUtil; import org.apache.metron.maas.queue.Queue; @@ -247,6 +248,10 @@ public static void updateHDFS(FileSystem fs, File localDir, Path hdfsPath) throw fs.mkdirs(hdfsPath); } for(File f : localDir.listFiles()) { + if(f.getName().equals(Constants.ENDPOINT_DAT)) { + //skip the endpoint if it exists accidentally, we don't want to localize that. + continue; + } Path p = new Path(hdfsPath, f.getName()); FSDataOutputStream out = fs.create(p); BufferedInputStream in = new BufferedInputStream(new FileInputStream(f)); From 4853d4a636019f109d6fb399995f444231906bb3 Mon Sep 17 00:00:00 2001 From: cstella Date: Thu, 24 Aug 2017 10:02:03 -0400 Subject: [PATCH 5/6] Missed a mkdir --- .../src/main/java/org/apache/metron/maas/service/Client.java | 1 + 1 file changed, 1 insertion(+) diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java index 646eb2e952..9cabf21f38 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java @@ -797,6 +797,7 @@ private void createMaaSDirectory(FileSystem fs, String appId) throws IOException ) ) { if(!fs.exists(p)) { + fs.mkdirs(p); fs.setPermission(p, new FsPermission((short)0755)); } } From b945e9f7d6158e0ec0137d209698a5aced2d1706 Mon Sep 17 00:00:00 2001 From: cstella Date: Tue, 29 Aug 2017 09:46:28 -0400 Subject: [PATCH 6/6] Updating docs to be more clear --- metron-analytics/metron-maas-service/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/metron-analytics/metron-maas-service/README.md b/metron-analytics/metron-maas-service/README.md index 59f2acc0f2..d23413069e 100644 --- a/metron-analytics/metron-maas-service/README.md +++ b/metron-analytics/metron-maas-service/README.md @@ -135,7 +135,16 @@ This service will treat `yahoo.com` and `amazon.com` as legit and everything els The following presumes that you are a logged in as a user who has a home directory in HDFS under `/user/$USER`. If you do not, please create one -and ensure the permissions are set appropriate. +and ensure the permissions are set appropriate: +``` +su - hdfs -c "hadoop fs -mkdir /user/$USER" +su - hdfs -c "hadoop fs -chown $USER:$USER /user/$USER" +``` +Or, in the common case for the `metron` user: +``` +su - hdfs -c "hadoop fs -mkdir /user/metron" +su - hdfs -c "hadoop fs -chown metron:metron /user/metron" +``` Now let's start MaaS and deploy the Mock DGA Service: * Start MaaS via `$METRON_HOME/bin/maas_service.sh -zq node1:2181`