METRON-1128: MAAS_GET_ENDPOINT - Unable to resolve function error seen for squid example

.gitignore

@@ -21,5 +21,5 @@ tmp/**
 tmp/**/*
 temp/**
 temp/**/*
-
+metron-interface/metron-alerts/node/
 repodata/

metron-analytics/metron-maas-common/pom.xml

@@ -33,6 +33,17 @@
     <curator.version>${global_curator_version}</curator.version>
   </properties>
   <dependencies>
+    <dependency>
+      <groupId>org.apache.metron</groupId>
+      <artifactId>stellar-common</artifactId>
+      <version>${project.parent.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-auth</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>commons-cli</groupId>
       <artifactId>commons-cli</artifactId>

metron-analytics/metron-maas-service/README.md

@@ -126,16 +126,29 @@ Now let's install some prerequisites:
 Start Squid via `service squid start`
 
 Now that we have flask and jinja, we can create a mock DGA service to deploy with MaaS:
-* Download the files in [this](https://gist.github.com/cestella/cba10aff0f970078a4c2c8cade3a4d1a) gist into the `/root/mock_dga` directory
-* Make `rest.sh` executable via `chmod +x /root/mock_dga/rest.sh`
+* Download the files in [this](https://gist.github.com/cestella/cba10aff0f970078a4c2c8cade3a4d1a) gist into the `$HOME/mock_dga` directory
+* Make `rest.sh` executable via `chmod +x $HOME/mock_dga/rest.sh`
 
 This service will treat `yahoo.com` and `amazon.com` as legit and everything else as malicious.  The contract is that the REST service exposes an endpoint `/apply` and returns back JSON maps with a single key `is_malicious` which can be `malicious` or `legit`.
 
 ## Deploy Mock DGA Service via MaaS
 
+The following presumes that you are a logged in as a user who has a
+home directory in HDFS under `/user/$USER`.  If you do not, please create one
+and ensure the permissions are set appropriate:
+```
+su - hdfs -c "hadoop fs -mkdir /user/$USER"
+su - hdfs -c "hadoop fs -chown $USER:$USER /user/$USER"
+```
+Or, in the common case for the `metron` user:
+```
+su - hdfs -c "hadoop fs -mkdir /user/metron"
+su - hdfs -c "hadoop fs -chown metron:metron /user/metron"
+```
+
 Now let's start MaaS and deploy the Mock DGA Service:
 * Start MaaS via `$METRON_HOME/bin/maas_service.sh -zq node1:2181`
-* Start one instance of the mock DGA model with 512M of memory via `$METRON_HOME/bin/maas_deploy.sh -zq node1:2181 -lmp /root/mock_dga -hmp /user/root/models -mo ADD -m 512 -n dga -v 1.0 -ni 1`
+* Start one instance of the mock DGA model with 512M of memory via `$METRON_HOME/bin/maas_deploy.sh -zq node1:2181 -lmp $HOME/mock_dga -hmp /user/$USER/models -mo ADD -m 512 -n dga -v 1.0 -ni 1`
 * As a sanity check:
   * Ensure that the model is running via `$METRON_HOME/bin/maas_deploy.sh -zq node1:2181 -mo LIST`.  You should see `Model dga @ 1.0` be displayed and under that a url such as (but not exactly) `http://node1:36161`
   * Try to hit the model via curl: `curl 'http://localhost:36161/apply?host=caseystella.com'` and ensure that it returns a JSON map indicating the domain is malicious.
@@ -170,8 +183,6 @@ Now that we have a deployed model, let's adjust the configurations for the Squid
 * Edit the squid enrichment configuration at `$METRON_HOME/config/zookeeper/enrichments/squid.json` (this file will not exist, so create a new one) to make the threat triage adjust the level of risk based on the model output:
 ```
 {
-  "index": "squid",
-  "batchSize": 1,
   "enrichment" : {
     "fieldMap": {}
   },

metron-analytics/metron-maas-service/pom.xml

@@ -48,17 +48,6 @@
       <artifactId>kryo-serializers</artifactId>
       <version>${global_kryo_serializers_version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.apache.metron</groupId>
-      <artifactId>stellar-common</artifactId>
-      <version>${project.parent.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-auth</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-server-common</artifactId>

metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Client.java

@@ -24,6 +24,7 @@
 import java.util.function.Function;
 
 import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import org.apache.commons.cli.*;
 import org.apache.commons.cli.CommandLine;
@@ -558,6 +559,7 @@ public boolean run() throws IOException, YarnException {
     // Copy the application master jar to the filesystem
     // Create a local resource to point to the destination jar path
     FileSystem fs = FileSystem.get(conf);
+    createMaaSDirectory(fs, appId.toString());
     Path ajPath = addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null);
 
     // Set the log4j properties if needed
@@ -789,6 +791,18 @@ private void forceKillApplication(ApplicationId appId)
     yarnClient.killApplication(appId);
   }
 
+  private void createMaaSDirectory(FileSystem fs, String appId) throws IOException {
+    for(Path p : ImmutableList.of(new Path(fs.getHomeDirectory(), appName)
+                                 , new Path(fs.getHomeDirectory(), appName + "/" + appId)
+                                 )
+       ) {
+      if(!fs.exists(p)) {
+        fs.mkdirs(p);
+        fs.setPermission(p, new FsPermission((short)0755));
+      }
+    }
+  }
+
   private Path addToLocalResources(FileSystem fs, String fileSrcPath,
                                    String fileDstPath, String appId, Map<String, LocalResource> localResources,
                                    String resources) throws IOException {
@@ -808,6 +822,7 @@ private Path addToLocalResources(FileSystem fs, String fileSrcPath,
     } else {
       fs.copyFromLocalFile(new Path(fileSrcPath), dst);
     }
+    fs.setPermission(dst, new FsPermission((short)0755));
     FileStatus scFileStatus = fs.getFileStatus(dst);
     LocalResource scRsrc =
             LocalResource.newInstance(

metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/Constants.java

@@ -31,5 +31,8 @@ public class Constants {
    * Environment key name denoting the timeline domain ID.
    */
   public static final String TIMELINEDOMAIN = "TIMELINEDOMAIN";
-
+  /*
+  The filename which communicates the endpoint information for a deployed model
+   */
+  public static final String ENDPOINT_DAT = "endpoint.dat";
 }

metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java

@@ -33,6 +33,7 @@
 import org.apache.curator.x.discovery.*;
 import org.apache.curator.x.discovery.details.JsonInstanceSerializer;
 import org.apache.metron.maas.config.Endpoint;
+import org.apache.metron.maas.service.Constants;
 import org.apache.metron.maas.util.ConfigUtil;
 import org.apache.metron.maas.config.MaaSConfig;
 import org.apache.metron.maas.config.ModelEndpoint;
@@ -202,7 +203,11 @@ public static void main(String... argv) throws Exception {
       serviceDiscovery.start();
 
       File cwd = new File(script).getParentFile();
-      final String cmd = new File(cwd, script).getAbsolutePath();
+      File scriptFile = new File(cwd, script);
+      if(scriptFile.exists() && !scriptFile.canExecute()) {
+        scriptFile.setExecutable(true);
+      }
+      final String cmd = scriptFile.getAbsolutePath();
         try {
           p = new ProcessBuilder(cmd).directory(cwd).start();
 
@@ -299,7 +304,7 @@ private static URL correctLocalUrl(String hostname, String tmpUrl) throws Malfor
 
   private static Endpoint readEndpoint(File cwd) throws Exception {
     String content = "";
-    File f = new File(cwd, "endpoint.dat");
+    File f = new File(cwd, Constants.ENDPOINT_DAT);
     for(int i = 0;i < NUM_ATTEMPTS;i++) {
       if(f.exists()) {
         try {

metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/submit/ModelSubmission.java

@@ -33,6 +33,7 @@
 import org.apache.log4j.PropertyConfigurator;
 import org.apache.metron.maas.config.*;
 import org.apache.metron.maas.discovery.ServiceDiscoverer;
+import org.apache.metron.maas.service.Constants;
 import org.apache.metron.maas.service.Log4jPropertyHelper;
 import org.apache.metron.maas.util.ConfigUtil;
 import org.apache.metron.maas.queue.Queue;
@@ -247,6 +248,10 @@ public static void updateHDFS(FileSystem fs, File localDir, Path hdfsPath) throw
         fs.mkdirs(hdfsPath);
       }
       for(File f : localDir.listFiles()) {
+        if(f.getName().equals(Constants.ENDPOINT_DAT)) {
+          //skip the endpoint if it exists accidentally, we don't want to localize that.
+          continue;
+        }
         Path p = new Path(hdfsPath, f.getName());
         FSDataOutputStream out = fs.create(p);
         BufferedInputStream in = new BufferedInputStream(new FileInputStream(f));