From e4cd0620c64bc67bde027caa224b99dbaf454636 Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 26 Sep 2016 18:01:01 -0400 Subject: [PATCH 01/11] Initial implementation --- metron-platform/metron-management/pom.xml | 19 +- .../management/FileSystemFunctions.java | 307 ++++++++++++++++++ 2 files changed, 325 insertions(+), 1 deletion(-) create mode 100644 metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml index b71d780bee..2c555d3084 100644 --- a/metron-platform/metron-management/pom.xml +++ b/metron-platform/metron-management/pom.xml @@ -47,6 +47,24 @@ ${global_guava_version} compile + + org.apache.hadoop + hadoop-hdfs + ${global_hadoop_version} + + + servlet-api + javax.servlet + + + provided + + + org.apache.hadoop + hadoop-common + ${global_hadoop_version} + provided + org.apache.curator curator-test @@ -66,7 +84,6 @@ test-jar test - diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java new file mode 100644 index 0000000000..bf526f7e8c --- /dev/null +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java @@ -0,0 +1,307 @@ +package org.apache.metron.management; + +import com.jakewharton.fliptables.FlipTable; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.*; +import org.apache.log4j.Logger; +import org.apache.metron.common.dsl.Context; +import org.apache.metron.common.dsl.ParseException; +import org.apache.metron.common.dsl.Stellar; +import org.apache.metron.common.dsl.StellarFunction; +import org.apache.metron.common.utils.ConversionUtils; + +import java.io.IOException; +import java.text.DateFormat; +import java.util.*; + +public class FileSystemFunctions { + private static final Logger LOG = Logger.getLogger(FileSystemFunctions.class); + + private static abstract class AbstractFileSystemFunction implements StellarFunction { + protected FileSystem fs; + + @Override + public void initialize(Context context) { + try { + fs = getFs(); + } catch (IOException e) { + String message = "Unable to get FileSystem: " + e.getMessage(); + LOG.error(message, e); + throw new IllegalStateException(message, e); + } + } + + @Override + public boolean isInitialized() { + return fs != null; + } + + public abstract FileSystem getFs() throws IOException; + } + + private static abstract class FileSystemGet extends AbstractFileSystemFunction { + + @Override + public Object apply(List args, Context context) throws ParseException { + String path = (String) args.get(0); + if(path == null) { + return null; + } + try(FSDataInputStream is = fs.open(new Path(path))) { + return IOUtils.toString(is); + } catch (IOException e) { + String message = "Unable to read " + path + ": " + e.getMessage(); + LOG.error(message, e); + return null; + } + } + } + + public static abstract class FileSystemRm extends AbstractFileSystemFunction { + + @Override + public Object apply(List args, Context context) throws ParseException { + + String path = (String) args.get(0); + if(path == null) { + return false; + } + + boolean recursive = false; + if(args.size() > 1) { + recursive = ConversionUtils.convert(args.get(1), Boolean.class); + } + + try { + fs.delete(new Path(path), recursive); + return true; + } catch (IOException e) { + String message = "Unable to remove " + path + (recursive?" recursively":"") + ": " + e.getMessage(); + LOG.error(message, e); + return false; + } + } + } + + public static abstract class FileSystemPut extends AbstractFileSystemFunction { + + @Override + public Object apply(List args, Context context) throws ParseException { + String content = (String)args.get(0); + if(content == null) { + return false; + } + String path = (String) args.get(1); + if(path == null) { + return false; + } + + try(FSDataOutputStream os = fs.create(new Path(path))) { + os.writeChars(content); + os.flush(); + return true; + } catch (IOException e) { + String message = "Unable to write " + path + ": " + e.getMessage(); + LOG.error(message, e); + return false; + } + } + } + + public static abstract class FileSystemLs extends AbstractFileSystemFunction { + private static ThreadLocal dateFormat = new ThreadLocal() { + + @Override + protected DateFormat initialValue() { + return DateFormat.getDateTimeInstance( + DateFormat.DEFAULT, + DateFormat.DEFAULT, + Locale.getDefault() + ); + } + }; + + @Override + public Object apply(List args, Context context) throws ParseException { + + String path = (String) args.get(1); + if(path == null) { + return false; + } + try { + String[] headers = new String[] {"PERMISSION", "OWNER", "GROUP", "SIZE", "LAST MOD TIME", "NAME"}; + List dataList = new ArrayList<>(); + for(RemoteIterator it = fs.listFiles(new Path(path), false);it.hasNext();) { + final FileStatus status = it.next(); + dataList.add(new String[] { + status.getPermission().toString() + ,status.getOwner() + ,status.getGroup() + ,status.getLen() + "" + ,dateFormat.get().format(new Date(status.getModificationTime())) + ,status.getPath().getName() + }); + } + Collections.sort(dataList, (o1, o2) -> { + try { + Date left = dateFormat.get().parse(o1[4]); + Date right = dateFormat.get().parse(o2[4]); + int ret = left.compareTo(right); + //first sort by date, then by name. + if(ret == 0) { + return o1[5].compareTo(o2[5]); + } + else { + return ret; + } + } catch (java.text.ParseException e) { + String message = "Unable to parse " + o1 + " or " + o2 + ": " + e.getMessage(); + LOG.error(message, e); + throw new IllegalStateException(message, e); + } + }); + String[][] data = new String[dataList.size()][headers.length]; + for(int i = 0;i < dataList.size();++i) { + data[i] = dataList.get(i); + } + return FlipTable.of(headers, data); + } catch (IOException e) { + String message = "Unable to list" + path + ": " + e.getMessage(); + LOG.error(message, e); + return null; + } + } + } + + @Stellar(namespace="FILE" + ,name="RM" + ,description="Removes the path" + ,params = { "path - The path of the file." + , "recursive - Recursively remove or not (optional and defaulted to false)" + } + ,returns = "boolean - true if successful, false otherwise." + + ) + public static class FileRm extends FileSystemRm { + + @Override + public FileSystem getFs() throws IOException { + return new LocalFileSystem(); + } + } + + @Stellar(namespace="HDFS" + ,name="RM" + ,description="Removes the path" + ,params = { "path - The path in HDFS of the file." + , "recursive - Recursively remove or not (optional and defaulted to false)" + } + ,returns = "boolean - true if successful, false otherwise." + + ) + public static class HDFSRm extends FileSystemRm { + + @Override + public FileSystem getFs() throws IOException { + return FileSystem.get(new Configuration()); + } + } + + @Stellar(namespace="FILE" + ,name="LS" + ,description="Lists the contents of a directory" + ,params = { "path - The path of the file." + } + ,returns = "The contents of the directory in tabular form sorted by last modification date." + + ) + public static class FileLs extends FileSystemLs { + + @Override + public FileSystem getFs() throws IOException { + return new LocalFileSystem(); + } + } + + @Stellar(namespace="HDFS" + ,name="LS" + ,description="Lists the contents of a directory" + ,params = { "path - The path in HDFS of the file." + } + ,returns = "The contents of the directory in tabular form sorted by last modification date." + + ) + public static class HDFSLs extends FileSystemLs { + + @Override + public FileSystem getFs() throws IOException { + return FileSystem.get(new Configuration()); + } + } + + @Stellar(namespace="HDFS" + ,name="PUT" + ,description="Writes the contents a string to a file in HDFS." + ,params = { "content - The content to write out" + , "path - The path in HDFS of the file." + } + ,returns = "true if the file was written and false otherwise." + + ) + public static class HDFSPut extends FileSystemPut { + + @Override + public FileSystem getFs() throws IOException { + return FileSystem.get(new Configuration()); + } + } + + @Stellar(namespace="FILE" + ,name="PUT" + ,description="Writes the contents a string to a local file." + ,params = { "content - The content to write out" + , "path - The path of the file." + } + ,returns = "true if the file was written and false otherwise." + + ) + public static class FilePut extends FileSystemPut { + + @Override + public FileSystem getFs() throws IOException { + return new LocalFileSystem(); + } + } + + @Stellar(namespace="HDFS" + ,name="GET" + ,description="Retrieves the contents as a string of a file in HDFS." + ,params = { "path - The path in HDFS of the file."} + ,returns = "The contents of the file in the path from HDFS and null otherwise." + + ) + public static class HDFSGet extends FileSystemGet { + + @Override + public FileSystem getFs() throws IOException { + return FileSystem.get(new Configuration()); + } + } + + @Stellar(namespace="FILE" + ,name="GET" + ,description="Retrieves the contents as a string of a file on the local filesystem." + ,params = { "path - The path of the file."} + ,returns = "The contents of the file or null otherwise." + + ) + public static class FileGet extends FileSystemGet { + + @Override + public FileSystem getFs() throws IOException { + return new LocalFileSystem(); + } + } + +} From 57897decf4a8eec03d6e32ec7fcaab54db8d8563 Mon Sep 17 00:00:00 2001 From: cstella Date: Mon, 26 Sep 2016 21:15:52 -0400 Subject: [PATCH 02/11] Added unit/integration tests. --- metron-platform/metron-management/pom.xml | 6 + .../management/FileSystemFunctions.java | 132 ++++++++++----- .../management/FileSystemFunctionsTest.java | 157 ++++++++++++++++++ .../src/test/resources/log4j.properties | 19 +++ 4 files changed, 271 insertions(+), 43 deletions(-) create mode 100644 metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java create mode 100644 metron-platform/metron-management/src/test/resources/log4j.properties diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml index 2c555d3084..e0ef0bc90f 100644 --- a/metron-platform/metron-management/pom.xml +++ b/metron-platform/metron-management/pom.xml @@ -84,6 +84,12 @@ test-jar test + + org.apache.hadoop + hadoop-minicluster + ${global_hadoop_version} + test + diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java index bf526f7e8c..2c1bcbb01d 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.metron.management; import com.jakewharton.fliptables.FlipTable; @@ -12,19 +29,48 @@ import org.apache.metron.common.utils.ConversionUtils; import java.io.IOException; +import java.net.URI; import java.text.DateFormat; import java.util.*; +import java.util.function.Function; public class FileSystemFunctions { private static final Logger LOG = Logger.getLogger(FileSystemFunctions.class); - private static abstract class AbstractFileSystemFunction implements StellarFunction { - protected FileSystem fs; + public interface FileSystemGetter { + FileSystem getSystem() throws IOException ; + } + + public static enum FS_TYPE implements FileSystemGetter { + LOCAL(() -> { + FileSystem fs = new LocalFileSystem(); + fs.initialize(URI.create("file:///"), new Configuration()); + return fs; + }) + ,HDFS(() -> FileSystem.get(new Configuration())) + ; + FileSystemGetter _func; + FS_TYPE(FileSystemGetter func) { + _func = func; + } + + @Override + public FileSystem getSystem() throws IOException { + return _func.getSystem(); + } + } + + private abstract static class FileSystemFunction implements StellarFunction { + protected FileSystem fs; + private FileSystemGetter getter; + FileSystemFunction(FileSystemGetter getter) { + this.getter = getter; + } @Override public void initialize(Context context) { try { - fs = getFs(); + fs = getter.getSystem(); } catch (IOException e) { String message = "Unable to get FileSystem: " + e.getMessage(); LOG.error(message, e); @@ -37,10 +83,13 @@ public boolean isInitialized() { return fs != null; } - public abstract FileSystem getFs() throws IOException; } - private static abstract class FileSystemGet extends AbstractFileSystemFunction { + static class FileSystemGet extends FileSystemFunction { + + FileSystemGet(FileSystemGetter getter) { + super(getter); + } @Override public Object apply(List args, Context context) throws ParseException { @@ -58,7 +107,11 @@ public Object apply(List args, Context context) throws ParseException { } } - public static abstract class FileSystemRm extends AbstractFileSystemFunction { + static class FileSystemRm extends FileSystemFunction { + + FileSystemRm(FileSystemGetter getter) { + super(getter); + } @Override public Object apply(List args, Context context) throws ParseException { @@ -84,7 +137,11 @@ public Object apply(List args, Context context) throws ParseException { } } - public static abstract class FileSystemPut extends AbstractFileSystemFunction { + static class FileSystemPut extends FileSystemFunction { + + FileSystemPut(FileSystemGetter getter) { + super(getter); + } @Override public Object apply(List args, Context context) throws ParseException { @@ -98,7 +155,7 @@ public Object apply(List args, Context context) throws ParseException { } try(FSDataOutputStream os = fs.create(new Path(path))) { - os.writeChars(content); + os.writeBytes(content); os.flush(); return true; } catch (IOException e) { @@ -109,7 +166,7 @@ public Object apply(List args, Context context) throws ParseException { } } - public static abstract class FileSystemLs extends AbstractFileSystemFunction { + static class FileSystemLs extends FileSystemFunction { private static ThreadLocal dateFormat = new ThreadLocal() { @Override @@ -122,18 +179,21 @@ protected DateFormat initialValue() { } }; + FileSystemLs(FileSystemGetter getter) { + super(getter); + } + @Override public Object apply(List args, Context context) throws ParseException { - String path = (String) args.get(1); + String path = (String) args.get(0); if(path == null) { return false; } try { String[] headers = new String[] {"PERMISSION", "OWNER", "GROUP", "SIZE", "LAST MOD TIME", "NAME"}; List dataList = new ArrayList<>(); - for(RemoteIterator it = fs.listFiles(new Path(path), false);it.hasNext();) { - final FileStatus status = it.next(); + for(FileStatus status : fs.listStatus(new Path(path))) { dataList.add(new String[] { status.getPermission().toString() ,status.getOwner() @@ -184,10 +244,8 @@ public Object apply(List args, Context context) throws ParseException { ) public static class FileRm extends FileSystemRm { - - @Override - public FileSystem getFs() throws IOException { - return new LocalFileSystem(); + public FileRm() { + super(FS_TYPE.LOCAL); } } @@ -201,10 +259,8 @@ public FileSystem getFs() throws IOException { ) public static class HDFSRm extends FileSystemRm { - - @Override - public FileSystem getFs() throws IOException { - return FileSystem.get(new Configuration()); + public HDFSRm() { + super(FS_TYPE.HDFS); } } @@ -217,10 +273,8 @@ public FileSystem getFs() throws IOException { ) public static class FileLs extends FileSystemLs { - - @Override - public FileSystem getFs() throws IOException { - return new LocalFileSystem(); + public FileLs() { + super(FS_TYPE.LOCAL); } } @@ -233,10 +287,8 @@ public FileSystem getFs() throws IOException { ) public static class HDFSLs extends FileSystemLs { - - @Override - public FileSystem getFs() throws IOException { - return FileSystem.get(new Configuration()); + public HDFSLs() { + super(FS_TYPE.HDFS); } } @@ -251,9 +303,9 @@ public FileSystem getFs() throws IOException { ) public static class HDFSPut extends FileSystemPut { - @Override - public FileSystem getFs() throws IOException { - return FileSystem.get(new Configuration()); + + public HDFSPut() { + super(FS_TYPE.HDFS); } } @@ -267,10 +319,8 @@ public FileSystem getFs() throws IOException { ) public static class FilePut extends FileSystemPut { - - @Override - public FileSystem getFs() throws IOException { - return new LocalFileSystem(); + public FilePut() { + super(FS_TYPE.LOCAL); } } @@ -282,10 +332,8 @@ public FileSystem getFs() throws IOException { ) public static class HDFSGet extends FileSystemGet { - - @Override - public FileSystem getFs() throws IOException { - return FileSystem.get(new Configuration()); + public HDFSGet() { + super(FS_TYPE.HDFS); } } @@ -297,10 +345,8 @@ public FileSystem getFs() throws IOException { ) public static class FileGet extends FileSystemGet { - - @Override - public FileSystem getFs() throws IOException { - return new LocalFileSystem(); + public FileGet() { + super(FS_TYPE.LOCAL); } } diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java new file mode 100644 index 0000000000..d371ce181a --- /dev/null +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.management; + +import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.metron.common.dsl.Context; +import org.apache.metron.common.dsl.StellarFunctions; +import org.apache.metron.common.stellar.StellarProcessor; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +@RunWith(Parameterized.class) +public class FileSystemFunctionsTest { + private FileSystemFunctions.FS_TYPE type; + private FileSystemFunctions.FileSystemGetter fsGetter = null; + private File baseDir; + private MiniDFSCluster hdfsCluster; + private String prefix; + private Context context = null; + private FileSystemFunctions.FileSystemGet get; + private FileSystemFunctions.FileSystemLs ls; + private FileSystemFunctions.FileSystemPut put; + private FileSystemFunctions.FileSystemRm rm; + + public FileSystemFunctionsTest(FileSystemFunctions.FS_TYPE type) { + this.type = type; + } + + @Parameterized.Parameters + public static Collection types() { + return Arrays.asList(new Object[][]{ + {FileSystemFunctions.FS_TYPE.HDFS} + , {FileSystemFunctions.FS_TYPE.LOCAL} + }); + } + + @Before + public void setup() throws IOException { + if(type == FileSystemFunctions.FS_TYPE.HDFS) { + baseDir = Files.createTempDirectory("test_hdfs").toFile().getAbsoluteFile(); + Configuration conf = new Configuration(); + conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); + MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); + hdfsCluster = builder.build(); + + fsGetter = () -> hdfsCluster.getFileSystem(); + prefix = "/"; + } + else { + fsGetter = FileSystemFunctions.FS_TYPE.LOCAL; + prefix = "target/fsTest/"; + if(new File(prefix).exists()) { + new File(prefix).delete(); + } + new File(prefix).mkdirs(); + } + + get = new FileSystemFunctions.FileSystemGet(fsGetter); + get.initialize(null); + ls = new FileSystemFunctions.FileSystemLs(fsGetter); + ls.initialize(null); + put = new FileSystemFunctions.FileSystemPut(fsGetter); + put.initialize(null); + rm = new FileSystemFunctions.FileSystemRm(fsGetter); + rm.initialize(null); + } + + @After + public void teardown() { + if(type == FileSystemFunctions.FS_TYPE.HDFS) { + hdfsCluster.shutdown(); + FileUtil.fullyDelete(baseDir); + } + else { + new File(prefix).delete(); + } + } + + @Test + public void testHappyPath() { + Object putOut = put.apply(Arrays.asList("foo", prefix + "testPut.dat"), null); + Assert.assertTrue((Boolean) putOut); + String getOut = (String)get.apply(Arrays.asList(prefix + "testPut.dat"), null); + Assert.assertEquals("foo", getOut); + Boolean rmRet = (Boolean)rm.apply(Arrays.asList(prefix + "testPut.dat"), null); + Assert.assertTrue(rmRet); + String lsOut = (String) ls.apply(Arrays.asList(prefix), null); + Assert.assertTrue(lsOut.contains("(empty)")); + } + + @Test + public void testPutMissingFile() { + Object o = put.apply(Arrays.asList("foo", null), null); + Assert.assertFalse((Boolean) o); + String lsOut = (String) ls.apply(Arrays.asList(prefix), null); + Assert.assertTrue(lsOut.contains("(empty)")); + } + + @Test + public void testRmTwice() { + Object putOut = put.apply(Arrays.asList("foo", prefix + "testPut.dat"), null); + Assert.assertTrue((Boolean) putOut); + Boolean rmRet = (Boolean)rm.apply(Arrays.asList(prefix + "testPut.dat"), null); + Assert.assertTrue(rmRet); + rmRet = (Boolean)rm.apply(Arrays.asList(prefix + "testPut.dat"), null); + Assert.assertTrue(rmRet); + String lsOut = (String) ls.apply(Arrays.asList(prefix), null); + Assert.assertTrue(lsOut.contains("(empty)")); + } + + @Test + public void testRecursiveRm() { + Object putOut = put.apply(Arrays.asList("foo", prefix + "blah/testPut.dat"), null); + Assert.assertTrue((Boolean) putOut); + putOut = put.apply(Arrays.asList("grok", prefix + "blah/testPut2.dat"), null); + Assert.assertTrue((Boolean) putOut); + Assert.assertEquals("foo", (String)get.apply(Arrays.asList(prefix + "blah/testPut.dat"), null)); + Assert.assertEquals("grok", (String)get.apply(Arrays.asList(prefix + "blah/testPut2.dat"), null)); + boolean rmRet = (Boolean)rm.apply(Arrays.asList(prefix + "blah", true), null); + Assert.assertTrue(rmRet); + String lsOut = (String) ls.apply(Arrays.asList(prefix), null); + Assert.assertTrue(lsOut.contains("(empty)")); + } + +} diff --git a/metron-platform/metron-management/src/test/resources/log4j.properties b/metron-platform/metron-management/src/test/resources/log4j.properties new file mode 100644 index 0000000000..74a9b22dc1 --- /dev/null +++ b/metron-platform/metron-management/src/test/resources/log4j.properties @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# log4j configuration used during build and unit tests + +log4j.rootLogger=error,stdout +log4j.threshhold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n \ No newline at end of file From 4a2893ecd058d1ce1591cb849291cbe767db1b31 Mon Sep 17 00:00:00 2001 From: cstella Date: Tue, 27 Sep 2016 08:47:54 -0400 Subject: [PATCH 03/11] Updated docs. --- metron-platform/metron-management/README.md | 51 +++++++++++++++++++ .../management/FileSystemFunctions.java | 2 +- .../management/FileSystemFunctionsTest.java | 4 +- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/metron-platform/metron-management/README.md b/metron-platform/metron-management/README.md index 7b4b061fdf..6f53f18ab9 100644 --- a/metron-platform/metron-management/README.md +++ b/metron-platform/metron-management/README.md @@ -19,11 +19,62 @@ project. The functions are split roughly into a few sections: * Shell functions - Functions surrounding interacting with the shell in either a nicer way or a more functional way. +* File functions - Functions around interacting with local or HDFS files * Configuration functions - Functions surrounding pulling and pushing configs from zookeeper * Parser functions - Functions surrounding adding, viewing, and removing Parser functions. * Enrichment functions - Functions surrounding adding, viewing and removing Stellar enrichments as well as managing batch size and index names for the enrichment topology configuration * Threat Triage functions - Functions surrounding adding, viewing and removing threat triage functions. +### File Functions + +* Local Files + * `FILE_LS` + * Description: Lists the contents of a directory. + * Input: + * path - The path of the file + * Returns: The contents of the directory in tabular form sorted by last modification date. + * `FILE_RM` + * Description: Removes the path + * Input: + * path - The path of the file or directory. + * recursive - Recursively remove or not (optional and defaulted to false) + * Returns: boolean - true if successful, false otherwise + * `FILE_GET` + * Description: Retrieves the contents as a string of a file. + * Input: + * path - The path of the file + * Returns: The contents of the file and null otherwise. + * `FILE_PUT` + * Description: Writes the contents of a string to a local file + * Input: + * content - The content to write out + * path - The path of the file + * Returns: true if the file was written and false otherwise. +* HDFS Files + * `HDFS_LS` + * Description: Lists the contents of a directory in HDFS. + * Input: + * path - The path of the file + * Returns: The contents of the directory in tabular form sorted by last modification date. + * `HDFS_RM` + * Description: Removes the path in HDFS. + * Input: + * path - The path of the file or directory. + * recursive - Recursively remove or not (optional and defaulted to false) + * Returns: boolean - true if successful, false otherwise + * `HDFS_GET` + * Description: Retrieves the contents as a string of a file in HDFS. + * Input: + * path - The path of the file + * Returns: The contents of the file and null otherwise. + * `HDFS_PUT` + * Description: Writes the contents of a string to a HDFS file + * Input: + * content - The content to write out + * path - The path of the file + * Returns: true if the file was written and false otherwise. + + ### Shell Functions * `SHELL_EDIT` diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java index 2c1bcbb01d..7090563728 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java @@ -237,7 +237,7 @@ public Object apply(List args, Context context) throws ParseException { @Stellar(namespace="FILE" ,name="RM" ,description="Removes the path" - ,params = { "path - The path of the file." + ,params = { "path - The path of the file or directory." , "recursive - Recursively remove or not (optional and defaulted to false)" } ,returns = "boolean - true if successful, false otherwise." diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java index d371ce181a..6ace70eae3 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java @@ -114,9 +114,11 @@ public void testHappyPath() { Assert.assertTrue((Boolean) putOut); String getOut = (String)get.apply(Arrays.asList(prefix + "testPut.dat"), null); Assert.assertEquals("foo", getOut); + String lsOut = (String) ls.apply(Arrays.asList(prefix), null); + Assert.assertFalse(lsOut.contains("(empty)")); Boolean rmRet = (Boolean)rm.apply(Arrays.asList(prefix + "testPut.dat"), null); Assert.assertTrue(rmRet); - String lsOut = (String) ls.apply(Arrays.asList(prefix), null); + lsOut = (String) ls.apply(Arrays.asList(prefix), null); Assert.assertTrue(lsOut.contains("(empty)")); } From 65ad9f375b810597779e3196f15756242dcd3dcd Mon Sep 17 00:00:00 2001 From: cstella Date: Tue, 27 Sep 2016 20:23:16 -0400 Subject: [PATCH 04/11] Added GET_LIST functions. --- metron-platform/metron-management/README.md | 10 ++++ .../management/FileSystemFunctions.java | 52 +++++++++++++++++++ .../management/FileSystemFunctionsTest.java | 20 +++++-- 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/metron-platform/metron-management/README.md b/metron-platform/metron-management/README.md index 6f53f18ab9..30e9b45ae9 100644 --- a/metron-platform/metron-management/README.md +++ b/metron-platform/metron-management/README.md @@ -44,6 +44,11 @@ The functions are split roughly into a few sections: * Input: * path - The path of the file * Returns: The contents of the file and null otherwise. + * `FILE_GET_LIST` + * Description: Retrieves the contents of a file as a list of strings. + * Input: + * path - The path of the file + * Returns: A list of lines * `FILE_PUT` * Description: Writes the contents of a string to a local file * Input: @@ -67,6 +72,11 @@ The functions are split roughly into a few sections: * Input: * path - The path of the file * Returns: The contents of the file and null otherwise. + * `HDFS_GET_LIST` + * Description: Retrieves the contents of a HDFS file as a list of strings. + * Input: + * path - The path of the file + * Returns: A list of lines * `HDFS_PUT` * Description: Writes the contents of a string to a HDFS file * Input: diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java index 7090563728..4dea25428e 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java @@ -17,6 +17,8 @@ */ package org.apache.metron.management; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; import com.jakewharton.fliptables.FlipTable; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; @@ -85,6 +87,31 @@ public boolean isInitialized() { } + static class FileSystemGetList extends FileSystemFunction { + + FileSystemGetList(FileSystemGetter getter) { + super(getter); + } + + @Override + public Object apply(List args, Context context) throws ParseException { + String path = (String) args.get(0); + if(path == null) { + return null; + } + try(FSDataInputStream is = fs.open(new Path(path))) { + List out = new ArrayList<>(); + Iterable lines = Splitter.on("\n").split(IOUtils.toString(is)); + Iterables.addAll(out, lines); + return out; + } catch (IOException e) { + String message = "Unable to read " + path + ": " + e.getMessage(); + LOG.error(message, e); + return null; + } + } + } + static class FileSystemGet extends FileSystemFunction { FileSystemGet(FileSystemGetter getter) { @@ -350,4 +377,29 @@ public FileGet() { } } + @Stellar(namespace="HDFS" + ,name="GET_LIST" + ,description="Retrieves the contents of a HDFS file as a list of strings." + ,params = { "path - The path in HDFS of the file."} + ,returns = "A list of lines" + + ) + public static class HDFSGetList extends FileSystemGetList { + public HDFSGetList() { + super(FS_TYPE.HDFS); + } + } + + @Stellar(namespace="FILE" + ,name="GET_LIST" + ,description="Retrieves the contents of a file as a list of strings." + ,params = { "path - The path of the file."} + ,returns = "A list of lines" + + ) + public static class FileGetList extends FileSystemGetList { + public FileGetList() { + super(FS_TYPE.LOCAL); + } + } } diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java index 6ace70eae3..f165f9cacc 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java @@ -36,10 +36,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; +import java.util.*; @RunWith(Parameterized.class) public class FileSystemFunctionsTest { @@ -50,6 +47,7 @@ public class FileSystemFunctionsTest { private String prefix; private Context context = null; private FileSystemFunctions.FileSystemGet get; + private FileSystemFunctions.FileSystemGetList getList; private FileSystemFunctions.FileSystemLs ls; private FileSystemFunctions.FileSystemPut put; private FileSystemFunctions.FileSystemRm rm; @@ -89,6 +87,8 @@ public void setup() throws IOException { get = new FileSystemFunctions.FileSystemGet(fsGetter); get.initialize(null); + getList = new FileSystemFunctions.FileSystemGetList(fsGetter); + getList.initialize(null); ls = new FileSystemFunctions.FileSystemLs(fsGetter); ls.initialize(null); put = new FileSystemFunctions.FileSystemPut(fsGetter); @@ -122,6 +122,18 @@ public void testHappyPath() { Assert.assertTrue(lsOut.contains("(empty)")); } + @Test + public void testGetList() { + Object putOut = put.apply(Arrays.asList("foo\nbar", prefix + "testPut.dat"), null); + Assert.assertTrue((Boolean) putOut); + String getOut = (String)get.apply(Arrays.asList(prefix + "testPut.dat"), null); + Assert.assertEquals("foo\nbar", getOut); + List list = (List) getList.apply(Arrays.asList(prefix + "testPut.dat"), null); + Assert.assertEquals(2,list.size()); + Assert.assertEquals("foo",list.get(0)); + Assert.assertEquals("bar",list.get(1)); + } + @Test public void testPutMissingFile() { Object o = put.apply(Arrays.asList("foo", null), null); From 5452557952f97af11118d2d9e37dbb1bc97b42f4 Mon Sep 17 00:00:00 2001 From: cstella Date: Tue, 27 Sep 2016 20:45:21 -0400 Subject: [PATCH 05/11] Updating function to use IOUtils --- .../org/apache/metron/management/FileSystemFunctions.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java index 4dea25428e..61ab5c35c3 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java @@ -100,10 +100,7 @@ public Object apply(List args, Context context) throws ParseException { return null; } try(FSDataInputStream is = fs.open(new Path(path))) { - List out = new ArrayList<>(); - Iterable lines = Splitter.on("\n").split(IOUtils.toString(is)); - Iterables.addAll(out, lines); - return out; + return IOUtils.readLines(is); } catch (IOException e) { String message = "Unable to read " + path + ": " + e.getMessage(); LOG.error(message, e); From 919eba12e3c7ad365bddd5ddf86a0840e827b6d9 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 28 Sep 2016 10:24:49 -0400 Subject: [PATCH 06/11] Updated naming to change FILE -> LOCAL and GET/PUT to READ/WRITE --- metron-platform/metron-management/README.md | 16 +++++++------- .../management/FileSystemFunctions.java | 22 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/metron-platform/metron-management/README.md b/metron-platform/metron-management/README.md index 30e9b45ae9..c432283c55 100644 --- a/metron-platform/metron-management/README.md +++ b/metron-platform/metron-management/README.md @@ -28,28 +28,28 @@ The functions are split roughly into a few sections: ### File Functions * Local Files - * `FILE_LS` + * `LOCAL_LS` * Description: Lists the contents of a directory. * Input: * path - The path of the file * Returns: The contents of the directory in tabular form sorted by last modification date. - * `FILE_RM` + * `LOCAL_RM` * Description: Removes the path * Input: * path - The path of the file or directory. * recursive - Recursively remove or not (optional and defaulted to false) * Returns: boolean - true if successful, false otherwise - * `FILE_GET` + * `LOCAL_READ` * Description: Retrieves the contents as a string of a file. * Input: * path - The path of the file * Returns: The contents of the file and null otherwise. - * `FILE_GET_LIST` + * `LOCAL_READ_LIST` * Description: Retrieves the contents of a file as a list of strings. * Input: * path - The path of the file * Returns: A list of lines - * `FILE_PUT` + * `LOCAL_WRITE` * Description: Writes the contents of a string to a local file * Input: * content - The content to write out @@ -67,17 +67,17 @@ The functions are split roughly into a few sections: * path - The path of the file or directory. * recursive - Recursively remove or not (optional and defaulted to false) * Returns: boolean - true if successful, false otherwise - * `HDFS_GET` + * `HDFS_READ` * Description: Retrieves the contents as a string of a file in HDFS. * Input: * path - The path of the file * Returns: The contents of the file and null otherwise. - * `HDFS_GET_LIST` + * `HDFS_READ_LIST` * Description: Retrieves the contents of a HDFS file as a list of strings. * Input: * path - The path of the file * Returns: A list of lines - * `HDFS_PUT` + * `HDFS_WRITE` * Description: Writes the contents of a string to a HDFS file * Input: * content - The content to write out diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java index 61ab5c35c3..83feedc3b9 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java @@ -258,7 +258,7 @@ public Object apply(List args, Context context) throws ParseException { } } - @Stellar(namespace="FILE" + @Stellar(namespace="LOCAL" ,name="RM" ,description="Removes the path" ,params = { "path - The path of the file or directory." @@ -288,7 +288,7 @@ public HDFSRm() { } } - @Stellar(namespace="FILE" + @Stellar(namespace="LOCAL" ,name="LS" ,description="Lists the contents of a directory" ,params = { "path - The path of the file." @@ -317,7 +317,7 @@ public HDFSLs() { } @Stellar(namespace="HDFS" - ,name="PUT" + ,name="WRITE" ,description="Writes the contents a string to a file in HDFS." ,params = { "content - The content to write out" , "path - The path in HDFS of the file." @@ -333,8 +333,8 @@ public HDFSPut() { } } - @Stellar(namespace="FILE" - ,name="PUT" + @Stellar(namespace="LOCAL" + ,name="WRITE" ,description="Writes the contents a string to a local file." ,params = { "content - The content to write out" , "path - The path of the file." @@ -349,7 +349,7 @@ public FilePut() { } @Stellar(namespace="HDFS" - ,name="GET" + ,name="READ" ,description="Retrieves the contents as a string of a file in HDFS." ,params = { "path - The path in HDFS of the file."} ,returns = "The contents of the file in the path from HDFS and null otherwise." @@ -361,8 +361,8 @@ public HDFSGet() { } } - @Stellar(namespace="FILE" - ,name="GET" + @Stellar(namespace="LOCAL" + ,name="READ" ,description="Retrieves the contents as a string of a file on the local filesystem." ,params = { "path - The path of the file."} ,returns = "The contents of the file or null otherwise." @@ -375,7 +375,7 @@ public FileGet() { } @Stellar(namespace="HDFS" - ,name="GET_LIST" + ,name="READ_LIST" ,description="Retrieves the contents of a HDFS file as a list of strings." ,params = { "path - The path in HDFS of the file."} ,returns = "A list of lines" @@ -387,8 +387,8 @@ public HDFSGetList() { } } - @Stellar(namespace="FILE" - ,name="GET_LIST" + @Stellar(namespace="LOCAL" + ,name="READ_LIST" ,description="Retrieves the contents of a file as a list of strings." ,params = { "path - The path of the file."} ,returns = "A list of lines" From 2ba8179df3ce0b6714ea0f73ae648fa5e87a427c Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 28 Sep 2016 10:55:37 -0400 Subject: [PATCH 07/11] Made LS default to ~ --- .../management/FileSystemFunctions.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java index 83feedc3b9..d8f2c456e9 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java @@ -209,15 +209,29 @@ protected DateFormat initialValue() { @Override public Object apply(List args, Context context) throws ParseException { - - String path = (String) args.get(0); - if(path == null) { - return false; + Path path = null; + if(args.size() == 0) { + path = fs.getHomeDirectory(); + } + else { + String pathStr = (String) args.get(0); + if (path == null) { + return false; + } + else { + try { + path = new Path(pathStr); + } + catch(IllegalArgumentException iae) { + LOG.error("Unable to convert " + pathStr + " to a path: " + iae.getMessage(), iae); + return false; + } + } } try { String[] headers = new String[] {"PERMISSION", "OWNER", "GROUP", "SIZE", "LAST MOD TIME", "NAME"}; List dataList = new ArrayList<>(); - for(FileStatus status : fs.listStatus(new Path(path))) { + for(FileStatus status : fs.listStatus(path)) { dataList.add(new String[] { status.getPermission().toString() ,status.getOwner() From 44a28b1160b2cff0be0a1c0be266b8c183741fe4 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 28 Sep 2016 11:01:47 -0400 Subject: [PATCH 08/11] typo --- .../apache/metron/management/FileSystemFunctions.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java index d8f2c456e9..eafc432a38 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java @@ -210,13 +210,14 @@ protected DateFormat initialValue() { @Override public Object apply(List args, Context context) throws ParseException { Path path = null; + String[] headers = new String[] {"PERMISSION", "OWNER", "GROUP", "SIZE", "LAST MOD TIME", "NAME"}; if(args.size() == 0) { path = fs.getHomeDirectory(); } else { String pathStr = (String) args.get(0); - if (path == null) { - return false; + if (pathStr == null) { + return FlipTable.of(headers, new String[][]{}); } else { try { @@ -224,12 +225,11 @@ public Object apply(List args, Context context) throws ParseException { } catch(IllegalArgumentException iae) { LOG.error("Unable to convert " + pathStr + " to a path: " + iae.getMessage(), iae); - return false; + return FlipTable.of(headers, new String[][]{}); } } } try { - String[] headers = new String[] {"PERMISSION", "OWNER", "GROUP", "SIZE", "LAST MOD TIME", "NAME"}; List dataList = new ArrayList<>(); for(FileStatus status : fs.listStatus(path)) { dataList.add(new String[] { @@ -267,7 +267,7 @@ public Object apply(List args, Context context) throws ParseException { } catch (IOException e) { String message = "Unable to list" + path + ": " + e.getMessage(); LOG.error(message, e); - return null; + return FlipTable.of(headers, new String[][]{}); } } } From dd3e3d63c2851feb8b0b66428ed24e3917bcdff1 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 28 Sep 2016 11:03:02 -0400 Subject: [PATCH 09/11] Changing to READ_LINES --- .../org/apache/metron/management/FileSystemFunctions.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java index eafc432a38..42a2a2a238 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java @@ -389,7 +389,7 @@ public FileGet() { } @Stellar(namespace="HDFS" - ,name="READ_LIST" + ,name="READ_LINES" ,description="Retrieves the contents of a HDFS file as a list of strings." ,params = { "path - The path in HDFS of the file."} ,returns = "A list of lines" @@ -402,7 +402,7 @@ public HDFSGetList() { } @Stellar(namespace="LOCAL" - ,name="READ_LIST" + ,name="READ_LINES" ,description="Retrieves the contents of a file as a list of strings." ,params = { "path - The path of the file."} ,returns = "A list of lines" From a2116b2c37ba9af48545fbba0d737329f6586ee0 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 28 Sep 2016 11:03:56 -0400 Subject: [PATCH 10/11] Changing to READ_LINES --- metron-platform/metron-management/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-platform/metron-management/README.md b/metron-platform/metron-management/README.md index c432283c55..49da9b9945 100644 --- a/metron-platform/metron-management/README.md +++ b/metron-platform/metron-management/README.md @@ -44,7 +44,7 @@ The functions are split roughly into a few sections: * Input: * path - The path of the file * Returns: The contents of the file and null otherwise. - * `LOCAL_READ_LIST` + * `LOCAL_READ_LINES` * Description: Retrieves the contents of a file as a list of strings. * Input: * path - The path of the file @@ -72,7 +72,7 @@ The functions are split roughly into a few sections: * Input: * path - The path of the file * Returns: The contents of the file and null otherwise. - * `HDFS_READ_LIST` + * `HDFS_READ_LINES` * Description: Retrieves the contents of a HDFS file as a list of strings. * Input: * path - The path of the file From f396b85ce940cec0d1c5cc8237c58eaa6eb3c72f Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 28 Sep 2016 11:08:22 -0400 Subject: [PATCH 11/11] Typo --- .../org/apache/metron/management/FileSystemFunctions.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java index 42a2a2a238..15a87dd922 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java @@ -254,7 +254,7 @@ public Object apply(List args, Context context) throws ParseException { return ret; } } catch (java.text.ParseException e) { - String message = "Unable to parse " + o1 + " or " + o2 + ": " + e.getMessage(); + String message = "Unable to parse " + o1 + " or " + o2 + " : " + e.getMessage(); LOG.error(message, e); throw new IllegalStateException(message, e); } @@ -265,7 +265,7 @@ public Object apply(List args, Context context) throws ParseException { } return FlipTable.of(headers, data); } catch (IOException e) { - String message = "Unable to list" + path + ": " + e.getMessage(); + String message = "Unable to list" + path + " : " + e.getMessage(); LOG.error(message, e); return FlipTable.of(headers, new String[][]{}); }