Make LogHandler configurable

container-core/abi-spec.json

@@ -94,24 +94,11 @@
       "public"
     ],
     "methods": [
-      "public void <init>(java.util.concurrent.Executor)",
-      "protected void <init>(java.util.concurrent.Executor, com.yahoo.container.handler.LogReader)",
+      "public void <init>(java.util.concurrent.Executor, com.yahoo.container.core.LogHandlerConfig)",
       "public com.yahoo.container.jdisc.HttpResponse handle(com.yahoo.container.jdisc.HttpRequest)"
     ],
     "fields": []
   },
-  "com.yahoo.container.handler.LogReader": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>()",
-      "protected org.json.JSONObject readLogs(java.lang.String, long, long)"
-    ],
-    "fields": []
-  },
   "com.yahoo.container.handler.Prefix": {
     "superClass": "java.lang.Object",
     "interfaces": [

container-core/src/main/java/com/yahoo/container/handler/LogHandler.java

@@ -2,6 +2,7 @@
 package com.yahoo.container.handler;
 
 import com.google.inject.Inject;
+import com.yahoo.container.core.LogHandlerConfig;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
@@ -11,21 +12,20 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
-import java.util.Arrays;
-import java.util.HashMap;
+import java.time.Instant;
+import java.util.Optional;
 import java.util.concurrent.Executor;
 
 public class LogHandler extends ThreadedHttpRequestHandler {
 
-    private static final String LOG_DIRECTORY = "/home/y/logs/vespa/logarchive/";
     private final LogReader logReader;
 
     @Inject
-    public LogHandler(Executor executor) {
-        this(executor, new LogReader());
+    public LogHandler(Executor executor, LogHandlerConfig config) {
+        this(executor, new LogReader(config.logDirectory(), config.logPattern()));
     }
 
-    protected LogHandler(Executor executor, LogReader logReader) {
+    LogHandler(Executor executor, LogReader logReader) {
         super(executor);
         this.logReader = logReader;
     }
@@ -34,18 +34,21 @@ protected LogHandler(Executor executor, LogReader logReader) {
     public HttpResponse handle(HttpRequest request) {
         JSONObject responseJSON = new JSONObject();
 
-        HashMap<String, String> apiParams = getParameters(request);
-        long earliestLogThreshold = getEarliestThreshold(apiParams);
-        long latestLogThreshold = getLatestThreshold(apiParams);
+        Instant earliestLogThreshold = Optional.ofNullable(request.getProperty("from"))
+                .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MIN);
+        Instant latestLogThreshold = Optional.ofNullable(request.getProperty("to"))
+                .map(Long::valueOf).map(Instant::ofEpochMilli).orElseGet(Instant::now);
+
         try {
-            JSONObject logJson = logReader.readLogs(LOG_DIRECTORY, earliestLogThreshold, latestLogThreshold);
+            JSONObject logJson = logReader.readLogs(earliestLogThreshold, latestLogThreshold);
             responseJSON.put("logs", logJson);
         } catch (IOException | JSONException e) {
             return new HttpResponse(404) {
                 @Override
                 public void render(OutputStream outputStream) {}
             };
         }
+
         return new HttpResponse(200) {
             @Override
             public void render(OutputStream outputStream) throws IOException {
@@ -55,28 +58,4 @@ public void render(OutputStream outputStream) throws IOException {
             }
         };
     }
-
-    private HashMap<String, String> getParameters(HttpRequest request) {
-        String query = request.getUri().getQuery();
-        HashMap<String, String> keyValPair = new HashMap<>();
-        Arrays.stream(query.split("&")).forEach(pair -> {
-            String[] splitPair = pair.split("=");
-            keyValPair.put(splitPair[0], splitPair[1]);
-        });
-        return keyValPair;
-    }
-
-    private long getEarliestThreshold(HashMap<String, String> map) {
-        if (map.containsKey("from")) {
-            return Long.valueOf(map.get("from"));
-        }
-        return Long.MIN_VALUE;
-    }
-
-    private long getLatestThreshold(HashMap<String, String> map) {
-        if (map.containsKey("to")) {
-            return Long.valueOf(map.get("to"));
-        }
-        return Long.MAX_VALUE;
-    }
 }

container-core/src/main/java/com/yahoo/container/handler/LogReader.java

@@ -1,40 +1,58 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.container.handler;
 
+import com.yahoo.vespa.defaults.Defaults;
 import org.json.JSONException;
 import org.json.JSONObject;
 
-import java.time.Duration;
-import java.util.Base64;
-import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Base64;
+import java.util.Iterator;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+class LogReader {
 
-public class LogReader {
+    private final Path logDirectory;
+    private final Pattern logFilePattern;
 
-    long earliestLogThreshold;
-    long latestLogThreshold;
+    LogReader(String logDirectory, String logFilePattern) {
+        this(Paths.get(Defaults.getDefaults().underVespaHome(logDirectory)), Pattern.compile(logFilePattern));
+    }
+
+    LogReader(Path logDirectory, Pattern logFilePattern) {
+        this.logDirectory = logDirectory;
+        this.logFilePattern = logFilePattern;
+    }
 
-    protected JSONObject readLogs(String logDirectory, long earliestLogThreshold, long latestLogThreshold) throws IOException, JSONException {
-        this.earliestLogThreshold = earliestLogThreshold;
-        this.latestLogThreshold = latestLogThreshold + Duration.ofMinutes(5).toMillis(); // Add some time to allow retrieving logs currently being modified
+    JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws IOException, JSONException {
         JSONObject json = new JSONObject();
-        File root = new File(logDirectory);
-        traverse_folder(root, json, "");
+        latestLogThreshold = latestLogThreshold.plus(Duration.ofMinutes(5)); // Add some time to allow retrieving logs currently being modified
+        traverseFolder(logDirectory, json, earliestLogThreshold, latestLogThreshold, "");
         return json;
     }
 
-    private void traverse_folder(File root, JSONObject json, String filename) throws IOException, JSONException {
-        File[] files = root.listFiles();
-        for(File child : files) {
-            long logTime = Files.getLastModifiedTime(child.toPath()).toMillis();
-            if(child.isFile() && earliestLogThreshold < logTime && logTime < latestLogThreshold) {
-                json.put(filename + child.getName(), Base64.getEncoder().encodeToString(Files.readAllBytes(child.toPath())));
-            }
-            else if (!child.isFile()){
-                traverse_folder(child, json, filename + child.getName() + "-");
+    private void traverseFolder(Path path, JSONObject json, Instant earliestLogThreshold, Instant latestLogThreshold, String filenamePrefix) throws IOException, JSONException {
+        try (Stream<Path> files = Files.list(path)) {
+            for (Iterator<Path> it = files.iterator(); it.hasNext(); ) {
+                Path child = it.next();
+                String filename = child.getFileName().toString();
+                Instant lastModified = Files.getLastModifiedTime(child).toInstant();
+                if (Files.isRegularFile(child)) {
+                    if (lastModified.isAfter(earliestLogThreshold) &&
+                            lastModified.isBefore(latestLogThreshold) &&
+                            logFilePattern.matcher(filename).matches()) {
+                        json.put(filenamePrefix + filename, Base64.getEncoder().encodeToString(Files.readAllBytes(child)));
+                    }
+                } else {
+                    traverseFolder(child, json, earliestLogThreshold, latestLogThreshold, filenamePrefix + filename + "-");
+                }
             }
         }
     }
-
 }

container-core/src/main/resources/configdefinitions/log-handler.def

@@ -0,0 +1,9 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+namespace=container.core
+
+## Path to log directory, can be relative or absolute.
+## Relative paths will be resolved relative to $VESPA_HOME
+logDirectory string default="logs/vespa/logarchive/"
+
+## File name regex of logs to include
+logPattern string default=".*"

container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java

@@ -9,9 +9,10 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.time.Instant;
 import java.util.concurrent.Executor;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
 import static org.mockito.Mockito.mock;
 
 public class LogHandlerTest {
@@ -43,9 +44,13 @@ public void handleCorrectlyParsesQueryParameters() throws IOException {
     }
 
     class MockLogReader extends LogReader {
+        MockLogReader() {
+            super("", "");
+        }
+
         @Override
-        protected JSONObject readLogs(String logDirectory, long earliestLogThreshold, long latestLogThreshold) throws JSONException {
-            if(latestLogThreshold > 1000) {
+        protected JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws JSONException {
+            if (latestLogThreshold.isAfter(Instant.ofEpochMilli(1000))) {
                 return new JSONObject("{\"one\":\"newer_log\"}");
             } else {
                 return new JSONObject("{\"two\":\"older_log\"}");

container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java

@@ -2,40 +2,43 @@
 package com.yahoo.container.handler;
 
 import org.json.JSONObject;
-import org.junit.Before;
 import org.junit.Test;
 
-import java.io.ByteArrayOutputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.time.Instant;
+import java.util.regex.Pattern;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
 
 public class LogReaderTest {
 
-    ByteArrayOutputStream outputStream;
-
-    @Before
-    public void setup() {
-        outputStream = new ByteArrayOutputStream();
-    }
+    private final Path logDirectory = Paths.get("src/test/resources/logfolder/");
 
     @Test
     public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{
-        String logDirectory = "src/test/resources/logfolder/";
-        LogReader logReader = new LogReader();
-        JSONObject json = logReader.readLogs(logDirectory, 21, Instant.now().toEpochMilli());
+        LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
+        JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now());
         String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\",\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}";
         String actual = json.toString();
         assertEquals(expected, actual);
     }
 
     @Test
     public void testThatLogsOutsideRangeAreExcluded() throws Exception {
-        String logDirectory = "src/test/resources/logfolder/";
-        LogReader logReader = new LogReader();
-        JSONObject json = logReader.readLogs(logDirectory, Long.MAX_VALUE, Long.MIN_VALUE);
+        LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
+        JSONObject json = logReader.readLogs(Instant.MAX, Instant.MIN);
         String expected = "{}";
         String actual = json.toString();
         assertEquals(expected, actual);
     }
+
+    @Test
+    public void testThatLogsNotMatchingRegexAreExcluded() throws Exception {
+        LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*2\\.log"));
+        JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now());
+        String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\"}";
+        String actual = json.toString();
+        assertEquals(expected, actual);
+    }
 }