Improve error handling of WriteableDiskCheck.

Addresses the issues that were brought up in #131

hydra-data/src/main/java/com/addthis/hydra/util/StreamGobbler.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.addthis.hydra.util;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class StreamGobbler extends Thread {
+
+    private static final Logger log = LoggerFactory.getLogger(StreamGobbler.class);
+
+    private final InputStream is;
+    private final OutputStream os;
+    private final BufferedReader reader;
+
+    public StreamGobbler(String name, InputStream is, OutputStream os) {
+        super(name);
+        this.is = is;
+        this.os = os;
+        InputStreamReader isr = new InputStreamReader(is);
+        reader = new BufferedReader(isr);
+    }
+
+    public void close() {
+        try {
+            reader.close();
+        } catch (IOException ex) {
+            log.error("Exception while attempting to close reader: ", ex);
+        }
+    }
+
+    public void run() {
+        try {
+            PrintWriter pw = new PrintWriter(os);
+            String line;
+            while ( (line = reader.readLine()) != null) {
+                pw.println(line);
+            }
+            pw.flush();
+            pw.close();
+        } catch (IOException ex) {
+            log.error("Exception while attempting to read from stream: ", ex);
+        }
+    }
+}

hydra-data/src/main/java/com/addthis/hydra/util/WriteableDiskCheck.java

@@ -13,33 +13,71 @@
  */
 package com.addthis.hydra.util;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.IOException;
 
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 public class WriteableDiskCheck extends MeteredHealthCheck {
 
-    private List<File> checkedFiles;
+    private static final Logger log = LoggerFactory.getLogger(WriteableDiskCheck.class);
+
+    private final List<File> checkedFiles;
 
     public WriteableDiskCheck(int maxFails, List<File> checkedFiles) {
         super(maxFails, "touch_disk_failure", TimeUnit.MINUTES);
         this.checkedFiles = checkedFiles;
     }
 
+    private static final String OUTPUT_THREAD_NAME = WriteableDiskCheck.class.getName() + " stdout";
+    private static final String ERROR_THREAD_NAME = WriteableDiskCheck.class.getName() + " stderr";
+
+    private static final int DEFAULT_WAIT_SECONDS = 30;
+
     @Override
     public boolean check() {
+
         for (File file : this.checkedFiles) {
             String[] cmdarray = {"touch", file.getAbsolutePath()};
             try {
                 Process process = Runtime.getRuntime().exec(cmdarray);
-                if (process.waitFor() == 0) {
-                    return true;
+                ByteArrayOutputStream output = new ByteArrayOutputStream();
+                ByteArrayOutputStream error = new ByteArrayOutputStream();
+                StreamGobbler outputThread = new StreamGobbler(OUTPUT_THREAD_NAME,
+                                                               process.getInputStream(),
+                                                               output);
+                StreamGobbler errorThread = new StreamGobbler(ERROR_THREAD_NAME,
+                                                               process.getErrorStream(),
+                                                               error);
+                outputThread.start();
+                errorThread.start();
+                int status = process.waitFor();
+                outputThread.join(TimeUnit.SECONDS.toMillis(DEFAULT_WAIT_SECONDS));
+                errorThread.join(TimeUnit.SECONDS.toMillis(DEFAULT_WAIT_SECONDS));
+                outputThread.close();
+                errorThread.close();
+                /**
+                 * From the very helpful Texinfo page on touch:
+                 * "An exit status of zero indicates success, and a nonzero value
+                 * indicates failure."
+                 */
+                if (status != 0) {
+                    log.error("Process 'touch {}' returned non-zero exit status {}. " +
+                              "Stdout was '{}' and stderr was '{}'", file.getAbsolutePath(),
+                              status, output.toString(), error.toString());
+                    return false;
                 }
-            } catch (Exception e) {
-                // keep trying files until one succeeds, or all fail
+            } catch (IOException|InterruptedException ex) {
+                log.error("Exception occurred attempting to execute process 'touch {}'",
+                          file.getAbsolutePath(), ex);
+                return false;
             }
         }
-        return false;
+        return true;
     }
 }

hydra-data/src/test/java/com/addthis/hydra/util/TestStreamGobbler.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.addthis.hydra.util;
+
+import java.io.ByteArrayOutputStream;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestStreamGobbler {
+
+    @Test
+    public void readProcessOutput() throws Exception {
+        String[] cmdarray = {"echo", "hello world"};
+            Process process = Runtime.getRuntime().exec(cmdarray);
+            ByteArrayOutputStream output = new ByteArrayOutputStream();
+            ByteArrayOutputStream error = new ByteArrayOutputStream();
+            StreamGobbler outputThread = new StreamGobbler(TestStreamGobbler.class.getName(),
+                                                           process.getInputStream(),
+                                                           output);
+            StreamGobbler errorThread = new StreamGobbler(TestStreamGobbler.class.getName(),
+                                                          process.getErrorStream(),
+                                                          error);
+            outputThread.start();
+            errorThread.start();
+            int status = process.waitFor();
+            outputThread.join(TimeUnit.SECONDS.toMillis(5));
+            errorThread.join(TimeUnit.SECONDS.toMillis(5));
+            outputThread.close();
+            errorThread.close();
+            assertEquals(0, status);
+            assertEquals("hello world", output.toString().trim());
+            assertEquals("", error.toString().trim());
+        }
+}