More error reporting and stats for ingestion tasks (apache#5418)

* Add more indexing task status and error reporting

* PR comments, add support in AppenderatorDriverRealtimeIndexTask

* Use TaskReport instead of metrics/context

* Fix tests

* Use TaskReport uploads

* Refactor fire department metrics retrieval

* Refactor input row serde in hadoop task

* Refactor hadoop task loader names

* Truncate error message in TaskStatus, add errorMsg to task report

* PR comments

api/src/main/java/io/druid/data/input/impl/MapInputRowParser.java

@@ -72,7 +72,7 @@ public List<InputRow> parseBatch(Map<String, Object> theMap)
       }
     }
     catch (Exception e) {
-      throw new ParseException(e, "Unparseable timestamp found!");
+      throw new ParseException(e, "Unparseable timestamp found! Event: %s", theMap);
     }
 
     return ImmutableList.of(new MapBasedInputRow(timestamp.getMillis(), dimensions, theMap));

api/src/main/java/io/druid/indexer/IngestionState.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.indexer;
+
+public enum IngestionState
+{
+  NOT_STARTED,
+  DETERMINE_PARTITIONS,
+  BUILD_SEGMENTS,
+  COMPLETED
+}

api/src/main/java/io/druid/indexer/TaskMetricsGetter.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.indexer;
+
+import java.util.List;
+import java.util.Map;
+
+public interface TaskMetricsGetter
+{
+  List<String> getKeys();
+  Map<String, Number> getTotalMetrics();
+}

api/src/main/java/io/druid/indexer/TaskMetricsUtils.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.indexer;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
+public class TaskMetricsUtils
+{
+  public static final String ROWS_PROCESSED = "rowsProcessed";
+  public static final String ROWS_PROCESSED_WITH_ERRORS = "rowsProcessedWithErrors";
+  public static final String ROWS_UNPARSEABLE = "rowsUnparseable";
+  public static final String ROWS_THROWN_AWAY = "rowsThrownAway";
+
+  public static Map<String, Object> makeIngestionRowMetrics(
+      long rowsProcessed,
+      long rowsProcessedWithErrors,
+      long rowsUnparseable,
+      long rowsThrownAway
+  )
+  {
+    Map<String, Object> metricsMap = Maps.newHashMap();
+    metricsMap.put(ROWS_PROCESSED, rowsProcessed);
+    metricsMap.put(ROWS_PROCESSED_WITH_ERRORS, rowsProcessedWithErrors);
+    metricsMap.put(ROWS_UNPARSEABLE, rowsUnparseable);
+    metricsMap.put(ROWS_THROWN_AWAY, rowsThrownAway);
+    return metricsMap;
+  }
+}

api/src/main/java/io/druid/indexer/TaskStatusPlus.java

@@ -38,6 +38,7 @@ public class TaskStatusPlus
   private final Long duration;
   private final TaskLocation location;
   private final String dataSource;
+
   @Nullable
   private final String errorMsg;
 

api/src/main/java/io/druid/utils/CircularBuffer.java

@@ -0,0 +1,92 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.utils;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A circular buffer that supports random bidirectional access.
+ *
+ * @param <E> Type of object to be stored in the buffer
+ */
+public class CircularBuffer<E>
+{
+  public E[] getBuffer()
+  {
+    return buffer;
+  }
+
+  private final E[] buffer;
+
+  private int start = 0;
+  private int size = 0;
+
+  public CircularBuffer(int capacity)
+  {
+    Preconditions.checkArgument(capacity > 0, "Capacity must be greater than 0.");
+    buffer = (E[]) new Object[capacity];
+  }
+
+  public void add(E item)
+  {
+    buffer[start++] = item;
+
+    if (start >= buffer.length) {
+      start = 0;
+    }
+
+    if (size < buffer.length) {
+      size++;
+    }
+  }
+
+  /**
+   * Access object at a given index, starting from the latest entry added and moving backwards.
+   */
+  public E getLatest(int index)
+  {
+    Preconditions.checkArgument(index >= 0 && index < size, "invalid index");
+
+    int bufferIndex = start - index - 1;
+    if (bufferIndex < 0) {
+      bufferIndex = buffer.length + bufferIndex;
+    }
+    return buffer[bufferIndex];
+  }
+
+  /**
+   * Access object at a given index, starting from the earliest entry added and moving forward.
+   */
+  public E get(int index)
+  {
+    Preconditions.checkArgument(index >= 0 && index < size, "invalid index");
+
+    int bufferIndex = (start - size + index) % buffer.length;
+    if (bufferIndex < 0) {
+      bufferIndex += buffer.length;
+    }
+    return buffer[bufferIndex];
+  }
+
+  public int size()
+  {
+    return size;
+  }
+}

benchmarks/src/main/java/io/druid/benchmark/FilteredAggregatorBenchmark.java

@@ -257,7 +257,7 @@ public void ingest(Blackhole blackhole) throws Exception
   {
     incIndexFilteredAgg = makeIncIndex(filteredMetrics);
     for (InputRow row : inputRows) {
-      int rv = incIndexFilteredAgg.add(row);
+      int rv = incIndexFilteredAgg.add(row).getRowCount();
       blackhole.consume(rv);
     }
   }

benchmarks/src/main/java/io/druid/benchmark/IncrementalIndexRowTypeBenchmark.java

@@ -162,7 +162,7 @@ public void normalLongs(Blackhole blackhole) throws Exception
   {
     for (int i = 0; i < maxRows; i++) {
       InputRow row = longRows.get(i);
-      int rv = incIndex.add(row);
+      int rv = incIndex.add(row).getRowCount();
       blackhole.consume(rv);
     }
   }
@@ -175,7 +175,7 @@ public void normalFloats(Blackhole blackhole) throws Exception
   {
     for (int i = 0; i < maxRows; i++) {
       InputRow row = floatRows.get(i);
-      int rv = incFloatIndex.add(row);
+      int rv = incFloatIndex.add(row).getRowCount();
       blackhole.consume(rv);
     }
   }
@@ -188,7 +188,7 @@ public void normalStrings(Blackhole blackhole) throws Exception
   {
     for (int i = 0; i < maxRows; i++) {
       InputRow row = stringRows.get(i);
-      int rv = incStrIndex.add(row);
+      int rv = incStrIndex.add(row).getRowCount();
       blackhole.consume(rv);
     }
   }

benchmarks/src/main/java/io/druid/benchmark/indexing/IndexIngestionBenchmark.java

@@ -120,7 +120,7 @@ public void addRows(Blackhole blackhole) throws Exception
   {
     for (int i = 0; i < rowsPerSegment; i++) {
       InputRow row = rows.get(i);
-      int rv = incIndex.add(row);
+      int rv = incIndex.add(row).getRowCount();
       blackhole.consume(rv);
     }
   }

common/src/main/java/io/druid/indexer/Jobby.java

@@ -19,9 +19,37 @@
 
 package io.druid.indexer;
 
+import io.druid.java.util.common.StringUtils;
+
+import javax.annotation.Nullable;
+import java.util.Map;
+
 /**
  */
 public interface Jobby
 {
   boolean run();
+
+  /**
+   * @return A map containing statistics for a Jobby, optionally null if the Jobby is unable to provide stats.
+   */
+  @Nullable
+  default Map<String, Object> getStats()
+  {
+    throw new UnsupportedOperationException(
+        StringUtils.format("This Jobby does not implement getJobStats(), Jobby class: [%s]", getClass())
+    );
+  }
+
+  /**
+   * @return A string representing the error that caused a Jobby to fail. Can be null if the Jobby did not fail,
+   * or is unable to provide an error message.
+   */
+  @Nullable
+  default String getErrorMessage()
+  {
+    throw new UnsupportedOperationException(
+        StringUtils.format("This Jobby does not implement getErrorMessage(), Jobby class: [%s]", getClass())
+    );
+  }
 }

extensions-contrib/orc-extensions/src/test/java/io/druid/data/input/orc/OrcIndexGeneratorJobTest.java

@@ -233,6 +233,8 @@ public void setUp() throws Exception
                 null,
                 false,
                 false,
+                null,
+                null,
                 null
             )
         )
@@ -251,7 +253,7 @@ public void testIndexGeneratorJob() throws IOException
 
   private void verifyJob(IndexGeneratorJob job) throws IOException
   {
-    JobHelper.runJobs(ImmutableList.<Jobby>of(job), config);
+    Assert.assertTrue(JobHelper.runJobs(ImmutableList.<Jobby>of(job), config));
 
     int segmentNum = 0;
     for (DateTime currTime = interval.getStart(); currTime.isBefore(interval.getEnd()); currTime = currTime.plusDays(1)) {