More error reporting and stats for ingestion tasks

api/src/main/java/io/druid/data/input/impl/MapInputRowParser.java

@@ -72,7 +72,7 @@ public List<InputRow> parseBatch(Map<String, Object> theMap)
       }
     }
     catch (Exception e) {
-      throw new ParseException(e, "Unparseable timestamp found!");
+      throw new ParseException(e, "Unparseable timestamp found! Event: " + theMap);
     }
 
     return ImmutableList.of(new MapBasedInputRow(timestamp.getMillis(), dimensions, theMap));

api/src/main/java/io/druid/indexer/IngestionState.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.indexer;
+
+public enum IngestionState
+{
+  NOT_STARTED,
+  DETERMINE_PARTITIONS,
+  BUILD_SEGMENTS,
+  COMPLETED
+}

api/src/main/java/io/druid/indexer/TaskMetricsGetter.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.indexer;
+
+import java.util.List;
+import java.util.Map;
+
+public interface TaskMetricsGetter
+{
+  List<String> getKeys();
+  Map<String, Double> getMetrics();
+}

api/src/main/java/io/druid/indexer/TaskMetricsUtils.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.indexer;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
+public class TaskMetricsUtils
+{
+  public static final String ROWS_PROCESSED = "rowsProcessed";
+  public static final String ROWS_PROCESSED_WITH_ERRORS = "rowsProcessedWithErrors";
+  public static final String ROWS_UNPARSEABLE = "rowsUnparseable";
+  public static final String ROWS_THROWN_AWAY = "rowsThrownAway";
+
+  public static Map<String, Object> makeIngestionRowMetrics(
+      long rowsProcessed,
+      long rowsProcessedWithErrors,
+      long rowsUnparseable,
+      long rowsThrownAway
+  )
+  {
+    Map<String, Object> metricsMap = Maps.newHashMap();
+    metricsMap.put(ROWS_PROCESSED, rowsProcessed);
+    metricsMap.put(ROWS_PROCESSED_WITH_ERRORS, rowsProcessedWithErrors);
+    metricsMap.put(ROWS_UNPARSEABLE, rowsUnparseable);
+    metricsMap.put(ROWS_THROWN_AWAY, rowsThrownAway);
+    return metricsMap;
+  }
+}

api/src/main/java/io/druid/indexer/TaskStatusPlus.java

@@ -25,6 +25,7 @@
 import org.joda.time.DateTime;
 
 import javax.annotation.Nullable;
+import java.util.Map;
 import java.util.Objects;
 
 public class TaskStatusPlus
@@ -38,6 +39,15 @@ public class TaskStatusPlus
   private final TaskLocation location;
   private final String dataSource;
 
+  @Nullable
+  private final Map<String, Object> metrics;
+
+  @Nullable
+  private final String errorMsg;
+
+  @Nullable
+  private final Map<String, Object> context;
+
   @JsonCreator
   public TaskStatusPlus(
       @JsonProperty("id") String id,
@@ -47,7 +57,10 @@ public TaskStatusPlus(
       @JsonProperty("statusCode") @Nullable TaskState state,
       @JsonProperty("duration") @Nullable Long duration,
       @JsonProperty("location") TaskLocation location,
-      @JsonProperty("dataSource") String dataSource
+      @JsonProperty("dataSource") String dataSource,
+      @JsonProperty("metrics") Map<String, Object> metrics,
+      @JsonProperty("errorMsg") String errorMsg,
+      @JsonProperty("context") Map<String, Object> context
   )
   {
     if (state != null && state.isComplete()) {
@@ -61,6 +74,9 @@ public TaskStatusPlus(
     this.duration = duration;
     this.location = Preconditions.checkNotNull(location, "location");
     this.dataSource = dataSource;
+    this.metrics = metrics;
+    this.errorMsg = errorMsg;
+    this.context = context;
   }
 
   @JsonProperty
@@ -108,6 +124,27 @@ public TaskLocation getLocation()
     return location;
   }
 
+  @Nullable
+  @JsonProperty("metrics")
+  public Map<String, Object> getMetrics()
+  {
+    return metrics;
+  }
+
+  @Nullable
+  @JsonProperty("errorMsg")
+  public String getErrorMsg()
+  {
+    return errorMsg;
+  }
+
+  @Nullable
+  @JsonProperty("context")
+  public Map<String, Object> getContext()
+  {
+    return context;
+  }
+
   @Override
   public boolean equals(Object o)
   {
@@ -138,13 +175,37 @@ public boolean equals(Object o)
     if (!Objects.equals(duration, that.duration)) {
       return false;
     }
-    return location.equals(that.location);
+
+    if (!Objects.equals(location, that.location)) {
+      return false;
+    }
+
+    if (!Objects.equals(errorMsg, that.errorMsg)) {
+      return false;
+    }
+
+    if (!Objects.equals(location, that.location)) {
+      return false;
+    }
+
+    return Objects.equals(context, that.context);
   }
 
   @Override
   public int hashCode()
   {
-    return Objects.hash(id, type, createdTime, queueInsertionTime, state, duration, location);
+    return Objects.hash(
+        id,
+        type,
+        createdTime,
+        queueInsertionTime,
+        state,
+        duration,
+        location,
+        metrics,
+        errorMsg,
+        context
+    );
   }
 
   @JsonProperty

api/src/main/java/io/druid/utils/CircularBuffer.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.utils;
+
+import com.google.common.base.Preconditions;
+
+public class CircularBuffer<E>
+{
+  public E[] getBuffer()
+  {
+    return buffer;
+  }
+
+  private final E[] buffer;
+
+  private int start = 0;
+  private int size = 0;
+
+  public CircularBuffer(int capacity)
+  {
+    buffer = (E[]) new Object[capacity];
+  }
+
+  public void add(E item)
+  {
+    buffer[start++] = item;
+
+    if (start >= buffer.length) {
+      start = 0;
+    }
+
+    if (size < buffer.length) {
+      size++;
+    }
+  }
+
+  public E getLatest(int index)
+  {
+    int bufferIndex = start - index - 1;
+    if (bufferIndex < 0) {
+      bufferIndex = buffer.length + bufferIndex;
+    }
+    return buffer[bufferIndex];
+  }
+
+  public E get(int index)
+  {
+    Preconditions.checkArgument(index >= 0 && index < size, "invalid index");
+
+    int bufferIndex = (start - size + index) % buffer.length;
+    if (bufferIndex < 0) {
+      bufferIndex += buffer.length;
+    }
+    return buffer[bufferIndex];
+  }
+
+  public int size()
+  {
+    return size;
+  }
+}

api/src/test/java/io/druid/indexer/TaskStatusPlusTest.java

@@ -53,7 +53,10 @@ public void testSerde() throws IOException
         TaskState.RUNNING,
         1000L,
         TaskLocation.create("testHost", 1010, -1),
-        "ds_test"
+        "ds_test",
+        null,
+        null,
+        null
     );
     final String json = mapper.writeValueAsString(status);
     Assert.assertEquals(status, mapper.readValue(json, TaskStatusPlus.class));

benchmarks/src/main/java/io/druid/benchmark/FilteredAggregatorBenchmark.java

@@ -255,7 +255,7 @@ public void ingest(Blackhole blackhole) throws Exception
   {
     incIndexFilteredAgg = makeIncIndex(filteredMetrics);
     for (InputRow row : inputRows) {
-      int rv = incIndexFilteredAgg.add(row);
+      int rv = incIndexFilteredAgg.add(row).getRowCount();
       blackhole.consume(rv);
     }
   }

benchmarks/src/main/java/io/druid/benchmark/IncrementalIndexRowTypeBenchmark.java

@@ -161,7 +161,7 @@ public void normalLongs(Blackhole blackhole) throws Exception
   {
     for (int i = 0; i < maxRows; i++) {
       InputRow row = longRows.get(i);
-      int rv = incIndex.add(row);
+      int rv = incIndex.add(row).getRowCount();
       blackhole.consume(rv);
     }
   }
@@ -174,7 +174,7 @@ public void normalFloats(Blackhole blackhole) throws Exception
   {
     for (int i = 0; i < maxRows; i++) {
       InputRow row = floatRows.get(i);
-      int rv = incFloatIndex.add(row);
+      int rv = incFloatIndex.add(row).getRowCount();
       blackhole.consume(rv);
     }
   }
@@ -187,7 +187,7 @@ public void normalStrings(Blackhole blackhole) throws Exception
   {
     for (int i = 0; i < maxRows; i++) {
       InputRow row = stringRows.get(i);
-      int rv = incStrIndex.add(row);
+      int rv = incStrIndex.add(row).getRowCount();
       blackhole.consume(rv);
     }
   }

benchmarks/src/main/java/io/druid/benchmark/indexing/IndexIngestionBenchmark.java

@@ -120,7 +120,7 @@ public void addRows(Blackhole blackhole) throws Exception
   {
     for (int i = 0; i < rowsPerSegment; i++) {
       InputRow row = rows.get(i);
-      int rv = incIndex.add(row);
+      int rv = incIndex.add(row).getRowCount();
       blackhole.consume(rv);
     }
   }

common/src/main/java/io/druid/indexer/Jobby.java

@@ -19,9 +19,24 @@
 
 package io.druid.indexer;
 
+import javax.annotation.Nullable;
+import java.util.Map;
+
 /**
  */
 public interface Jobby
 {
   boolean run();
+
+  @Nullable
+  default Map<String, Object> getStats()
+  {
+    throw new UnsupportedOperationException("This Jobby does not implement getJobStats().");
+  }
+
+  @Nullable
+  default String getErrorMessage()
+  {
+    throw new UnsupportedOperationException("This Jobby does not implement getErrorMessage().");
+  }
 }