Merge 98eebfc into c76fb3e

.settings/org.eclipse.jdt.core.prefs

@@ -9,15 +9,103 @@ org.eclipse.jdt.core.codeComplete.staticFieldPrefixes=
 org.eclipse.jdt.core.codeComplete.staticFieldSuffixes=
 org.eclipse.jdt.core.codeComplete.staticFinalFieldPrefixes=
 org.eclipse.jdt.core.codeComplete.staticFinalFieldSuffixes=
+org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=enabled
+org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore
+org.eclipse.jdt.core.compiler.annotation.nonnull=javax.annotation.Nonnull
+org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
+org.eclipse.jdt.core.compiler.annotation.nullable=javax.annotation.Nullable
+org.eclipse.jdt.core.compiler.annotation.nullanalysis=enabled
 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
 org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
 org.eclipse.jdt.core.compiler.compliance=1.7
 org.eclipse.jdt.core.compiler.debug.lineNumber=generate
 org.eclipse.jdt.core.compiler.debug.localVariable=generate
 org.eclipse.jdt.core.compiler.debug.sourceFile=generate
+org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
 org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
+org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning
+org.eclipse.jdt.core.compiler.problem.deadCode=warning
+org.eclipse.jdt.core.compiler.problem.deprecation=warning
+org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
+org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
+org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
+org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
 org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
+org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
+org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
+org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
+org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
+org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=error
+org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
+org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=enabled
+org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
+org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
+org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
+org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
+org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
+org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
+org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
+org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
+org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore
+org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
+org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled
+org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning
+org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore
+org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
+org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
+org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
+org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning
+org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
+org.eclipse.jdt.core.compiler.problem.nullReference=error
+org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
+org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=ignore
+org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
+org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
+org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
+org.eclipse.jdt.core.compiler.problem.potentialNullReference=error
+org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore
+org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning
+org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning
+org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
+org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore
+org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore
+org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore
+org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore
+org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled
+org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning
+org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled
+org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled
+org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=enabled
+org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore
+org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning
+org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled
+org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning
+org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning
+org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore
+org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning
+org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore
+org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore
+org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore
+org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore
+org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled
+org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled
+org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled
+org.eclipse.jdt.core.compiler.problem.unusedImport=warning
+org.eclipse.jdt.core.compiler.problem.unusedLabel=warning
+org.eclipse.jdt.core.compiler.problem.unusedLocal=warning
+org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore
+org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore
+org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled
+org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled
+org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled
+org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
+org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore
+org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
+org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
 org.eclipse.jdt.core.compiler.source=1.7
 org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
 org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16

speedtest/edu/washington/escience/myria/TupleBatchBuildingTest.java

@@ -74,8 +74,8 @@ public void buildMutableTB() {
     long start = System.nanoTime();
     for (long i = 0; i < testSize; i++) {
       for (int j = 0; j < TupleBatch.BATCH_SIZE; j++) {
-        icb1.replace(j, icb1Source[j]);
-        icb2.replace(j, icb2Source[j]);
+        icb1.replaceLong(icb1Source[j], j);
+        icb2.replaceLong(icb2Source[j], j);
       }
       totalCount += icb1.size();
     }

speedtest/edu/washington/escience/myria/mrbenchmarks/GlobalAvg.java

@@ -46,7 +46,7 @@ protected TupleBatch fetchNextReady() throws DbException {
       DoubleColumnBuilder rc = new DoubleColumnBuilder();
       rc.expandAll();
       for (int idx = 0; idx < tb.numTuples(); ++idx) {
-        rc.replace(idx, inputColumns.get(sumIdx).getLong(idx) * 1.0 / inputColumns.get(countIdx).getLong(idx));
+        rc.replaceDouble(inputColumns.get(sumIdx).getLong(idx) * 1.0 / inputColumns.get(countIdx).getLong(idx), idx);
       }
 
       ImmutableList.Builder<Column<?>> newColumnsB = ImmutableList.builder();

speedtest/edu/washington/escience/myria/mrbenchmarks/SubStr.java

@@ -46,7 +46,7 @@ protected TupleBatch fetchNextReady() throws DbException {
       ImmutableList<Column<?>> source = tb.getDataColumns();
       for (int idx = 0; idx < tb.numTuples(); ++idx) {
         String subStr = source.get(substrColumnIdx).getString(idx).substring(fromCharIdx, endCharIdx);
-        builder.replace(idx, subStr);
+        builder.replaceString(subStr, idx);
       }
 
       StringColumn sc = builder.build();

src/edu/washington/escience/myria/accessmethod/SQLiteAccessMethod.java

@@ -170,19 +170,14 @@ public Iterator<TupleBatch> tupleBatchIteratorFromQuery(final String queryString
      * is released pretty fast. However if another connection comes during the exclusive lock period, a
      * "database is locked" exception will still be thrown. The following code simply tries to call prepare again.
      */
-    boolean conflict = true;
     int count = 0;
-    while (conflict) {
-      conflict = false;
+    Throwable cause = null;
+    while (statement == null && count < MAX_RETRY_ATTEMPTS) {
       try {
         statement = sqliteConnection.prepare(queryString);
       } catch (final SQLiteException e) {
-        conflict = true;
+        cause = e;
         count++;
-        if (count >= MAX_RETRY_ATTEMPTS) {
-          LOGGER.error(e.getMessage(), e);
-          throw new DbException(e);
-        }
         try {
           Thread.sleep(MyriaConstants.SHORT_WAITING_INTERVAL_10_MS);
         } catch (InterruptedException e1) {
@@ -192,6 +187,11 @@ public Iterator<TupleBatch> tupleBatchIteratorFromQuery(final String queryString
       }
     }
 
+    if (statement == null) {
+      LOGGER.error("SQLite database maximum retry attempts exceeded", cause);
+      throw new DbException(cause);
+    }
+
     try {
       /* Step the statement once so we can figure out the Schema */
       statement.step();

src/edu/washington/escience/myria/api/DatasetResource.java

@@ -361,6 +361,7 @@ private Response doIngest(final RelationKey relationKey, final Operator source,
       status = server.ingestDataset(relationKey, actualWorkers, indexes, source);
     } catch (InterruptedException e) {
       Thread.currentThread().interrupt();
+      return Response.status(Status.SERVICE_UNAVAILABLE).entity("Interrupted").build();
     }
 
     /* In the response, tell the client the path to the relation. */

src/edu/washington/escience/myria/client/JsonQueryBaseBuilder.java

@@ -14,6 +14,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
@@ -158,6 +159,7 @@ public static int nextOpNumber() {
   /**
    * The workers on which the current operator is going to run, i.e. the operator partition.
    */
+  @Nonnull
   private final Set<Integer> runOnWorkers;
   /**
    * An operator is going to run on any single worker.
@@ -210,10 +212,11 @@ public static int nextOpNumber() {
    * @param workers2 worker set 2
    * @return compatible worker set.
    */
+  @Nullable
   private static Set<Integer> workerSetAlgebra(@Nonnull final Set<Integer> workers1,
       @Nonnull final Set<Integer> workers2) {
-    Preconditions.checkNotNull(workers1);
-    Preconditions.checkNotNull(workers2);
+    Preconditions.checkNotNull(workers1, "workers1");
+    Preconditions.checkNotNull(workers2, "workers2");
 
     if (workers1 == ANY_SINGLE_WORKER) {
       if (workers2 == ANY_SINGLE_WORKER || workers2 == NO_PREFERENCE) {
@@ -297,23 +300,24 @@ private JsonQueryBaseBuilder(final OperatorEncoding<?> currentOp, final String[]
     parents = new HashSet<JsonQueryBaseBuilder>();
 
     Set<Integer> childrenWorkers = NO_PREFERENCE;
-    for (final JsonQueryBaseBuilder c : children) {
+    for (final JsonQueryBaseBuilder c : this.children) {
       childrenWorkers = workerSetAlgebra(childrenWorkers, c.runOnWorkers);
       if (childrenWorkers == null) {
         throw new IllegalArgumentException("Workers of a child are not compatible with other children. Current op: "
             + getOpName(this) + ", conflicting child: " + getOpName(c));
       }
     }
     if (compatibleWithChildrenWorkers) {
-      runOnWorkers = workerSetAlgebra(childrenWorkers, runningWorkers);
-      if (runOnWorkers == null) {
-        String[] childrenNames = new String[children.length];
-        for (int i = 0; i < children.length; i++) {
-          childrenNames[i] = getOpName(children[i]);
+      Set<Integer> workers = workerSetAlgebra(childrenWorkers, runningWorkers);
+      if (workers == null) {
+        String[] childrenNames = new String[this.children.length];
+        for (int i = 0; i < this.children.length; i++) {
+          childrenNames[i] = getOpName(this.children[i]);
         }
         throw new IllegalArgumentException("Running workers are not compatible with children workers. Current op: "
             + getOpName(this) + ", children: " + StringUtils.join(childrenNames, ','));
       }
+      runOnWorkers = workers;
     } else {
       runOnWorkers = runningWorkers;
     }

src/edu/washington/escience/myria/column/builder/BooleanColumnBuilder.java

@@ -128,24 +128,10 @@ public BooleanMutableColumn buildMutable() {
   }
 
   @Override
-  @Deprecated
-  public BooleanColumnBuilder replace(final int idx, final Boolean value) throws IndexOutOfBoundsException {
-    return replace(idx, value.booleanValue());
-  }
-
-  /**
-   * Replace the specified element.
-   * 
-   * @param value element to be inserted.
-   * @param idx where to insert the element.
-   * @return this column builder.
-   * @throws IndexOutOfBoundsException if the idx exceeds the currently valid indices, i.e. the currently built size.
-   */
-  public BooleanColumnBuilder replace(final int idx, final boolean value) throws IndexOutOfBoundsException {
+  public void replaceBoolean(final boolean value, final int row) throws IndexOutOfBoundsException {
     Preconditions.checkArgument(!built, "No further changes are allowed after the builder has built the column.");
-    Preconditions.checkElementIndex(idx, numBits);
-    data.set(idx, value);
-    return this;
+    Preconditions.checkElementIndex(row, numBits);
+    data.set(row, value);
   }
 
   @Override

src/edu/washington/escience/myria/column/builder/ColumnBuilder.java

@@ -4,6 +4,8 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
+import javax.annotation.Nonnull;
+
 import org.joda.time.DateTime;
 
 import com.almworks.sqlite4java.SQLiteException;
@@ -12,12 +14,14 @@
 import edu.washington.escience.myria.column.Column;
 import edu.washington.escience.myria.column.mutable.MutableColumn;
 import edu.washington.escience.myria.storage.ReadableColumn;
+import edu.washington.escience.myria.storage.ReplaceableColumn;
 
 /**
  * @param <T> type of the objects in this column.
  * 
  */
-public abstract class ColumnBuilder<T extends Comparable<?>> implements ReadableColumn, WritableColumn {
+public abstract class ColumnBuilder<T extends Comparable<?>> implements ReadableColumn, WritableColumn,
+    ReplaceableColumn {
 
   @Override
   public ColumnBuilder<T> appendBoolean(final boolean value) throws BufferOverflowException {
@@ -144,14 +148,38 @@ public abstract ColumnBuilder<T> appendFromSQLite(SQLiteStatement statement, int
    */
   public abstract ColumnBuilder<T> forkNewBuilder();
 
-  /**
-   * Replace the specified element.
-   * 
-   * @param value element to be inserted.
-   * @param idx where to insert the element.
-   * @return this column builder.
-   * @throws IndexOutOfBoundsException if the idx exceeds the currently valid indices, i.e. the currently built size.
-   */
-  public abstract ColumnBuilder<T> replace(int idx, T value) throws IndexOutOfBoundsException;
+  @Override
+  public void replaceBoolean(final boolean value, final int row) {
+    throw new UnsupportedOperationException(getClass().getName());
+  }
+
+  @Override
+  public void replaceDateTime(@Nonnull final DateTime value, final int row) {
+    throw new UnsupportedOperationException(getClass().getName());
+  }
+
+  @Override
+  public void replaceDouble(final double value, final int row) {
+    throw new UnsupportedOperationException(getClass().getName());
+  }
 
+  @Override
+  public void replaceFloat(final float value, final int row) {
+    throw new UnsupportedOperationException(getClass().getName());
+  }
+
+  @Override
+  public void replaceInt(final int value, final int row) {
+    throw new UnsupportedOperationException(getClass().getName());
+  }
+
+  @Override
+  public void replaceLong(final long value, final int row) {
+    throw new UnsupportedOperationException(getClass().getName());
+  }
+
+  @Override
+  public void replaceString(@Nonnull final String value, final int row) {
+    throw new UnsupportedOperationException(getClass().getName());
+  }
 }

src/edu/washington/escience/myria/column/builder/DateTimeColumnBuilder.java

@@ -124,12 +124,11 @@ public DateTimeMutableColumn buildMutable() {
   }
 
   @Override
-  public DateTimeColumnBuilder replace(final int idx, final DateTime value) throws IndexOutOfBoundsException {
+  public void replaceDateTime(final DateTime value, final int row) throws IndexOutOfBoundsException {
     Preconditions.checkState(!built, "No further changes are allowed after the builder has built the column.");
-    Preconditions.checkElementIndex(idx, numDates);
+    Preconditions.checkElementIndex(row, numDates);
     Preconditions.checkNotNull(value);
-    data[idx] = value;
-    return this;
+    data[row] = value;
   }
 
   @Override

src/edu/washington/escience/myria/column/builder/DoubleColumnBuilder.java

@@ -131,24 +131,10 @@ public DoubleMutableColumn buildMutable() {
   }
 
   @Override
-  @Deprecated
-  public DoubleColumnBuilder replace(final int idx, final Double value) throws IndexOutOfBoundsException {
-    return replace(idx, value.doubleValue());
-  }
-
-  /**
-   * Replace the specified element.
-   * 
-   * @param value element to be inserted.
-   * @param idx where to insert the element.
-   * @return this column builder.
-   * @throws IndexOutOfBoundsException if the idx exceeds the currently valid indices, i.e. the currently built size.
-   */
-  public DoubleColumnBuilder replace(final int idx, final double value) throws IndexOutOfBoundsException {
+  public void replaceDouble(final double value, final int row) throws IndexOutOfBoundsException {
     Preconditions.checkArgument(!built, "No further changes are allowed after the builder has built the column.");
-    Preconditions.checkElementIndex(idx, data.position());
-    data.put(idx, value);
-    return this;
+    Preconditions.checkElementIndex(row, data.position());
+    data.put(row, value);
   }
 
   @Override